From e41e59871cec2f042e2ea6cc031675cd18523106 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:56:52 +0000 Subject: [PATCH 01/10] refactor: move the front layer of the Python bindings into sqlbuild-python-front Discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the consumption, observation and audit result state tables move into a new crate between sqlbuild-python-base and sqlbuild-python, one area per helper bucket. sqlbuild-python imports their entry points directly; the extension module surface is unchanged. --- Cargo.lock | 32 +- Cargo.toml | 5 +- crates/README.md | 5 +- crates/sqlbuild-python-front/Cargo.toml | 37 + .../attachments/_helpers}/argument_values.rs | 3 +- .../_helpers}/ceremonial_select.rs | 2 +- .../_helpers}/cursor_intrinsics.rs | 9 +- .../attachments/_helpers}/function_headers.rs | 0 .../src/attachments/_helpers/mod.rs | 5 + .../_helpers}/reference_markers.rs | 0 .../src/attachments/constants.rs | 7 + .../main/analysis_cursor_splices.rs | 13 + .../attachments/main/analysis_cursor_sql.rs | 17 + .../src/attachments/main/argument_pairs.rs | 13 + .../main/assert_no_unresolved_sql_markers.rs | 17 + .../attachments/main/compile_input_error.rs | 12 + .../attachments/main/config_argument_value.rs | 13 + .../src/attachments/main/mod.rs | 13 + .../src/attachments/main/parsed_header.rs | 17 + .../main/register_ceremonial_select.rs | 12 + .../main/register_cursor_intrinsics.rs | 12 + .../main/register_reference_markers.rs | 12 + .../main/reserved_cursor_markers.rs | 9 + .../main/resolve_cursor_intrinsics.rs | 16 + .../main/validated_model_cursor_sql.rs | 17 + .../src/attachments/mod.rs | 5 + .../src/authored_values/_helpers}/fields.rs | 0 .../src/authored_values/_helpers/mod.rs | 4 + .../authored_values/_helpers}/project_vars.rs | 7 + .../authored_values/_helpers}/python_nodes.rs | 2 +- .../authored_values/_helpers}/sql_values.rs | 27 +- .../src/authored_values/main/mod.rs | 10 + .../authored_values/main/project_var_text.rs | 12 + .../src/authored_values/main/raw_value.rs | 14 + .../authored_values/main/register_fields.rs | 12 + .../main/register_project_vars.rs | 12 + .../main/register_python_nodes.rs | 12 + .../main/register_sql_values.rs | 12 + .../main/render_project_var_text.rs | 12 + .../main/render_project_var_text_function.rs | 11 + .../src/authored_values/main/sql_value.rs | 17 + .../authored_values/main/sql_value_error.rs | 13 + .../src/authored_values/mod.rs | 5 + .../src/authored_values/models.rs | 33 + .../command_runtime/_helpers}/concurrency.rs | 7 +- .../src/command_runtime/_helpers/mod.rs | 1 + .../src/command_runtime/main/mod.rs | 2 + .../main/resolve_concurrency.rs | 17 + .../main/resolve_configured_concurrency.rs | 21 + .../src/command_runtime/mod.rs | 4 + .../_helpers}/effective_config.rs | 66 +- .../src/compile_attachment/_helpers/mod.rs | 1 + .../main/build_effective_connection.rs | 22 + .../main/build_effective_connection_config.rs | 22 + ...ld_effective_connection_config_function.rs | 11 + .../build_effective_connection_function.rs | 11 + .../main/build_effective_runtime_config.rs | 17 + ...build_effective_runtime_config_function.rs | 11 + .../main/build_effective_settings.rs | 15 + .../main/build_effective_settings_config.rs | 14 + ...uild_effective_settings_config_function.rs | 11 + .../main/build_effective_settings_function.rs | 11 + .../main/build_effective_target_config.rs | 15 + .../build_effective_target_config_function.rs | 11 + .../main/build_effective_vars.rs | 17 + .../main/build_effective_vars_function.rs | 11 + .../src/compile_attachment/main/mod.rs | 16 + .../compile_attachment/main/project_vars.rs | 15 + .../main/target_selection.rs | 15 + .../src/compile_attachment/mod.rs | 4 + .../_helpers}/config_validation.rs | 18 +- .../src/compile_driver/_helpers/mod.rs | 1 + .../src/compile_driver/main/mod.rs | 3 + ...reject_invalid_connection_configuration.rs | 13 + .../reject_invalid_project_configuration.rs | 11 + .../main/session_warehouse_support.rs | 17 + .../src/compile_driver/mod.rs | 5 + .../src/compile_driver/models.rs | 12 + .../contracts/_helpers}/effective_config.rs | 2 +- .../src/contracts/_helpers/mod.rs | 2 + .../contracts/_helpers}/spec_resolution.rs | 5 +- .../main/effective_config_settings.rs | 20 + .../contracts/main/effective_config_vars.rs | 19 + .../main/effective_connection_fields.rs | 25 + .../main/loader_destination_parts.rs | 18 + .../src/contracts/main/loader_parts.rs | 17 + .../main/matching_dynamic_families.rs | 16 + .../src/contracts/main/mod.rs | 13 + .../main/register_effective_config.rs | 12 + .../main/register_spec_resolution.rs | 12 + .../main/resolve_effective_adapter_name.rs | 15 + .../resolve_effective_collection_rendering.rs | 15 + .../main/resolve_effective_scenario_config.rs | 15 + .../contracts/main/resolve_target_config.rs | 18 + .../src/contracts/main/resolve_target_name.rs | 19 + .../src/contracts/mod.rs | 4 + .../src/cursor/_helpers/conversion.rs | 11 + .../src/cursor/_helpers}/durations.rs | 9 +- .../src/cursor/_helpers}/intervals.rs | 4 +- .../src/cursor/_helpers/mod.rs | 6 + .../src/cursor/_helpers}/overrides.rs | 4 +- .../src/cursor/_helpers}/scalars.rs | 5 +- .../src/cursor/_helpers}/values.rs | 2 +- .../src/cursor/constants.rs | 15 + .../cursor/main/conversion/algebra_error.rs | 11 + .../src/cursor/main/conversion/mod.rs | 1 + .../src/cursor/main/durations/mod.rs | 4 + .../main/durations/parse_cursor_duration.rs | 16 + .../main/durations/register_durations.rs | 12 + .../durations/shift_by_cursor_duration.rs | 16 + .../cursor/main/durations/shifted_moment.rs | 17 + .../main/intervals/interval_validation.rs | 16 + .../src/cursor/main/intervals/mod.rs | 3 + .../src/cursor/main/intervals/partition.rs | 15 + .../main/intervals/register_intervals.rs | 12 + .../src/cursor/main/mod.rs | 6 + .../cursor/main/overrides/cursor_integer.rs | 12 + .../cursor/main/overrides/cursor_timestamp.rs | 12 + .../src/cursor/main/overrides/mod.rs | 2 + .../src/cursor/main/scalars/as_scalar.rs | 18 + .../src/cursor/main/scalars/compare_values.rs | 12 + .../src/cursor/main/scalars/mod.rs | 10 + .../src/cursor/main/scalars/parse.rs | 15 + .../src/cursor/main/scalars/parse_value.rs | 15 + .../cursor/main/scalars/register_scalars.rs | 12 + .../src/cursor/main/scalars/render_value.rs | 12 + .../cursor/main/scalars/sentinel_to_token.rs | 12 + .../src/cursor/main/scalars/to_exclusive.rs | 15 + .../src/cursor/main/scalars/to_inclusive.rs | 15 + .../src/cursor/main/scalars/try_parse.rs | 15 + .../src/cursor/main/values/local_date_time.rs | 13 + .../src/cursor/main/values/mod.rs | 2 + .../src/cursor/main/values/replaced.rs | 17 + .../sqlbuild-python-front/src/cursor/mod.rs | 6 + .../src/cursor/models.rs} | 115 +- .../discovery/_helpers/failures}/faults.rs | 0 .../discovery/_helpers/failures}/guidance.rs | 0 .../src/discovery/_helpers/failures/mod.rs | 3 + .../discovery/_helpers/failures}/raised.rs | 2 +- .../_helpers/headers}/header_declarations.rs | 7 +- .../_helpers/headers}/header_keys.rs | 0 .../_helpers/headers}/header_values.rs | 8 +- .../_helpers/headers}/hook_entries.rs | 0 .../discovery/_helpers/headers}/key_checks.rs | 0 .../src/discovery/_helpers/headers/mod.rs | 5 + .../src/discovery/_helpers/mod.rs | 7 + .../_helpers/project}/configuration.rs | 17 +- .../discovery/_helpers/project}/locations.rs | 0 .../src/discovery/_helpers/project/mod.rs | 6 + .../_helpers/project}/project_config.rs | 0 .../_helpers/project}/project_facts.rs | 18 +- .../_helpers/project}/project_files.rs | 21 +- .../_helpers/project}/project_validation.rs | 8 +- .../python_files}/imported_modules.rs | 4 +- .../python_files}/imported_providers.rs | 17 +- .../python_files}/imported_python_nodes.rs | 39 +- .../discovery/_helpers/python_files/mod.rs | 7 + .../python_files}/python_function_files.rs | 14 +- .../python_files}/python_hook_files.rs | 17 +- .../python_files}/python_module_files.rs | 8 +- .../_helpers/python_files}/python_paths.rs | 2 +- .../_helpers/records}/declaration_records.rs | 25 +- .../_helpers/records}/description_inputs.rs | 15 +- .../discovery/_helpers/records}/getters.rs | 22 +- .../src/discovery/_helpers/records/mod.rs | 7 + .../_helpers/records/model_file_record.rs | 383 +++++ .../_helpers/records}/model_records.rs | 27 +- .../_helpers/records}/sql_test_records.rs | 25 +- .../_helpers/records}/yaml_records.rs | 27 +- .../src/discovery/_helpers/snapshots/mod.rs | 3 + .../_helpers/snapshots/native_classes.rs | 55 + .../_helpers/snapshots}/scope_snapshot.rs | 31 +- .../discovery/_helpers/snapshots}/snapshot.rs | 8 +- .../_helpers/sources}/declaration_files.rs | 3 +- .../src/discovery/_helpers/sources/mod.rs | 6 + .../_helpers/sources}/orchestration_files.rs | 8 +- .../_helpers/sources}/orchestration_values.rs | 0 .../_helpers/sources}/provider_secrets.rs | 51 +- .../_helpers/sources}/sql_test_blocks.rs | 6 +- .../_helpers/sources}/yaml_entries.rs | 7 +- .../src/discovery/constants.rs | 42 + .../main/failures/explicit_references_help.rs | 9 + .../main/failures/microbatch_guidance.rs | 9 + .../src/discovery/main/failures/mod.rs | 6 + .../main/failures/register_guidance.rs | 12 + .../main/failures/register_raised.rs | 12 + .../discovery/main/failures/setting_help.rs | 13 + .../main/failures/sql_analysis_guidance.rs | 15 + .../discovery/main/headers/hook_entries.rs | 18 + .../src/discovery/main/headers/mod.rs | 5 + .../headers/register_header_declarations.rs | 12 + .../main/headers/register_header_values.rs | 12 + .../main/headers/register_hook_entries.rs | 12 + .../main/headers/register_key_checks.rs | 12 + .../src/discovery/main/mod.rs | 7 + .../src/discovery/main/project/column_name.rs | 12 + .../main/project/configured_changes.rs | 13 + .../main/project/configured_git_base_ref.rs | 12 + .../main/project/configured_review.rs | 13 + .../project/discover_configured_providers.rs | 18 + .../discover_connection_configuration.rs | 16 + .../project/discover_project_configuration.rs | 16 + .../src/discovery/main/project/each.rs | 16 + .../main/project/load_local_config_file.rs | 17 + .../main/project/load_project_config_file.rs | 17 + .../src/discovery/main/project/mod.rs | 23 + .../discovery/main/project/named_node_rows.rs | 13 + .../discovery/main/project/project_facts.rs | 19 + .../main/project/register_locations.rs | 12 + .../main/project/register_project_config.rs | 12 + .../main/project/register_project_files.rs | 12 + .../project/register_project_validation.rs | 12 + .../main/project/resolve_config_file_path.rs | 17 + .../main/project/seed_csv_settings.rs | 13 + .../src/discovery/main/project/stem.rs | 12 + .../project/validate_local_sql_analysis.rs | 17 + .../main/project/validation_error.rs | 12 + .../main/project/warn_deprecated_settings.rs | 13 + .../discovery/main/project/yaml_entry_line.rs | 9 + .../discover_event_exporter_functions.rs | 15 + .../python_files/discover_hook_functions.rs | 15 + .../discover_materialization_files.rs | 15 + .../python_files/discover_provider_classes.rs | 15 + .../discover_python_function_files.rs | 15 + .../discover_python_node_functions.rs | 15 + .../python_files/install_recall_writer.rs | 11 + .../src/discovery/main/python_files/mod.rs | 15 + .../main/python_files/node_identity.rs | 17 + .../main/python_files/provider_by_name.rs | 13 + .../main/python_files/provider_usages.rs | 16 + .../register_imported_python_nodes.rs | 12 + .../register_python_function_files.rs | 12 + .../register_python_hook_files.rs | 12 + .../register_python_module_files.rs | 12 + .../reject_unsupported_python_paths.rs | 12 + .../main/records/discover_adapter_file.rs | 12 + .../main/records/discover_audit_files.rs | 16 + .../main/records/discover_constant_files.rs | 17 + .../main/records/discover_enum_files.rs | 17 + .../main/records/discover_macro_files.rs | 16 + .../main/records/discover_model_files.rs | 17 + .../records/discover_model_schema_files.rs | 16 + .../main/records/discover_scenario_files.rs | 16 + .../main/records/discover_schema_files.rs | 15 + .../main/records/discover_seed_files.rs | 13 + .../main/records/discover_source_files.rs | 16 + .../records/discover_sql_function_files.rs | 16 + .../main/records/discover_sql_hook_files.rs | 16 + .../main/records/discover_test_files.rs | 17 + .../main/records/discovered_model_files.rs | 18 + .../records/externally_described_models.rs | 15 + .../src/discovery/main/records/mod.rs | 24 + .../main/records/parse_sql_test_file.rs | 16 + .../discovery/main/records/projected_text.rs | 18 + .../records/register_declaration_records.rs | 12 + .../records/register_description_inputs.rs | 12 + .../main/records/register_model_records.rs | 12 + .../main/records/register_sql_test_records.rs | 12 + .../main/records/register_yaml_records.rs | 12 + .../records/retained_discovery_session.rs | 14 + .../snapshots/directory_snapshot_scope.rs | 12 + .../main/snapshots/discover_scope_snapshot.rs | 14 + .../src/discovery/main/snapshots/escaped.rs | 12 + .../discovery/main/snapshots/in_snapshot.rs | 15 + .../src/discovery/main/snapshots/mod.rs | 6 + .../main/snapshots/register_scope_snapshot.rs | 12 + .../main/snapshots/register_snapshot.rs | 12 + .../discovery/main/sources/consumer_uses.rs | 12 + .../invocation_allows_missing_secrets.rs | 9 + .../src/discovery/main/sources/mod.rs | 8 + .../sources/orchestration_declarations.rs | 15 + .../discovery/main/sources/provider_uses.rs | 15 + .../sources/register_declaration_files.rs | 12 + .../main/sources/register_yaml_entries.rs | 12 + .../main/sources/require_used_secrets.rs | 15 + .../src/discovery/main/sources/sink_uses.rs | 12 + .../src/discovery/mod.rs | 6 + .../src/discovery/models.rs | 391 +++++ crates/sqlbuild-python-front/src/lib.rs | 21 + .../macro_bridge/_helpers}/first_compile.rs | 0 .../src/macro_bridge/_helpers/mod.rs | 3 + .../macro_bridge/_helpers}/module_digests.rs | 2 +- .../src/macro_bridge/_helpers}/store_keys.rs | 0 .../macro_bridge/main/context_store_token.rs | 16 + .../src/macro_bridge/main/keep_delta_claim.rs | 9 + .../macro_bridge/main/macro_store_token.rs | 16 + .../src/macro_bridge/main/mod.rs | 7 + .../main/register_first_compile.rs | 12 + .../main/register_module_digests.rs | 12 + .../macro_bridge/main/register_store_keys.rs | 12 + .../src/macro_bridge/main/text.rs | 13 + .../src/macro_bridge/mod.rs | 4 + .../_helpers/config}/config_classes.rs | 32 +- .../_helpers/config}/config_errors.rs | 4 +- .../_helpers/config}/config_nodes.rs | 35 +- .../_helpers/config/config_templates.rs | 211 +++ .../_helpers/config}/config_values.rs | 5 +- .../model_config/_helpers/config}/getters.rs | 12 +- .../src/model_config/_helpers/config/mod.rs | 8 + .../_helpers/config}/template_data.rs | 20 +- .../_helpers/config}/typed_config.rs | 28 +- .../_helpers/headers/header_audits.rs | 107 ++ .../_helpers/headers}/header_columns.rs | 11 +- .../_helpers/headers/header_families.rs | 24 + .../_helpers/headers}/header_metadata.rs | 329 +--- .../src/model_config/_helpers/headers/mod.rs | 4 + .../src/model_config/_helpers/mod.rs | 3 + .../_helpers/model_nodes/authored_nodes.rs | 10 + .../model_nodes}/column_migrations.rs | 0 .../_helpers/model_nodes}/contract_classes.rs | 0 .../_helpers/model_nodes}/described_models.rs | 13 +- .../model_config/_helpers/model_nodes/mod.rs | 8 + .../model_nodes/model_config_builder.rs | 275 +++ .../_helpers/model_nodes/model_validator.rs | 107 ++ .../_helpers/model_nodes}/project_defaults.rs | 0 .../model_nodes}/unicode_environment.rs | 0 .../src/model_config/constants.rs | 12 + .../src/model_config/errors.rs | 14 + .../model_config/main/config/carried_node.rs | 13 + .../main/config/config_cursor_bound.rs | 18 + .../model_config/main/config/config_fields.rs | 18 + .../model_config/main/config/config_flag.rs | 12 + .../main/config/config_integer_object.rs | 18 + .../model_config/main/config/config_node.rs | 13 + .../main/config/config_node_of.rs | 13 + .../model_config/main/config/config_object.rs | 14 + .../model_config/main/config/config_str.rs | 15 + .../model_config/main/config/config_text.rs | 16 + .../main/config/config_text_is.rs | 17 + .../main/config/config_text_object.rs | 18 + .../main/config/config_text_tuple.rs | 18 + .../model_config/main/config/config_value.rs | 18 + .../main/config/early_model_templates.rs | 16 + .../src/model_config/main/config/expanded.rs | 18 + .../main/config/expanded_effective_vars.rs | 15 + .../main/config/expanded_template_data.rs | 18 + .../src/model_config/main/config/mod.rs | 31 + .../model_config/main/config/model_values.rs | 13 + .../main/config/native_config_error.rs | 14 + .../main/config/raised_config_exception.rs | 15 + .../model_config/main/config/record_reads.rs | 14 + .../config/record_template_reads_function.rs | 11 + .../main/config/register_config_errors.rs | 12 + .../main/config/register_config_values.rs | 12 + .../main/config/register_template_data.rs | 12 + .../model_config/main/config/retain_values.rs | 13 + .../src/model_config/main/config/run_id.rs | 12 + .../main/config/template_error.rs | 11 + .../model_config/main/config/typed_values.rs | 13 + .../src/model_config/main/config/view.rs | 20 + .../model_config/main/headers/audit_tuple.rs | 20 + .../model_config/main/headers/family_specs.rs | 13 + .../src/model_config/main/headers/mod.rs | 8 + .../main/headers/native_column_record.rs | 19 + .../main/headers/native_schema_column.rs | 22 + .../main/headers/parsed_audit_identity.rs | 16 + .../main/headers/register_header_columns.rs | 12 + .../main/headers/schema_columns.rs | 21 + .../model_config/main/headers/thresholds.rs | 19 + .../src/model_config/main/mod.rs | 3 + .../model_nodes/column_migration_error.rs | 18 + .../main/model_nodes/contract_classes.rs | 13 + .../externally_described_models.rs | 16 + .../src/model_config/main/model_nodes/mod.rs | 9 + .../main/model_nodes/raised_config_error.rs | 12 + .../register_model_config_builder.rs | 12 + .../model_nodes/register_model_validator.rs | 12 + .../register_unicode_environment.rs | 12 + .../main/model_nodes/unicode_environment.rs | 13 + .../main/model_nodes/yaml_entry_classes.rs | 13 + .../src/model_config/mod.rs | 8 + .../src/model_config/models.rs | 1477 +++++++++++++++++ .../src/model_config/types.rs | 36 + .../_helpers}/audit_results_state.rs | 2 +- .../_helpers}/consumption_state.rs | 159 +- .../src/native_host/_helpers/mod.rs | 3 + .../_helpers}/observations_state.rs | 19 +- .../src/native_host/constants.rs | 4 + .../native_host/main/append_observations.rs | 19 + .../src/native_host/main/execute.rs | 16 + .../native_host/main/fixed_state_table_sql.rs | 20 + .../src/native_host/main/integer_literal.rs | 11 + .../src/native_host/main/literal.rs | 11 + .../src/native_host/main/mod.rs | 11 + .../src/native_host/main/optional_text.rs | 12 + .../native_host/main/recorder_callbacks.rs | 14 + .../main/register_audit_results_state.rs | 12 + .../main/register_consumption_state.rs | 12 + .../main/register_observations_state.rs | 12 + .../src/native_host/main/state_table_name.rs | 16 + .../src/native_host/mod.rs | 6 + .../src/native_host/models.rs | 170 ++ .../refactor/_helpers}/contract_compare.rs | 2 +- .../src/refactor/_helpers}/contract_edits.rs | 0 .../src/refactor/_helpers/mod.rs | 4 + .../src/refactor/_helpers}/python_strings.rs | 0 .../src/refactor/_helpers}/refactor_facts.rs | 11 +- .../src/refactor/main/compare_contracts.rs | 15 + .../refactor/main/declaration_placement.rs | 18 + .../src/refactor/main/edit_contract_model.rs | 18 + .../src/refactor/main/edit_contract_source.rs | 17 + .../src/refactor/main/mod.rs | 11 + .../src/refactor/main/model_facts.rs | 13 + .../src/refactor/main/refactor_facts.rs | 17 + .../main/register_contract_compare.rs | 12 + .../refactor/main/register_contract_edits.rs | 12 + .../refactor/main/register_python_strings.rs | 12 + .../main/remaining_contract_findings.rs | 19 + .../src/refactor/main/unit_test_rebindings.rs | 15 + .../sqlbuild-python-front/src/refactor/mod.rs | 4 + .../render/_helpers/calls}/macro_arguments.rs | 0 .../_helpers/calls}/macro_call_classes.rs | 5 +- .../render/_helpers/calls}/macro_call_memo.rs | 2 +- .../render/_helpers/calls}/macro_call_rows.rs | 16 +- .../src/render/_helpers/calls}/macro_calls.rs | 12 +- .../src/render/_helpers/calls/mod.rs | 5 + .../_helpers/declarations}/authored_sql.rs | 113 +- .../declarations}/declaration_expansion.rs | 69 +- .../declarations}/declaration_indexes.rs | 0 .../declarations}/declaration_values.rs | 51 +- .../src/render/_helpers/declarations/mod.rs | 4 + .../_helpers/expansion}/macro_evaluation.rs | 10 +- .../_helpers/expansion/macro_expansion.rs | 282 ++++ .../expansion}/macro_expansion_facts.rs | 2 +- .../expansion}/macro_expansion_result.rs | 29 +- .../expansion}/macro_generated_references.rs | 2 +- .../_helpers/expansion}/macro_modules.rs | 0 .../src/render/_helpers/expansion/mod.rs | 6 + .../src/render/_helpers/mod.rs | 4 + .../src/render/_helpers/sql_macros/mod.rs | 4 + .../_helpers/sql_macros}/sql_macro_host.rs | 40 +- .../sql_macros}/sql_macro_inventory.rs | 0 .../_helpers/sql_macros}/sql_macro_mocks.rs | 0 .../_helpers/sql_macros}/sql_macro_tests.rs | 0 .../src/render/main/calls/mod.rs | 6 + .../main/calls/register_macro_arguments.rs | 12 + .../render/main/calls/register_macro_calls.rs | 12 + .../src/render/main/calls/span_expansion.rs | 11 + .../src/render/main/calls/span_row.rs | 11 + .../src/render/main/calls/span_tuple.rs | 11 + .../main/calls/top_level_macro_call_names.rs | 18 + .../declarations/constant_declaration_sql.rs | 20 + .../declaration_scope_resolver.rs | 16 + .../main/declarations/expand_declarations.rs | 14 + .../main/declarations/expand_with_inputs.rs | 18 + .../src/render/main/declarations/mod.rs | 11 + .../public_declaration_indexes.rs | 15 + .../declarations/public_model_schema_index.rs | 13 + .../declarations/register_authored_sql.rs | 12 + .../register_declaration_indexes.rs | 12 + .../register_declaration_values.rs | 12 + .../declarations/render_enum_member_value.rs | 12 + .../src/render/main/declarations/span_rows.rs | 11 + .../src/render/main/expansion/expanded.rs | 18 + .../render/main/expansion/expansion_result.rs | 13 + .../main/expansion/macro_expansion_classes.rs | 13 + .../src/render/main/expansion/mod.rs | 6 + .../expansion/register_macro_expansion.rs | 12 + .../register_macro_expansion_result.rs | 12 + .../main/expansion/register_macro_modules.rs | 12 + .../src/render/main/mod.rs | 4 + .../src/render/main/sql_macros/mod.rs | 3 + .../register_sql_macro_inventory.rs | 12 + .../main/sql_macros/with_sql_macro_mocks.rs | 17 + .../sql_macros/without_with_list_calls.rs | 9 + .../sqlbuild-python-front/src/render/mod.rs | 6 + .../src/render/models.rs} | 668 ++++---- .../sqlbuild-python-front/src/render/types.rs | 20 + .../src/scopes/_helpers/checks/mod.rs | 3 + .../scopes/_helpers/checks}/scope_tolerant.rs | 12 +- .../_helpers/checks}/scope_validation.rs | 25 +- .../_helpers/checks}/scope_visibility.rs | 14 +- .../declarations/declaration_contexts.rs | 66 + .../declarations}/declaration_resolution.rs | 10 +- .../src/scopes/_helpers/declarations/mod.rs | 3 + .../_helpers/declarations}/placement.rs | 0 .../src/scopes/_helpers/index}/getters.rs | 12 +- .../src/scopes/_helpers/index/mod.rs | 5 + .../src/scopes/_helpers/index}/scope_cache.rs | 7 +- .../src/scopes/_helpers/index}/scope_index.rs | 157 +- .../scopes/_helpers/index}/scope_lookup.rs | 10 +- .../scopes/_helpers/index}/scope_objects.rs | 45 +- .../src/scopes/_helpers/mod.rs | 5 + .../_helpers/relationships}/compile_usages.rs | 2 - .../src/scopes/_helpers/relationships/mod.rs | 3 + .../relationships}/relationship_names.rs | 2 +- .../relationships}/relationship_scan.rs | 2 +- .../src/scopes/_helpers/reports/mod.rs | 3 + .../scopes/_helpers/reports}/scope_errors.rs | 0 .../_helpers/reports}/scope_report_objects.rs | 11 +- .../scopes/_helpers/reports}/scope_reports.rs | 54 +- .../src/scopes/constants.rs | 15 + .../src/scopes/main/checks/mod.rs | 6 + .../main/checks/register_scope_validation.rs | 12 + .../main/checks/register_scope_visibility.rs | 12 + .../main/checks/relocated_declarations.rs | 17 + .../checks/scope_index_with_compile_usages.rs | 14 + .../main/checks/scope_tolerant_index.rs | 19 + .../main/checks/validate_scope_index.rs | 12 + .../src/scopes/main/declarations/mod.rs | 1 + .../register_declaration_contexts.rs | 12 + .../scopes/main/index/build_scope_lookup.rs | 14 + .../src/scopes/main/index/mod.rs | 8 + .../main/index/open_native_scope_index.rs | 19 + .../scopes/main/index/project_fingerprint.rs | 12 + .../main/index/read_scope_index_cache.rs | 18 + .../scopes/main/index/register_scope_cache.rs | 12 + .../scopes/main/index/register_scope_index.rs | 12 + .../main/index/register_scope_lookup.rs | 12 + .../main/index/write_scope_index_cache.rs | 19 + .../src/scopes/main/mod.rs | 5 + .../src/scopes/main/relationships/mod.rs | 2 + .../relationships/register_compile_usages.rs | 12 + .../register_relationship_names.rs | 12 + .../src/scopes/main/reports/json_to_python.rs | 13 + .../main/reports/metadata_projection_value.rs | 12 + .../src/scopes/main/reports/mod.rs | 12 + .../src/scopes/main/reports/read_result.rs | 14 + .../main/reports/register_scope_reports.rs | 12 + .../main/reports/report_scope_index_errors.rs | 12 + .../src/scopes/main/reports/scope_browse.rs | 18 + .../src/scopes/main/reports/scope_explain.rs | 19 + .../src/scopes/main/reports/scope_list.rs | 19 + .../scopes/main/reports/scope_move_preview.rs | 19 + .../main/reports/scope_rebound_usages.rs | 17 + .../src/scopes/main/reports/scope_report.rs | 19 + .../main/reports/serialize_scope_report.rs | 13 + .../sqlbuild-python-front/src/scopes/mod.rs | 6 + .../src/scopes/models.rs | 487 ++++++ .../sql_test_glue/_helpers}/authored_ctes.rs | 2 +- .../src/sql_test_glue/_helpers/mod.rs | 1 + .../src/sql_test_glue/main/mod.rs | 3 + .../src/sql_test_glue/main/prefixed_ctes.rs | 13 + .../main/register_authored_ctes.rs | 12 + .../main/validate_authored_sql_test_ctes.rs | 19 + .../src/sql_test_glue/mod.rs | 4 + .../src/sqltext/_helpers}/authored_values.rs | 0 .../src/sqltext/_helpers/code_scanner.rs | 9 + .../src/sqltext/_helpers}/ddl_effects.rs | 0 .../_helpers}/declaration_references.rs | 12 +- .../src/sqltext/_helpers}/lexical_syntax.rs | 14 +- .../src/sqltext/_helpers/mod.rs | 9 + .../src/sqltext/_helpers}/model_headers.rs | 25 +- .../src/sqltext/_helpers}/placeholders.rs | 0 .../src/sqltext/_helpers}/sql_references.rs | 2 +- .../src/sqltext/_helpers}/static_sql.rs | 23 +- .../sqltext/main/authored_value_to_python.rs | 14 + .../src/sqltext/main/declaration_scan.rs | 11 + .../src/sqltext/main/extract_references.rs | 16 + .../src/sqltext/main/header_failure.rs | 13 + .../src/sqltext/main/interpolated.rs | 16 + .../src/sqltext/main/interpolation_batch.rs | 15 + .../src/sqltext/main/location_rows.rs | 12 + .../src/sqltext/main/map_to_python.rs | 14 + .../src/sqltext/main/mod.rs | 20 + .../src/sqltext/main/model_output_columns.rs | 14 + .../main/optional_authored_value_to_python.rs | 17 + .../sqltext/main/optional_record_syntax.rs | 13 + .../src/sqltext/main/parse_model_headers.rs | 15 + .../src/sqltext/main/record_syntax.rs | 13 + .../src/sqltext/main/register_code_scanner.rs | 12 + .../src/sqltext/main/register_ddl_effects.rs | 12 + .../sqltext/main/register_model_headers.rs | 12 + .../src/sqltext/main/register_placeholders.rs | 12 + .../sqltext/main/register_sql_references.rs | 12 + .../src/sqltext/main/sorted_texts.rs | 12 + .../src/sqltext/main/syntax_record.rs | 14 + .../sqlbuild-python-front/src/sqltext/mod.rs | 6 + .../src/sqltext/models.rs | 140 ++ .../src/sqltext/types.rs | 10 + .../state_rows/_helpers/event_rows}/events.rs | 3 +- .../_helpers/event_rows}/freshness.rs | 10 +- .../src/state_rows/_helpers/event_rows/mod.rs | 3 + .../_helpers/event_rows}/timestamps.rs | 3 +- .../_helpers/migrations}/migration_io.rs | 36 +- .../_helpers/migrations}/migration_objects.rs | 9 +- .../migrations}/migration_projections.rs | 6 +- .../_helpers/migrations}/migration_rows.rs | 17 +- .../src/state_rows/_helpers/migrations/mod.rs | 4 + .../src/state_rows/_helpers/mod.rs | 3 + .../src/state_rows/_helpers/tables/mod.rs | 1 + .../_helpers/tables}/state_tables.rs | 2 +- .../src}/state_rows/constants.rs | 19 +- .../state_rows/main/event_rows/age_policy.rs | 16 + .../main/event_rows/date_time_typed.rs | 12 + .../state_rows/main/event_rows/equivalent.rs | 16 + .../src/state_rows/main/event_rows/in_utc.rs | 12 + .../src/state_rows/main/event_rows/mod.rs | 10 + .../state_rows/main/event_rows/normalize.rs | 15 + .../main/event_rows/offset_aware.rs | 12 + .../main/event_rows/presumed_utc.rs | 13 + .../main/event_rows/register_events.rs | 12 + .../main/event_rows/register_freshness.rs | 12 + .../main/event_rows/version_hash.rs | 15 + .../src/state_rows/main/migrations/layout.rs | 11 + .../src/state_rows/main/migrations/mod.rs | 12 + .../main/migrations/newest_column.rs | 16 + .../state_rows/main/migrations/newest_move.rs | 15 + .../main/migrations/old_name_retention.rs | 15 + .../main/migrations/old_name_views.rs | 16 + .../state_rows/main/migrations/read_events.rs | 20 + .../main/migrations/read_histories.rs | 13 + .../main/migrations/register_migration_io.rs | 12 + .../register_migration_projections.rs | 12 + .../migrations/register_migration_rows.rs | 12 + .../main/migrations/stored_columns_binding.rs | 18 + .../migrations/stored_old_name_columns.rs | 16 + .../src/state_rows/main/mod.rs | 3 + .../src/state_rows/main/tables/mod.rs | 2 + .../main/tables/register_state_tables.rs | 12 + .../main/tables/state_table_create_sql.rs | 24 + .../src/state_rows/mod.rs | 6 + .../src}/state_rows/models.rs | 11 +- crates/sqlbuild-python/Cargo.toml | 2 +- .../analysis_session/model_analyses.rs | 2 +- .../analysis_session/stage_functions.rs | 2 +- .../src/bindings/_helpers/attachments/mod.rs | 5 - .../_helpers/attachments/registration.rs | 12 +- .../_helpers/audit_command/command.rs | 2 +- .../_helpers/auditing/schema_audits.rs | 7 +- .../bindings/_helpers/authored_values/mod.rs | 4 - .../_helpers/authored_values/registration.rs | 10 +- .../_helpers/build_command/command.rs | 7 +- .../_helpers/build_command/defer_clone.rs | 2 +- .../_helpers/build_command/execution.rs | 5 +- .../_helpers/build_command/invocation.rs | 12 +- .../change_selection/base_snapshot.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 6 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../src/bindings/_helpers/cli/registration.rs | 6 +- .../bindings/_helpers/clone/clone_command.rs | 6 +- .../_helpers/clone/clone_execution.rs | 2 +- .../bindings/_helpers/clone/clone_pipeline.rs | 2 +- .../_helpers/command_runtime/connection.rs | 8 +- .../_helpers/command_runtime/invocation.rs | 4 +- .../bindings/_helpers/command_runtime/mod.rs | 1 - .../_helpers/command_runtime/warehouses.rs | 12 +- .../compile_attachment/declaration_scope.rs | 12 +- .../_helpers/compile_attachment/mod.rs | 1 - .../compile_attachment/model_config.rs | 4 +- .../compile_attachment/model_inputs.rs | 24 +- .../compile_attachment/registration.rs | 45 +- .../sql_expansion_context.rs | 13 +- .../_helpers/compile_attachment/target.rs | 9 +- .../_helpers/compile_command/report.rs | 4 +- .../_helpers/compile_command/reuse_attempt.rs | 4 +- .../compile_command/runtime_identity.rs | 2 +- .../_helpers/compile_driver/analysis.rs | 2 +- .../compile_driver/assembly_analysis.rs | 4 +- .../compile_driver/assembly_checks.rs | 13 +- .../_helpers/compile_driver/assembly_facts.rs | 2 +- .../compile_driver/assembly_models.rs | 4 +- .../compile_driver/assembly_project.rs | 11 +- .../_helpers/compile_driver/deferral.rs | 2 +- .../_helpers/compile_driver/discovery.rs | 69 +- .../_helpers/compile_driver/inputs.rs | 15 +- .../bindings/_helpers/compile_driver/mod.rs | 1 - .../_helpers/compile_driver/phases.rs | 2 +- .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compile_driver/project.rs | 7 +- .../_helpers/compile_driver/targets.rs | 10 +- .../_helpers/compiled_project/job_catalog.rs | 2 +- .../_helpers/compiled_project/project.rs | 2 +- .../compiled_project/snapshot_models.rs | 2 +- .../_helpers/compiled_views/audit_views.rs | 2 +- .../compiled_views/compiled_model_analysis.rs | 4 +- .../compiled_views/compiled_model_view.rs | 6 +- .../compiled_views/model_input_view.rs | 13 +- .../_helpers/compiled_views/relation_keys.rs | 2 +- .../bindings/_helpers/consumption/policy.rs | 2 +- .../bindings/_helpers/consumption/recorder.rs | 6 +- .../_helpers/consumption/relation_reads.rs | 2 +- .../_helpers/contract_command/command.rs | 2 +- .../_helpers/contract_command/write.rs | 6 +- .../contracts/contract_diagnostics.rs | 3 +- .../src/bindings/_helpers/contracts/mod.rs | 2 - .../_helpers/contracts/registration.rs | 18 +- .../bindings/_helpers/cost_command/refresh.rs | 4 +- .../_helpers/cost_report/functions.rs | 2 +- .../_helpers/cost_telemetry/ledger.rs | 2 +- .../_helpers/cost_telemetry/snowflake.rs | 2 +- .../_helpers/cost_telemetry/values.rs | 4 +- .../src/bindings/_helpers/cursor/mod.rs | 6 - .../bindings/_helpers/cursor/registration.rs | 10 +- .../_helpers/dbt_interop/pipeline_phases.rs | 4 +- .../_helpers/dbt_interop/pipeline_prologue.rs | 2 +- .../_helpers/dbt_interop/plan_output.rs | 2 +- .../_helpers/debug_command/run_debug.rs | 6 +- .../bindings/_helpers/diff_report/command.rs | 6 +- .../bindings/_helpers/diff_report/query.rs | 4 +- .../bindings/_helpers/discovery/constants.rs | 21 - .../src/bindings/_helpers/discovery/mod.rs | 37 - .../_helpers/discovery/model_file_record.rs | 630 ------- .../_helpers/discovery/registration.rs | 73 +- .../_helpers/execution_history/conversion.rs | 3 +- .../_helpers/exported_constants/constants.rs | 7 +- .../exported_constants/registration.rs | 2 +- .../bindings/_helpers/format_command/run.rs | 2 +- .../_helpers/format_command/selection.rs | 19 +- .../_helpers/freshness/freshness_command.rs | 2 +- .../_helpers/freshness/freshness_observe.rs | 3 +- .../_helpers/freshness/observation.rs | 6 +- .../bindings/_helpers/freshness/planning.rs | 3 +- .../bindings/_helpers/freshness/state_io.rs | 8 +- .../bindings/_helpers/freshness/support.rs | 6 +- .../_helpers/graph/python_node_graph.rs | 4 +- .../_helpers/input_loops/audit_loop.rs | 11 +- .../_helpers/input_loops/function_loop.rs | 4 +- .../_helpers/input_loops/model_loop.rs | 41 +- .../_helpers/input_loops/model_loop_config.rs | 18 +- .../input_loops/model_loop_contracts.rs | 6 +- .../_helpers/input_loops/model_loop_header.rs | 25 +- .../_helpers/input_loops/model_loop_hooks.rs | 20 +- .../_helpers/input_loops/model_loop_host.rs | 18 +- .../_helpers/input_loops/model_loop_macros.rs | 9 +- .../_helpers/input_loops/model_loop_named.rs | 2 +- .../_helpers/input_loops/model_loop_schema.rs | 2 +- .../input_loops/model_loop_visibility.rs | 4 +- .../input_loops/resource_diagnostics.rs | 2 +- .../_helpers/input_loops/source_loop.rs | 32 +- .../_helpers/input_loops/sql_test_loop.rs | 2 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/janitor/janitor_old_names.rs | 5 +- .../_helpers/janitor/janitor_phases.rs | 2 +- .../_helpers/lineage/lineage_command.rs | 2 +- .../bindings/_helpers/load/ingestr_command.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 18 +- .../bindings/_helpers/load/load_selection.rs | 11 +- .../_helpers/location_command/snapshot.rs | 7 +- .../_helpers/location_command/worker.rs | 4 +- .../src/bindings/_helpers/macro_bridge/mod.rs | 3 - .../_helpers/macro_bridge/registration.rs | 10 +- .../_helpers/microbatches/conversion.rs | 4 +- .../_helpers/model_config/authored_nodes.rs | 127 -- .../_helpers/model_config/config_templates.rs | 422 ----- .../bindings/_helpers/model_config/errors.rs | 13 - .../_helpers/model_config/header_audits.rs | 445 ----- .../_helpers/model_config/header_families.rs | 65 - .../src/bindings/_helpers/model_config/mod.rs | 21 - .../model_config/model_config_builder.rs | 458 ----- .../_helpers/model_config/model_validator.rs | 257 --- .../_helpers/model_config/registration.rs | 25 +- .../src/bindings/_helpers/native_host/mod.rs | 3 - .../native_host/orchestration_sensors.rs | 9 +- .../native_host/orchestration_triggers.rs | 7 +- .../native_host/resource_costs_state.rs | 12 +- .../native_host/warehouse_metadata.rs | 3 +- .../_helpers/observations/publication.rs | 8 +- .../_helpers/output_capture/values.rs | 2 +- .../bindings/_helpers/plan_command/plan.rs | 2 +- .../bindings/_helpers/plan_command/rules.rs | 2 +- .../_helpers/plan_output/cursor_details.rs | 2 +- .../bindings/_helpers/plan_output/display.rs | 7 +- .../_helpers/plan_output/plan_json.rs | 2 +- .../_helpers/plan_output/plan_text.rs | 2 +- .../bindings/_helpers/plan_output/safety.rs | 2 +- .../_helpers/planner_changes/detect.rs | 5 +- .../_helpers/planner_entries/clone_plans.rs | 3 +- .../_helpers/planner_entries/display_plan.rs | 3 +- .../_helpers/planner_entries/model_entries.rs | 2 +- .../_helpers/planner_identity/metadata.rs | 5 +- .../planner_migrations/column_entries.rs | 2 +- .../planner_migrations/column_objects.rs | 3 +- .../planner_migrations/compatibility.rs | 3 +- .../planner_migrations/discovery_events.rs | 2 +- .../planner_migrations/model_entries.rs | 2 +- .../planner_migrations/model_handover.rs | 2 +- .../planner_migrations/model_planning.rs | 6 +- .../planner_migrations/old_name_objects.rs | 2 +- .../planner_migrations/state_inspection.rs | 13 +- .../planner_phases/deferred_origins.rs | 3 +- .../planner_phases/execution_changes.rs | 2 +- .../_helpers/planner_phases/full_refresh.rs | 3 +- .../_helpers/planner_phases/objects.rs | 3 +- .../_helpers/planner_phases/retention.rs | 5 +- .../planner_resolution/cursor_bounds.rs | 5 +- .../planner_resolution/cursor_model.rs | 6 +- .../planner_resolution/cursor_overrides.rs | 6 +- .../planner_resolution/cursor_policies.rs | 3 +- .../planner_resolution/cursor_ranges.rs | 14 +- .../planner_resolution/cursor_safety.rs | 2 +- .../planner_resolution/cursor_warnings.rs | 2 +- .../planner_resolution/cursor_windows.rs | 4 +- .../planner_resolution/future_safety.rs | 2 +- .../planner_resolution/maximum_safety.rs | 4 +- .../planner_resolution/orchestration.rs | 14 +- .../_helpers/planner_resolution/refs.rs | 2 +- .../_helpers/planner_resolution/values.rs | 2 +- .../_helpers/planner_scenario/execution.rs | 3 +- .../_helpers/planner_scenario/fixtures.rs | 4 +- .../_helpers/planner_scenario/helper_scope.rs | 2 +- .../bindings/_helpers/planner_scenario/sql.rs | 2 +- .../_helpers/planner_scope/loaders.rs | 2 +- .../planner_selection/model_selectors.rs | 3 +- .../_helpers/planner_selection/unit_tests.rs | 2 +- .../planner_strategy/audit_entries.rs | 2 +- .../planner_strategy/cursor_inputs.rs | 9 +- .../planner_strategy/fixture_metadata.rs | 3 +- .../_helpers/planner_strategy/lineage.rs | 3 +- .../_helpers/planner_strategy/microbatches.rs | 2 +- .../planner_strategy/model_entries.rs | 14 +- .../_helpers/planner_strategy/values.rs | 5 +- .../_helpers/planner_warehouse/cursors.rs | 17 +- .../planner_warehouse/semantic_sources.rs | 3 +- .../_helpers/planner_warehouse/snapshot.rs | 3 +- .../planner_warehouse/source_deferral.rs | 7 +- .../planner_warehouse/source_tables.rs | 2 +- .../_helpers/planner_warehouse/watermarks.rs | 3 +- .../project_assembly/native_resources.rs | 4 +- .../project_assembly/syntax_checks.rs | 2 +- .../_helpers/query_command/project.rs | 6 +- .../src/bindings/_helpers/recall/events.rs | 2 +- .../bindings/_helpers/recall/registration.rs | 2 +- .../_helpers/record_behaviours/constants.rs | 11 +- .../_helpers/record_behaviours/cursors.rs | 2 +- .../_helpers/record_behaviours/durations.rs | 7 +- .../_helpers/record_behaviours/lexical.rs | 3 +- .../_helpers/record_behaviours/macros.rs | 5 +- .../_helpers/record_behaviours/planner.rs | 2 +- .../src/bindings/_helpers/refactor/mod.rs | 4 - .../_helpers/refactor/refactor_command.rs | 7 +- .../_helpers/refactor/registration.rs | 13 +- .../src/bindings/_helpers/render/mod.rs | 19 - .../bindings/_helpers/render/registration.rs | 31 +- .../bindings/_helpers/result_output/checks.rs | 2 +- .../result_output/integration_envelope.rs | 2 +- .../_helpers/review_command/execution.rs | 2 +- .../_helpers/review_command/planning.rs | 10 +- .../_helpers/review_command/schema.rs | 2 +- .../_helpers/rules/custom_host_columns.rs | 2 +- .../bindings/_helpers/rules/early_sql_lint.rs | 2 +- .../bindings/_helpers/rules/fix_session.rs | 4 +- .../_helpers/rules/fixture_formatting.rs | 4 +- .../_helpers/rules/fixture_schemas.rs | 6 +- .../bindings/_helpers/rules/lint_expansion.rs | 4 +- .../bindings/_helpers/rules/lint_inputs.rs | 2 +- .../bindings/_helpers/rules/lint_pipeline.rs | 7 +- .../_helpers/rules/lint_preparation.rs | 2 +- .../_helpers/rules/native_header_parser.rs | 2 +- .../src/bindings/_helpers/rules/rows.rs | 2 +- .../src/bindings/_helpers/rules/run_lint.rs | 2 +- .../src/bindings/_helpers/rules/sql_rules.rs | 2 +- .../_helpers/scenario_command/setup.rs | 6 +- .../_helpers/scenario_command/test_command.rs | 2 +- .../_helpers/scopes/declaration_contexts.rs | 314 ---- .../src/bindings/_helpers/scopes/mod.rs | 17 - .../bindings/_helpers/scopes/registration.rs | 32 +- .../bindings/_helpers/scopes/scope_command.rs | 19 +- .../bindings/_helpers/scopes/scope_loading.rs | 16 +- .../bindings/_helpers/seed_command/command.rs | 2 +- .../_helpers/semantic_checks/metadata.rs | 6 +- .../semantic_shapes/source_bindings.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- .../_helpers/sql_test_command/timeouts.rs | 2 +- .../_helpers/sql_test_glue/assembly.rs | 2 +- .../_helpers/sql_test_glue/cursor_window.rs | 13 +- .../bindings/_helpers/sql_test_glue/mod.rs | 1 - .../_helpers/sql_test_glue/plan_entries.rs | 2 +- .../_helpers/sql_test_glue/plan_fixtures.rs | 2 +- .../_helpers/sql_test_glue/planning.rs | 2 +- .../_helpers/sql_test_glue/registration.rs | 5 +- .../bindings/_helpers/sqltext/code_scanner.rs | 67 - .../src/bindings/_helpers/sqltext/mod.rs | 9 - .../bindings/_helpers/sqltext/registration.rs | 18 +- .../src/bindings/_helpers/state_rows/mod.rs | 10 - .../_helpers/state_rows/registration.rs | 21 +- crates/sqlbuild-python/src/bindings/models.rs | 61 - fensu.toml | 399 +++-- scripts/fensu_policy/constants.py | 1 + scripts/native_module_paths/constants.py | 1 + tests/native_field_fallbacks.txt | 74 +- 871 files changed, 12236 insertions(+), 5835 deletions(-) create mode 100644 crates/sqlbuild-python-front/Cargo.toml rename crates/{sqlbuild-python/src/bindings/_helpers/attachments => sqlbuild-python-front/src/attachments/_helpers}/argument_values.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/attachments => sqlbuild-python-front/src/attachments/_helpers}/ceremonial_select.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/attachments => sqlbuild-python-front/src/attachments/_helpers}/cursor_intrinsics.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/attachments => sqlbuild-python-front/src/attachments/_helpers}/function_headers.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/attachments/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/attachments => sqlbuild-python-front/src/attachments/_helpers}/reference_markers.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/attachments/constants.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs create mode 100644 crates/sqlbuild-python-front/src/attachments/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/authored_values => sqlbuild-python-front/src/authored_values/_helpers}/fields.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/authored_values/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/authored_values => sqlbuild-python-front/src/authored_values/_helpers}/project_vars.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/authored_values => sqlbuild-python-front/src/authored_values/_helpers}/python_nodes.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/authored_values => sqlbuild-python-front/src/authored_values/_helpers}/sql_values.rs (88%) create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/mod.rs create mode 100644 crates/sqlbuild-python-front/src/authored_values/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/command_runtime => sqlbuild-python-front/src/command_runtime/_helpers}/concurrency.rs (90%) create mode 100644 crates/sqlbuild-python-front/src/command_runtime/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/command_runtime/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs create mode 100644 crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs create mode 100644 crates/sqlbuild-python-front/src/command_runtime/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-front/src/compile_attachment/_helpers}/effective_config.rs (76%) create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs create mode 100644 crates/sqlbuild-python-front/src/compile_attachment/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-front/src/compile_driver/_helpers}/config_validation.rs (93%) create mode 100644 crates/sqlbuild-python-front/src/compile_driver/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/mod.rs create mode 100644 crates/sqlbuild-python-front/src/compile_driver/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-front/src/contracts/_helpers}/effective_config.rs (99%) create mode 100644 crates/sqlbuild-python-front/src/contracts/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-front/src/contracts/_helpers}/spec_resolution.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs create mode 100644 crates/sqlbuild-python-front/src/contracts/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cursor => sqlbuild-python-front/src/cursor/_helpers}/durations.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/cursor => sqlbuild-python-front/src/cursor/_helpers}/intervals.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cursor => sqlbuild-python-front/src/cursor/_helpers}/overrides.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/cursor => sqlbuild-python-front/src/cursor/_helpers}/scalars.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/cursor => sqlbuild-python-front/src/cursor/_helpers}/values.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/cursor/constants.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/conversion/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/durations/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/intervals/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/overrides/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/values/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs create mode 100644 crates/sqlbuild-python-front/src/cursor/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs => sqlbuild-python-front/src/cursor/models.rs} (74%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/failures}/faults.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/failures}/guidance.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/failures/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/failures}/raised.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/headers}/header_declarations.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/headers}/header_keys.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/headers}/header_values.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/headers}/hook_entries.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/headers}/key_checks.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/headers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/configuration.rs (85%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/locations.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/project/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/project_config.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/project_facts.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/project_files.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/project}/project_validation.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/imported_modules.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/imported_providers.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/imported_python_nodes.rs (93%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/python_files/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/python_function_files.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/python_hook_files.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/python_module_files.rs (83%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/python_files}/python_paths.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/declaration_records.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/description_inputs.rs (82%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/getters.rs (75%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/records/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/model_records.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/sql_test_records.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/records}/yaml_records.rs (93%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/snapshots}/scope_snapshot.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/snapshots}/snapshot.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/declaration_files.rs (93%) create mode 100644 crates/sqlbuild-python-front/src/discovery/_helpers/sources/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/orchestration_files.rs (85%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/orchestration_values.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/provider_secrets.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/sql_test_blocks.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/discovery => sqlbuild-python-front/src/discovery/_helpers/sources}/yaml_entries.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/discovery/constants.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/each.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/stem.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/mod.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/models.rs create mode 100644 crates/sqlbuild-python-front/src/lib.rs rename crates/{sqlbuild-python/src/bindings/_helpers/macro_bridge => sqlbuild-python-front/src/macro_bridge/_helpers}/first_compile.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/macro_bridge => sqlbuild-python-front/src/macro_bridge/_helpers}/module_digests.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/macro_bridge => sqlbuild-python-front/src/macro_bridge/_helpers}/store_keys.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/main/text.rs create mode 100644 crates/sqlbuild-python-front/src/macro_bridge/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/config_classes.rs (80%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/config_errors.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/config_nodes.rs (87%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/config_values.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/getters.rs (65%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/config/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/template_data.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/config}/typed_config.rs (92%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/headers}/header_columns.rs (87%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/headers}/header_metadata.rs (58%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/authored_nodes.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/model_nodes}/column_migrations.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/model_nodes}/contract_classes.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/model_nodes}/described_models.rs (87%) create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/model_nodes}/project_defaults.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-python-front/src/model_config/_helpers/model_nodes}/unicode_environment.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/model_config/constants.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/errors.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/config/view.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/mod.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/models.rs create mode 100644 crates/sqlbuild-python-front/src/model_config/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-front/src/native_host/_helpers}/audit_results_state.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-front/src/native_host/_helpers}/consumption_state.rs (73%) create mode 100644 crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-front/src/native_host/_helpers}/observations_state.rs (90%) create mode 100644 crates/sqlbuild-python-front/src/native_host/constants.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/append_observations.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/execute.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/literal.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/optional_text.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/mod.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/refactor => sqlbuild-python-front/src/refactor/_helpers}/contract_compare.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/refactor => sqlbuild-python-front/src/refactor/_helpers}/contract_edits.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/refactor/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/refactor => sqlbuild-python-front/src/refactor/_helpers}/python_strings.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/refactor => sqlbuild-python-front/src/refactor/_helpers}/refactor_facts.rs (97%) create mode 100644 crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/model_facts.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs create mode 100644 crates/sqlbuild-python-front/src/refactor/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/calls}/macro_arguments.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/calls}/macro_call_classes.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/calls}/macro_call_memo.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/calls}/macro_call_rows.rs (79%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/calls}/macro_calls.rs (90%) create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/calls/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/declarations}/authored_sql.rs (77%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/declarations}/declaration_expansion.rs (82%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/declarations}/declaration_indexes.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/declarations}/declaration_values.rs (90%) create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/declarations/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/expansion}/macro_evaluation.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/expansion}/macro_expansion_facts.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/expansion}/macro_expansion_result.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/expansion}/macro_generated_references.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/expansion}/macro_modules.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/expansion/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/_helpers/sql_macros/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/sql_macros}/sql_macro_host.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/sql_macros}/sql_macro_inventory.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/sql_macros}/sql_macro_mocks.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/render => sqlbuild-python-front/src/render/_helpers/sql_macros}/sql_macro_tests.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/span_row.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/expand_declarations.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/sql_macros/mod.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs create mode 100644 crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs create mode 100644 crates/sqlbuild-python-front/src/render/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs => sqlbuild-python-front/src/render/models.rs} (59%) create mode 100644 crates/sqlbuild-python-front/src/render/types.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/checks/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/checks}/scope_tolerant.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/checks}/scope_validation.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/checks}/scope_visibility.rs (94%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/declarations}/declaration_resolution.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/declarations/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/declarations}/placement.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/index}/getters.rs (62%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/index/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/index}/scope_cache.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/index}/scope_index.rs (85%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/index}/scope_lookup.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/index}/scope_objects.rs (94%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/relationships}/compile_usages.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/relationships/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/relationships}/relationship_names.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/relationships}/relationship_scan.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/scopes/_helpers/reports/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/reports}/scope_errors.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/reports}/scope_report_objects.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-front/src/scopes/_helpers/reports}/scope_reports.rs (92%) create mode 100644 crates/sqlbuild-python-front/src/scopes/constants.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/declarations/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/relationships/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/mod.rs create mode 100644 crates/sqlbuild-python-front/src/scopes/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-front/src/sql_test_glue/_helpers}/authored_ctes.rs (98%) create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs create mode 100644 crates/sqlbuild-python-front/src/sql_test_glue/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/authored_values.rs (100%) create mode 100644 crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/ddl_effects.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/declaration_references.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/lexical_syntax.rs (89%) create mode 100644 crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/model_headers.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/placeholders.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/sql_references.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/sqltext => sqlbuild-python-front/src/sqltext/_helpers}/static_sql.rs (88%) create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/mod.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/models.rs create mode 100644 crates/sqlbuild-python-front/src/sqltext/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/event_rows}/events.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/event_rows}/freshness.rs (96%) create mode 100644 crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/event_rows}/timestamps.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/migrations}/migration_io.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/migrations}/migration_objects.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/migrations}/migration_projections.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/migrations}/migration_rows.rs (96%) create mode 100644 crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/_helpers/tables/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/state_rows => sqlbuild-python-front/src/state_rows/_helpers/tables}/state_tables.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-front/src}/state_rows/constants.rs (61%) create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/tables/mod.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs create mode 100644 crates/sqlbuild-python-front/src/state_rows/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-front/src}/state_rows/models.rs (77%) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs diff --git a/Cargo.lock b/Cargo.lock index f502b30315..5bf2f44b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4460,6 +4460,7 @@ dependencies = [ "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python-base", + "sqlbuild-python-front", "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", @@ -4468,7 +4469,6 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "sqlclean", - "stacker", "tokio", "uuid", ] @@ -4497,6 +4497,36 @@ dependencies = [ "sqlbuild-state", ] +[[package]] +name = "sqlbuild-python-front" +version = "0.0.0" +dependencies = [ + "chrono", + "chrono-tz", + "pyo3", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-orchestration", + "sqlbuild-pysource", + "sqlbuild-python-base", + "sqlbuild-refactor", + "sqlbuild-render", + "sqlbuild-scopes", + "sqlbuild-sqltext", + "sqlbuild-state", + "stacker", + "uuid", +] + [[package]] name = "sqlbuild-recall" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 6b69927a79..f6fc2c295a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/sqlbuild-project", "crates/sqlbuild-cli", "crates/sqlbuild-python-base", + "crates/sqlbuild-python-front", "crates/sqlbuild-python", ] resolver = "2" @@ -57,9 +58,10 @@ order = [ "sqlbuild-project", "sqlbuild-cli", "sqlbuild-python-base", + "sqlbuild-python-front", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -180,6 +182,7 @@ sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" } sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } +sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index c0b283aff3..3b6a0de5bd 100644 --- a/crates/README.md +++ b/crates/README.md @@ -29,14 +29,15 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-base`. | +| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-front/Cargo.toml b/crates/sqlbuild-python-front/Cargo.toml new file mode 100644 index 0000000000..a89a503d5e --- /dev/null +++ b/crates/sqlbuild-python-front/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sqlbuild-python-front" +description = "The front layer of the sqlbuild._native bindings: discovery, model configuration, scopes, rendering and SQL text." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +chrono.workspace = true +chrono-tz.workspace = true +pyo3.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-cli.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true +sqlbuild-pysource.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-refactor.workspace = true +sqlbuild-render.workspace = true +sqlbuild-scopes.workspace = true +sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true +stacker.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs rename to crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs index cad5b27291..4b5621ff57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs @@ -1,6 +1,7 @@ //! Python audit argument values read as plain argument values. -use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_object}; +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::types::PyConfigNode; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs rename to crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs index 1ffcd001fa..9e154442e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs @@ -6,7 +6,7 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The code-point offset to insert `SELECT 1` at, or `None`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs rename to crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs index 6d52d7f64c..a6222f0c50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs @@ -1,5 +1,7 @@ //! Cursor intrinsic validation, rejection and rendering. +use crate::attachments::constants::MICROBATCH_END_SENTINEL; +use crate::attachments::constants::MICROBATCH_START_SENTINEL; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{Python, pyfunction, wrap_pyfunction}; @@ -23,10 +25,7 @@ const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const MODEL_SQL_CONTEXT: &str = "Model SQL"; const SQL_CONTEXT: &str = "SQL"; -/// Executor placeholder replaced with a microbatch's start bound. -pub(crate) const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__"; -/// Executor placeholder replaced with a microbatch's end bound. -pub(crate) const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__"; + const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; /// The reserved executor bound markers no authored SQL may contain. @@ -53,7 +52,7 @@ fn text_value(value: &Bound<'_, PyAny>) -> Option { } /// `sql` with each intrinsic replaced by its bound and whether any was, raising for `context`. -fn replaced( +pub(crate) fn replaced( py: Python<'_>, sql: &str, context: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs rename to crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/mod.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/mod.rs new file mode 100644 index 0000000000..791c2acc02 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod argument_values; +pub(crate) mod ceremonial_select; +pub(crate) mod cursor_intrinsics; +pub(crate) mod function_headers; +pub(crate) mod reference_markers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs rename to crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs diff --git a/crates/sqlbuild-python-front/src/attachments/constants.rs b/crates/sqlbuild-python-front/src/attachments/constants.rs new file mode 100644 index 0000000000..2f9c28b99c --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/constants.rs @@ -0,0 +1,7 @@ +//! Public constants of attachments. + +/// Executor placeholder replaced with a microbatch's start bound. +pub const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__"; + +/// Executor placeholder replaced with a microbatch's end bound. +pub const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__"; diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs new file mode 100644 index 0000000000..f7caf3f420 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs @@ -0,0 +1,13 @@ +//! `analysis_cursor_sql` with each replaced call as a splice of `sql`. + +use crate::attachments::_helpers::cursor_intrinsics; +use sqlbuild_analysis::sql_macros::models::TextSplice; + +/// `analysis_cursor_sql` with each replaced call as a splice of `sql`. +#[inline] +pub fn analysis_cursor_splices( + sql: &str, + cursor_type: Option<&str>, +) -> Result<(String, Vec), String> { + cursor_intrinsics::analysis_cursor_splices(sql, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs new file mode 100644 index 0000000000..6aa9f8b277 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs @@ -0,0 +1,17 @@ +//! `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. + +use crate::attachments::_helpers::cursor_intrinsics; +use pyo3::Python; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. +#[inline] +pub fn analysis_cursor_sql( + py: Python<'_>, + sql: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + cursor_intrinsics::analysis_cursor_sql(py, sql, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs new file mode 100644 index 0000000000..9b2119bd47 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs @@ -0,0 +1,13 @@ +//! Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. + +use crate::attachments::_helpers::argument_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_attachments::audits::models::ArgumentValue; + +/// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. +#[inline] +pub fn argument_pairs(arguments: &Bound<'_, PyDict>) -> PyResult> { + argument_values::argument_pairs(arguments) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs new file mode 100644 index 0000000000..67a209e017 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs @@ -0,0 +1,17 @@ +//! Fail if executable SQL still contains unresolved reference markers or an unclosed construct. + +use crate::attachments::_helpers::reference_markers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyString; + +/// Fail if executable SQL still contains unresolved reference markers or an unclosed construct. +#[inline] +pub fn assert_no_unresolved_sql_markers( + py: Python<'_>, + sql: &Bound<'_, PyString>, + context: &str, +) -> PyResult<()> { + reference_markers::assert_no_unresolved_sql_markers(py, sql, context) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs new file mode 100644 index 0000000000..72483c1606 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs @@ -0,0 +1,12 @@ +//! A `CompileInputError` carrying `message`. + +use crate::attachments::_helpers::cursor_intrinsics; +use pyo3::Python; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; + +/// A `CompileInputError` carrying `message`. +#[inline] +pub fn compile_input_error(py: Python<'_>, message: String) -> PyResult { + cursor_intrinsics::compile_input_error(py, message) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs new file mode 100644 index 0000000000..e7f68a7202 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs @@ -0,0 +1,13 @@ +//! Native-built argument containers need no intermediate Python list or tuple. + +use crate::attachments::_helpers::argument_values; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_attachments::audits::models::ArgumentValue; + +/// Native-built argument containers need no intermediate Python list or tuple. +#[inline] +pub fn config_argument_value(py: Python<'_>, value: &PyConfigNode) -> PyResult { + argument_values::config_argument_value(py, value) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/mod.rs b/crates/sqlbuild-python-front/src/attachments/main/mod.rs new file mode 100644 index 0000000000..9dbfdd42c4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/mod.rs @@ -0,0 +1,13 @@ +pub mod analysis_cursor_splices; +pub mod analysis_cursor_sql; +pub mod argument_pairs; +pub mod assert_no_unresolved_sql_markers; +pub mod compile_input_error; +pub mod config_argument_value; +pub mod parsed_header; +pub mod register_ceremonial_select; +pub mod register_cursor_intrinsics; +pub mod register_reference_markers; +pub mod reserved_cursor_markers; +pub mod resolve_cursor_intrinsics; +pub mod validated_model_cursor_sql; diff --git a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs new file mode 100644 index 0000000000..ca1c8b5033 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs @@ -0,0 +1,17 @@ +//! One function file's parsed header; non-string keys and `None` values are never read. + +use crate::attachments::_helpers::function_headers; +use pyo3::prelude::Bound; +use pyo3::types::PyDict; +use sqlbuild_attachments::functions::models::FunctionHeader; +use sqlbuild_attachments::functions::models::FunctionLanguage; + +/// One function file's parsed header; non-string keys and `None` values are never read. +#[inline] +pub fn parsed_header( + header_values: &Bound<'_, PyDict>, + language: FunctionLanguage, + relative_path: &str, +) -> FunctionHeader { + function_headers::parsed_header(header_values, language, relative_path) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs new file mode 100644 index 0000000000..a84d8eb71e --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs @@ -0,0 +1,12 @@ +//! Register the ceremonial select functions and classes. + +use crate::attachments::_helpers::ceremonial_select; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the ceremonial select functions and classes. +#[inline] +pub fn register_ceremonial_select(module: &Bound<'_, PyModule>) -> PyResult<()> { + ceremonial_select::register(module) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs new file mode 100644 index 0000000000..4efba4d901 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs @@ -0,0 +1,12 @@ +//! Register the cursor intrinsics functions and classes. + +use crate::attachments::_helpers::cursor_intrinsics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor intrinsics functions and classes. +#[inline] +pub fn register_cursor_intrinsics(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_intrinsics::register(module) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs new file mode 100644 index 0000000000..9f79437e34 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs @@ -0,0 +1,12 @@ +//! Register the reference markers functions and classes. + +use crate::attachments::_helpers::reference_markers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the reference markers functions and classes. +#[inline] +pub fn register_reference_markers(module: &Bound<'_, PyModule>) -> PyResult<()> { + reference_markers::register(module) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs new file mode 100644 index 0000000000..1b3973bc25 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs @@ -0,0 +1,9 @@ +//! The reserved executor bound markers no authored SQL may contain. + +use crate::attachments::_helpers::cursor_intrinsics; + +/// The reserved executor bound markers no authored SQL may contain. +#[inline] +pub fn reserved_cursor_markers() -> Vec { + cursor_intrinsics::reserved_cursor_markers() +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs new file mode 100644 index 0000000000..752c0d4d25 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs @@ -0,0 +1,16 @@ +//! Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. + +use crate::attachments::_helpers::cursor_intrinsics; +use pyo3::Python; +use pyo3::prelude::PyResult; + +/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. +#[inline] +pub fn resolve_cursor_intrinsics( + py: Python<'_>, + sql: &str, + start_sql: Option<&str>, + end_sql: Option<&str>, +) -> PyResult<(String, bool)> { + cursor_intrinsics::resolve_cursor_intrinsics(py, sql, start_sql, end_sql) +} diff --git a/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs new file mode 100644 index 0000000000..e93c8a78c7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs @@ -0,0 +1,17 @@ +//! One model query with canonical intrinsic calls, raising when the model may not use them. + +use crate::attachments::_helpers::cursor_intrinsics; +use pyo3::Python; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One model query with canonical intrinsic calls, raising when the model may not use them. +#[inline] +pub fn validated_model_cursor_sql( + py: Python<'_>, + sql: &str, + (model_name, config_values): (&str, &Bound<'_, PyAny>), +) -> PyResult { + cursor_intrinsics::validated_model_cursor_sql(py, sql, (model_name, config_values)) +} diff --git a/crates/sqlbuild-python-front/src/attachments/mod.rs b/crates/sqlbuild-python-front/src/attachments/mod.rs new file mode 100644 index 0000000000..ec9bd3513c --- /dev/null +++ b/crates/sqlbuild-python-front/src/attachments/mod.rs @@ -0,0 +1,5 @@ +//! Authored SQL attachments: argument values, cursor intrinsics, function headers and reference markers. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs rename to crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/mod.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/mod.rs new file mode 100644 index 0000000000..8f6fc721c3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod fields; +pub(crate) mod project_vars; +pub(crate) mod python_nodes; +pub(crate) mod sql_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs rename to crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs index 3d2fafc277..605701b73f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs @@ -87,3 +87,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_project_var_text, module)?)?; Ok(()) } + +/// `render_project_var_text` as a Python function. +pub(crate) fn render_project_var_text_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(render_project_var_text, py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs rename to crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs index aa668a4b4c..d6f912de6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs @@ -14,7 +14,7 @@ use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; use sqlbuild_model_config::header_metadata::main::json_meta_problem::json_meta_problem; use sqlbuild_python_base::native_host::main::host_error::host_error; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::model_config::models::PyNode; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs rename to crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs index 598c9906ca..b0be7fdc1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs @@ -1,5 +1,6 @@ //! Typed SQL value normalization between Python authored values and native `SqlValue`s. +use crate::authored_values::models::SqlValueClasses; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -13,35 +14,11 @@ use sqlbuild_model_config::sql_values::errors::SqlValueFailure; use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value as normalized; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; -/// The Python classes authored values are told apart by and normalized values are built from. -pub(crate) struct SqlValueClasses<'py> { - decimal: Bound<'py, PyAny>, - authored_set: Bound<'py, PyAny>, - value_call: Bound<'py, PyAny>, - sql_value: Bound<'py, PyAny>, - logical_type: Bound<'py, PyAny>, - kind: Bound<'py, PyAny>, -} - -impl<'py> SqlValueClasses<'py> { - pub(crate) fn load(py: Python<'py>) -> PyResult { - Ok(Self { - decimal: py.import("decimal")?.getattr("Decimal")?, - authored_set: record_class(py, RecordId::AuthoredSqlSet)?, - value_call: record_class(py, RecordId::AuthoredSqlValueCall)?, - sql_value: record_class(py, RecordId::SqlValue)?, - logical_type: record_class(py, RecordId::SqlLogicalType)?, - kind: record_class(py, RecordId::SqlValueKind)?, - }) - } -} - fn is_exactly(value: &Bound<'_, PyAny>, class: &Bound<'_, PyAny>) -> bool { value.get_type().is(class) } @@ -121,7 +98,7 @@ pub(crate) fn raw_value( )) } -fn logical_type<'py>( +pub(crate) fn logical_type<'py>( logical: &SqlLogicalType, classes: &SqlValueClasses<'py>, ) -> PyResult> { diff --git a/crates/sqlbuild-python-front/src/authored_values/main/mod.rs b/crates/sqlbuild-python-front/src/authored_values/main/mod.rs new file mode 100644 index 0000000000..aa6faab84c --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/mod.rs @@ -0,0 +1,10 @@ +pub mod project_var_text; +pub mod raw_value; +pub mod register_fields; +pub mod register_project_vars; +pub mod register_python_nodes; +pub mod register_sql_values; +pub mod render_project_var_text; +pub mod render_project_var_text_function; +pub mod sql_value; +pub mod sql_value_error; diff --git a/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs new file mode 100644 index 0000000000..81300b47ea --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs @@ -0,0 +1,12 @@ +//! One project var value as interpolated text; structured values are rejected. + +use crate::authored_values::_helpers::project_vars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One project var value as interpolated text; structured values are rejected. +#[inline] +pub fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { + project_vars::project_var_text(value, label) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs new file mode 100644 index 0000000000..1415e788e2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs @@ -0,0 +1,14 @@ +//! An authored Python value read by exact type, as `normalize_sql_value` dispatches. + +use crate::authored_values::_helpers::sql_values; +use crate::authored_values::models::SqlValueClasses; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_model_config::sql_values::models::RawSqlValue; + +/// An authored Python value read by exact type, as `normalize_sql_value` dispatches. +#[inline] +pub fn raw_value(value: &Bound<'_, PyAny>, classes: &SqlValueClasses<'_>) -> PyResult { + sql_values::raw_value(value, classes) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs new file mode 100644 index 0000000000..a47e495da2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs @@ -0,0 +1,12 @@ +//! Register the fields functions and classes. + +use crate::authored_values::_helpers::fields; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fields functions and classes. +#[inline] +pub fn register_fields(module: &Bound<'_, PyModule>) -> PyResult<()> { + fields::register(module) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs new file mode 100644 index 0000000000..52821b1733 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs @@ -0,0 +1,12 @@ +//! Register the project vars functions and classes. + +use crate::authored_values::_helpers::project_vars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the project vars functions and classes. +#[inline] +pub fn register_project_vars(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_vars::register(module) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs new file mode 100644 index 0000000000..7db72c6561 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs @@ -0,0 +1,12 @@ +//! Register the python nodes functions and classes. + +use crate::authored_values::_helpers::python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python nodes functions and classes. +#[inline] +pub fn register_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_nodes::register(module) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs new file mode 100644 index 0000000000..07a16cc1d4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs @@ -0,0 +1,12 @@ +//! Register the sql values functions and classes. + +use crate::authored_values::_helpers::sql_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sql values functions and classes. +#[inline] +pub fn register_sql_values(module: &Bound<'_, PyModule>) -> PyResult<()> { + sql_values::register(module) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs new file mode 100644 index 0000000000..d3a2aac8df --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs @@ -0,0 +1,12 @@ +//! One project var value as interpolated text; structured values are rejected. + +use crate::authored_values::_helpers::project_vars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One project var value as interpolated text; structured values are rejected. +#[inline] +pub fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { + project_vars::render_project_var_text(value, label) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs new file mode 100644 index 0000000000..9cfdde9bd9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs @@ -0,0 +1,11 @@ +//! `render_project_var_text` as a Python function. + +use crate::authored_values::_helpers::project_vars; + +/// `render_project_var_text` as a Python function. +#[inline] +pub fn render_project_var_text_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + project_vars::render_project_var_text_function(py) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs new file mode 100644 index 0000000000..24b7318920 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs @@ -0,0 +1,17 @@ +//! A Python `SqlValue` for a native one. + +use crate::authored_values::_helpers::sql_values; +use crate::authored_values::models::SqlValueClasses; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::models::SqlValue; + +/// A Python `SqlValue` for a native one. +#[inline] +pub fn sql_value<'py>( + value: &SqlValue, + classes: &SqlValueClasses<'py>, +) -> PyResult> { + sql_values::sql_value(value, classes) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs new file mode 100644 index 0000000000..a3fbc75484 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs @@ -0,0 +1,13 @@ +//! The exception a normalization failure raises. + +use crate::authored_values::_helpers::sql_values; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_model_config::sql_values::errors::SqlValueFailure; + +/// The exception a normalization failure raises. +#[inline] +pub fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { + sql_values::sql_value_error(py, failure) +} diff --git a/crates/sqlbuild-python-front/src/authored_values/mod.rs b/crates/sqlbuild-python-front/src/authored_values/mod.rs new file mode 100644 index 0000000000..b0d56982ab --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/mod.rs @@ -0,0 +1,5 @@ +//! Values authored in project files: fields, project variables, Python nodes and SQL values. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/authored_values/models.rs b/crates/sqlbuild-python-front/src/authored_values/models.rs new file mode 100644 index 0000000000..1597333a81 --- /dev/null +++ b/crates/sqlbuild-python-front/src/authored_values/models.rs @@ -0,0 +1,33 @@ +//! Public records of authored values. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; + +/// The Python classes authored values are told apart by and normalized values are built from. +#[derive(Debug)] +pub struct SqlValueClasses<'py> { + pub(crate) decimal: Bound<'py, PyAny>, + pub(crate) authored_set: Bound<'py, PyAny>, + pub(crate) value_call: Bound<'py, PyAny>, + pub(crate) sql_value: Bound<'py, PyAny>, + pub(crate) logical_type: Bound<'py, PyAny>, + pub(crate) kind: Bound<'py, PyAny>, +} + +impl<'py> SqlValueClasses<'py> { + pub fn load(py: Python<'py>) -> PyResult { + Ok(Self { + decimal: py.import("decimal")?.getattr("Decimal")?, + authored_set: record_class(py, RecordId::AuthoredSqlSet)?, + value_call: record_class(py, RecordId::AuthoredSqlValueCall)?, + sql_value: record_class(py, RecordId::SqlValue)?, + logical_type: record_class(py, RecordId::SqlLogicalType)?, + kind: record_class(py, RecordId::SqlValueKind)?, + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs rename to crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs index 047a888def..41f7b25158 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs @@ -4,10 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings; -use crate::bindings::_helpers::contracts::spec_resolution::{ - resolve_target_config, resolve_target_name, -}; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_base::records::main::fields::field::field as native_field; const CONCURRENCY: &str = "concurrency"; diff --git a/crates/sqlbuild-python-front/src/command_runtime/_helpers/mod.rs b/crates/sqlbuild-python-front/src/command_runtime/_helpers/mod.rs new file mode 100644 index 0000000000..afd92b7347 --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod concurrency; diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs b/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs new file mode 100644 index 0000000000..7482b588aa --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs @@ -0,0 +1,2 @@ +pub mod resolve_concurrency; +pub mod resolve_configured_concurrency; diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs new file mode 100644 index 0000000000..0c69f9bdda --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs @@ -0,0 +1,17 @@ +//! The effective concurrency of a command on `selected_target`, and the layer that set it. + +use crate::command_runtime::_helpers::concurrency; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_config::effective::models::EffectiveConcurrency; + +/// The effective concurrency of a command on `selected_target`, and the layer that set it. +#[inline] +pub fn resolve_concurrency( + requested: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + concurrency::resolve_concurrency(requested, discovered, selected_target) +} diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs new file mode 100644 index 0000000000..5faa1fac25 --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs @@ -0,0 +1,21 @@ +//! `resolve_concurrency` from the project and local configs themselves. + +use crate::command_runtime::_helpers::concurrency; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_config::effective::models::EffectiveConcurrency; + +/// `resolve_concurrency` from the project and local configs themselves. +#[inline] +pub fn resolve_configured_concurrency( + requested: &Bound<'_, PyAny>, + (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + concurrency::resolve_configured_concurrency( + requested, + (project_config, local_config), + selected_target, + ) +} diff --git a/crates/sqlbuild-python-front/src/command_runtime/mod.rs b/crates/sqlbuild-python-front/src/command_runtime/mod.rs new file mode 100644 index 0000000000..0d218ebb66 --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/mod.rs @@ -0,0 +1,4 @@ +//! Concurrency limits that commands read from configuration. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs rename to crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs index e16814e876..30d675df80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs @@ -6,15 +6,14 @@ use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::contracts::effective_config::effective_config_settings; -use crate::bindings::_helpers::contracts::effective_config::effective_config_vars; -use crate::bindings::_helpers::contracts::effective_config::effective_connection_fields; -use crate::bindings::_helpers::contracts::spec_resolution::{ - resolve_target_config, resolve_target_name, -}; -use crate::bindings::_helpers::model_config::template_data::{ - expanded_effective_vars, expanded_template_data, run_id, -}; +use crate::contracts::main::effective_config_settings::effective_config_settings; +use crate::contracts::main::effective_config_vars::effective_config_vars; +use crate::contracts::main::effective_connection_fields::effective_connection_fields; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; +use crate::model_config::main::config::expanded_template_data::expanded_template_data; +use crate::model_config::main::config::run_id::run_id; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -234,3 +233,52 @@ pub(crate) fn project_vars<'py>( PyDict::new(discovered.py()).as_any(), ) } + +/// `build_effective_connection` as a function of `module`. +pub(crate) fn build_effective_connection_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_connection, module) +} + +/// `build_effective_connection_config` as a function of `module`. +pub(crate) fn build_effective_connection_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_connection_config, module) +} + +/// `build_effective_runtime_config` as a function of `module`. +pub(crate) fn build_effective_runtime_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_runtime_config, module) +} + +/// `build_effective_settings` as a function of `module`. +pub(crate) fn build_effective_settings_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_settings, module) +} + +/// `build_effective_settings_config` as a function of `module`. +pub(crate) fn build_effective_settings_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_settings_config, module) +} + +/// `build_effective_target_config` as a function of `module`. +pub(crate) fn build_effective_target_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_target_config, module) +} + +/// `build_effective_vars` as a function of `module`. +pub(crate) fn build_effective_vars_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_effective_vars, module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/_helpers/mod.rs b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/mod.rs new file mode 100644 index 0000000000..89e5014658 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod effective_config; diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs new file mode 100644 index 0000000000..1e07462d8f --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs @@ -0,0 +1,22 @@ +//! Merge base project connection with the selected environment overrides. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Merge base project connection with the selected environment overrides. +#[inline] +pub fn build_effective_connection<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::build_effective_connection( + project_config, + local_config, + target_config, + effective_vars, + ) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs new file mode 100644 index 0000000000..6bde72927b --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs @@ -0,0 +1,22 @@ +//! Build the effective project connection config without compiling resources. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build the effective project connection config without compiling resources. +#[inline] +pub fn build_effective_connection_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, + catalog: bool, +) -> PyResult> { + effective_config::build_effective_connection_config( + discovered_inputs, + selected_target, + cli_vars, + catalog, + ) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs new file mode 100644 index 0000000000..74e098cc13 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_connection_config` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_connection_config` as a function of `module`. +#[inline] +pub fn build_effective_connection_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_connection_config_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs new file mode 100644 index 0000000000..1fd50f688d --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_connection` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_connection` as a function of `module`. +#[inline] +pub fn build_effective_connection_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_connection_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs new file mode 100644 index 0000000000..c4eeb120e1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs @@ -0,0 +1,17 @@ +//! Build environment, vars and invocation id without compiling resources. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Build environment, vars and invocation id without compiling resources. +#[inline] +pub fn build_effective_runtime_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult> { + effective_config::build_effective_runtime_config(discovered_inputs, selected_target, cli_vars) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs new file mode 100644 index 0000000000..2626522098 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_runtime_config` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_runtime_config` as a function of `module`. +#[inline] +pub fn build_effective_runtime_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_runtime_config_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs new file mode 100644 index 0000000000..d5b3f04d24 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs @@ -0,0 +1,15 @@ +//! Merge project settings with local developer overrides. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Merge project settings with local developer overrides. +#[inline] +pub fn build_effective_settings<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::build_effective_settings(project_config, local_config) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs new file mode 100644 index 0000000000..7212e9a1bc --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs @@ -0,0 +1,14 @@ +//! Build effective settings without compiling resources. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build effective settings without compiling resources. +#[inline] +pub fn build_effective_settings_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::build_effective_settings_config(discovered_inputs) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs new file mode 100644 index 0000000000..2bc134c68c --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_settings_config` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_settings_config` as a function of `module`. +#[inline] +pub fn build_effective_settings_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_settings_config_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs new file mode 100644 index 0000000000..4031736371 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_settings` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_settings` as a function of `module`. +#[inline] +pub fn build_effective_settings_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_settings_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs new file mode 100644 index 0000000000..e7f58c2b4e --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs @@ -0,0 +1,15 @@ +//! Build effective target config without compiling resources. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build effective target config without compiling resources. +#[inline] +pub fn build_effective_target_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, +) -> PyResult> { + effective_config::build_effective_target_config(discovered_inputs, selected_target) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs new file mode 100644 index 0000000000..c15c82fa8c --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_target_config` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_target_config` as a function of `module`. +#[inline] +pub fn build_effective_target_config_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_target_config_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs new file mode 100644 index 0000000000..5225b0caad --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs @@ -0,0 +1,17 @@ +//! Merge effective vars using the locked precedence order. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Merge effective vars using the locked precedence order. +#[inline] +pub fn build_effective_vars<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + cli_vars: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::build_effective_vars(project_config, local_config, target_config, cli_vars) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs new file mode 100644 index 0000000000..fa6d8a7d07 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs @@ -0,0 +1,11 @@ +//! `build_effective_vars` as a function of `module`. + +use crate::compile_attachment::_helpers::effective_config; + +/// `build_effective_vars` as a function of `module`. +#[inline] +pub fn build_effective_vars_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + effective_config::build_effective_vars_function(module) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/mod.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/mod.rs new file mode 100644 index 0000000000..0e39ae7323 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/mod.rs @@ -0,0 +1,16 @@ +pub mod build_effective_connection; +pub mod build_effective_connection_config; +pub mod build_effective_connection_config_function; +pub mod build_effective_connection_function; +pub mod build_effective_runtime_config; +pub mod build_effective_runtime_config_function; +pub mod build_effective_settings; +pub mod build_effective_settings_config; +pub mod build_effective_settings_config_function; +pub mod build_effective_settings_function; +pub mod build_effective_target_config; +pub mod build_effective_target_config_function; +pub mod build_effective_vars; +pub mod build_effective_vars_function; +pub mod project_vars; +pub mod target_selection; diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs new file mode 100644 index 0000000000..3215cf81e9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs @@ -0,0 +1,15 @@ +//! `build_effective_vars` for a discovered project and target, without CLI vars. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `build_effective_vars` for a discovered project and target, without CLI vars. +#[inline] +pub fn project_vars<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::project_vars(discovered, target_config) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs new file mode 100644 index 0000000000..db99738dd0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs @@ -0,0 +1,15 @@ +//! The selected target name of a discovered project and its effective config, if any. + +use crate::compile_attachment::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The selected target name of a discovered project and its effective config, if any. +#[inline] +pub fn target_selection<'py>( + discovered: &Bound<'py, PyAny>, + selected: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + effective_config::target_selection(discovered, selected) +} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/mod.rs b/crates/sqlbuild-python-front/src/compile_attachment/mod.rs new file mode 100644 index 0000000000..a775d94fbe --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_attachment/mod.rs @@ -0,0 +1,4 @@ +//! The effective model configuration a compile attaches to each node. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs rename to crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs index ed882ddeec..3cc8fdb394 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs @@ -1,5 +1,6 @@ //! Validate loaded project and local configuration before discovery reads project files. +use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_target_warehouse; @@ -8,12 +9,10 @@ use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::bindings::_helpers::contracts::spec_resolution::{ - resolve_effective_adapter_name, resolve_target_config, -}; -use crate::bindings::_helpers::discovery::project_config::{ - resolve_config_file_path, validate_local_sql_analysis, -}; +use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::discovery::main::project::resolve_config_file_path::resolve_config_file_path; +use crate::discovery::main::project::validate_local_sql_analysis::validate_local_sql_analysis; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -25,13 +24,6 @@ const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// `WarehouseGroup` members in declaration order. const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; -/// The loaded configuration of one project. -pub(crate) struct ProjectConfiguration<'a, 'py> { - pub(crate) project_dir: &'a Bound<'py, PyAny>, - pub(crate) project_config: &'a Bound<'py, PyAny>, - pub(crate) local_config: &'a Bound<'py, PyAny>, -} - fn project_config_error(py: Python<'_>, message: &str, help: Option<&str>) -> PyErr { let raised = (|| -> PyResult> { let keywords: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python-front/src/compile_driver/_helpers/mod.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/mod.rs new file mode 100644 index 0000000000..d59a71cbdd --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod config_validation; diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/mod.rs b/crates/sqlbuild-python-front/src/compile_driver/main/mod.rs new file mode 100644 index 0000000000..908dcdd46b --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/main/mod.rs @@ -0,0 +1,3 @@ +pub mod reject_invalid_connection_configuration; +pub mod reject_invalid_project_configuration; +pub mod session_warehouse_support; diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs new file mode 100644 index 0000000000..4dab78eccb --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs @@ -0,0 +1,13 @@ +//! Validate the target warehouses and target connections a warehouse connection depends on. + +use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::models::ProjectConfiguration; +use pyo3::prelude::PyResult; + +/// Validate the target warehouses and target connections a warehouse connection depends on. +#[inline] +pub fn reject_invalid_connection_configuration( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { + config_validation::reject_invalid_connection_configuration(config) +} diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs new file mode 100644 index 0000000000..f7afdf5744 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs @@ -0,0 +1,11 @@ +//! Validate the SQL analysis policy, target warehouses and target connections, in that order. + +use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::models::ProjectConfiguration; +use pyo3::prelude::PyResult; + +/// Validate the SQL analysis policy, target warehouses and target connections, in that order. +#[inline] +pub fn reject_invalid_project_configuration(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { + config_validation::reject_invalid_project_configuration(config) +} diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs new file mode 100644 index 0000000000..72ca9d4c1e --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs @@ -0,0 +1,17 @@ +//! Whether the adapter can switch warehouses per session, as the adapter host answers. + +use crate::compile_driver::_helpers::config_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Whether the adapter can switch warehouses per session, as the adapter host answers. +#[inline] +pub fn session_warehouse_support( + py: Python<'_>, + adapter_name: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + config_validation::session_warehouse_support(py, adapter_name, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/compile_driver/mod.rs b/crates/sqlbuild-python-front/src/compile_driver/mod.rs new file mode 100644 index 0000000000..3892f4cd62 --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/mod.rs @@ -0,0 +1,5 @@ +//! Validation of the project configuration before a compile starts. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/compile_driver/models.rs b/crates/sqlbuild-python-front/src/compile_driver/models.rs new file mode 100644 index 0000000000..a119c3b8ac --- /dev/null +++ b/crates/sqlbuild-python-front/src/compile_driver/models.rs @@ -0,0 +1,12 @@ +//! Public records of compile driver. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// The loaded configuration of one project. +#[derive(Debug)] +pub struct ProjectConfiguration<'a, 'py> { + pub project_dir: &'a Bound<'py, PyAny>, + pub project_config: &'a Bound<'py, PyAny>, + pub local_config: &'a Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs rename to crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs index d8b03c59bc..4539fa9c21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs @@ -64,7 +64,7 @@ type Pairs<'py> = Entries, Value<'py>>; /// `(group, name, value)` of each merged `TargetConfig` field. type FieldRow<'py> = (Option<&'static str>, &'static str, Value<'py>); /// A result, else the precedence failure message. -type Outcome<'py, T> = (Option, Option>); +pub(crate) type Outcome<'py, T> = (Option, Option>); fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { let handle: Bound<'py, PyString> = value.downcast::()?.clone(); diff --git a/crates/sqlbuild-python-front/src/contracts/_helpers/mod.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/mod.rs new file mode 100644 index 0000000000..4027e9787a --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod effective_config; +pub(crate) mod spec_resolution; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs rename to crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs index 7d7525be19..6fd358ee29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs @@ -12,9 +12,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; use sqlbuild_model_config::name_patterns::models::NamePattern; -use crate::bindings::_helpers::contracts::effective_config::{ - effective_config_target_fields, effective_config_target_name, -}; +use crate::contracts::_helpers::effective_config::effective_config_target_fields; +use crate::contracts::_helpers::effective_config::effective_config_target_name; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs new file mode 100644 index 0000000000..bfe3c29867 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs @@ -0,0 +1,20 @@ +//! `build_effective_settings`: setting values by field name, local overrides applied. + +use crate::contracts::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyString; + +/// `build_effective_settings`: setting values by field name, local overrides applied. +#[inline] +pub fn effective_config_settings<'py>( + py: Python<'py>, + settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, + names: Vec>, +) -> PyResult> { + effective_config::effective_config_settings(py, settings, overrides, names) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs new file mode 100644 index 0000000000..f1cad2a5b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs @@ -0,0 +1,19 @@ +//! `build_effective_vars` before template expansion. + +use crate::contracts::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// `build_effective_vars` before template expansion. +#[inline] +pub fn effective_config_vars<'py>( + py: Python<'py>, + configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + cli_vars: &Bound<'py, PyAny>, +) -> PyResult> { + effective_config::effective_config_vars(py, configs, target_config, cli_vars) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs new file mode 100644 index 0000000000..24e86bbc01 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs @@ -0,0 +1,25 @@ +//! Catalog selection uses the same named-layer resolver, with no writer layers. + +use crate::contracts::_helpers::effective_config; +use crate::contracts::_helpers::effective_config::Outcome; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Catalog selection uses the same named-layer resolver, with no writer layers. +#[inline] +pub fn effective_connection_fields<'py>( + py: Python<'py>, + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + catalog: bool, +) -> PyResult>> { + effective_config::effective_connection_fields( + py, + (project_config, local_config), + target_config, + catalog, + ) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs new file mode 100644 index 0000000000..77dd1720fb --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs @@ -0,0 +1,18 @@ +//! Parse a one-, two-, or three-part loader destination. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Parse a one-, two-, or three-part loader destination. +#[inline] +pub fn loader_destination_parts<'py>( + py: Python<'py>, + destination: &str, + default_database: Option>, + default_schema: Option>, +) -> PyResult> { + spec_resolution::loader_destination_parts(py, destination, default_database, default_schema) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs new file mode 100644 index 0000000000..b710005f65 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs @@ -0,0 +1,17 @@ +//! Loader parts. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Loader parts. +#[inline] +pub fn loader_parts<'py>( + py: Python<'py>, + destination: &str, + (default_database, default_schema): (Option>, Option>), +) -> PyResult> { + spec_resolution::loader_parts(py, destination, (default_database, default_schema)) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs new file mode 100644 index 0000000000..22ee8aafaf --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs @@ -0,0 +1,16 @@ +//! Return declared dynamic families accepting a physical column name. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Return declared dynamic families accepting a physical column name. +#[inline] +pub fn matching_dynamic_families<'py>( + families: &Bound<'py, PyAny>, + column_name: &str, +) -> PyResult> { + spec_resolution::matching_dynamic_families(families, column_name) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/mod.rs b/crates/sqlbuild-python-front/src/contracts/main/mod.rs new file mode 100644 index 0000000000..d4e642d29a --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/mod.rs @@ -0,0 +1,13 @@ +pub mod effective_config_settings; +pub mod effective_config_vars; +pub mod effective_connection_fields; +pub mod loader_destination_parts; +pub mod loader_parts; +pub mod matching_dynamic_families; +pub mod register_effective_config; +pub mod register_spec_resolution; +pub mod resolve_effective_adapter_name; +pub mod resolve_effective_collection_rendering; +pub mod resolve_effective_scenario_config; +pub mod resolve_target_config; +pub mod resolve_target_name; diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs new file mode 100644 index 0000000000..489cc16675 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs @@ -0,0 +1,12 @@ +//! Register the effective config functions and classes. + +use crate::contracts::_helpers::effective_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the effective config functions and classes. +#[inline] +pub fn register_effective_config(module: &Bound<'_, PyModule>) -> PyResult<()> { + effective_config::register(module) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs new file mode 100644 index 0000000000..f866a7667f --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs @@ -0,0 +1,12 @@ +//! Register the spec resolution functions and classes. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the spec resolution functions and classes. +#[inline] +pub fn register_spec_resolution(module: &Bound<'_, PyModule>) -> PyResult<()> { + spec_resolution::register(module) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs new file mode 100644 index 0000000000..5da624dbdf --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs @@ -0,0 +1,15 @@ +//! Resolve the effective adapter name, allowing local override. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve the effective adapter name, allowing local override. +#[inline] +pub fn resolve_effective_adapter_name<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + spec_resolution::resolve_effective_adapter_name(project_config, local_config) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs new file mode 100644 index 0000000000..9c29dfa1aa --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs @@ -0,0 +1,15 @@ +//! Resolve declaration rendering over project configuration. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve declaration rendering over project configuration. +#[inline] +pub fn resolve_effective_collection_rendering<'py>( + project_config: &Bound<'py, PyAny>, + declaration_override: &Bound<'py, PyAny>, +) -> PyResult> { + spec_resolution::resolve_effective_collection_rendering(project_config, declaration_override) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs new file mode 100644 index 0000000000..edce6d7b7b --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs @@ -0,0 +1,15 @@ +//! Resolve scenario config, allowing local overrides to replace project rules. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve scenario config, allowing local overrides to replace project rules. +#[inline] +pub fn resolve_effective_scenario_config<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + spec_resolution::resolve_effective_scenario_config(project_config, local_config) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs new file mode 100644 index 0000000000..1b48b4863a --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs @@ -0,0 +1,18 @@ +//! Merge project target config with local developer overrides. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Merge project target config with local developer overrides. +#[inline] +pub fn resolve_target_config<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + spec_resolution::resolve_target_config(py, project_config, local_config, target_name) +} diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs new file mode 100644 index 0000000000..ea408772b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs @@ -0,0 +1,19 @@ +//! Resolve the effective target name. + +use crate::contracts::_helpers::spec_resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyString; + +/// Resolve the effective target name. +#[inline] +pub fn resolve_target_name<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult>> { + spec_resolution::resolve_target_name(py, project_config, local_config, selected_target) +} diff --git a/crates/sqlbuild-python-front/src/contracts/mod.rs b/crates/sqlbuild-python-front/src/contracts/mod.rs new file mode 100644 index 0000000000..28ac569e0d --- /dev/null +++ b/crates/sqlbuild-python-front/src/contracts/mod.rs @@ -0,0 +1,4 @@ +//! Effective contract configuration and contract spec resolution. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs new file mode 100644 index 0000000000..3e3dd1995e --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs @@ -0,0 +1,11 @@ +//! One call's cursor values read from and built as the native cursor value records. + +use sqlbuild_python_base::native_host::main::host_error::host_error; + +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_host::entries::models::FacadeHost; + +/// `CursorAlgebraError(message)`. +pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { + host_error(py, FacadeHost::CursorAlgebraError, message) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs rename to crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs index a3d345c172..93150ef70a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs @@ -8,10 +8,11 @@ use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan, Moment}; -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::values::{ - DATETIME_MODULE, civil_date, local_date_time, replaced, -}; +use crate::cursor::_helpers::values::civil_date; +use crate::cursor::_helpers::values::local_date_time; +use crate::cursor::_helpers::values::replaced; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs rename to crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs index 01f2d9d977..a63c908d17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs @@ -18,8 +18,8 @@ use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; use sqlbuild_cursor::algebra::models::{Interval, Scalar}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::as_scalar; +use crate::cursor::_helpers::scalars::as_scalar; +use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs new file mode 100644 index 0000000000..fffa7c18fe --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod conversion; +pub(crate) mod durations; +pub(crate) mod intervals; +pub(crate) mod overrides; +pub(crate) mod scalars; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs rename to crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs index 635960276c..05e7930c02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs @@ -5,8 +5,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyInt, PyString, PyStringMethods}; use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; -use crate::bindings::_helpers::cursor::conversion::algebra_error; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::cursor::_helpers::conversion::algebra_error; +use crate::cursor::constants::DATETIME_MODULE; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs rename to crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs index 7efb469c6c..a7cdb7c2f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs @@ -25,8 +25,9 @@ use sqlbuild_cursor::algebra::main::render::render; use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, SortKey}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::cursor::_helpers::conversion::algebra_error; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs rename to crates/sqlbuild-python-front/src/cursor/_helpers/values.rs index 30e3e7bd88..de30f9f798 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs @@ -1,5 +1,6 @@ //! Python `date`, `datetime` and `timedelta` objects read into and built from cursor values. +use crate::cursor::constants::DATETIME_MODULE; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; @@ -8,7 +9,6 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; -pub(crate) const DATETIME_MODULE: &str = "datetime"; const MICROS_PER_SECOND: i64 = 1_000_000; const SECONDS_PER_DAY: i64 = 86_400; diff --git a/crates/sqlbuild-python-front/src/cursor/constants.rs b/crates/sqlbuild-python-front/src/cursor/constants.rs new file mode 100644 index 0000000000..064954ef2f --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/constants.rs @@ -0,0 +1,15 @@ +//! Public constants of cursor. + +use sqlbuild_cursor::algebra::models::Grain; + +/// Cursor grain names, finest first. +pub const GRAIN_NAMES: [(&str, Grain); 6] = [ + ("second", Grain::Second), + ("minute", Grain::Minute), + ("hour", Grain::Hour), + ("day", Grain::Day), + ("month", Grain::Month), + ("year", Grain::Year), +]; + +pub const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs new file mode 100644 index 0000000000..67da83f9c4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs @@ -0,0 +1,11 @@ +//! `CursorAlgebraError(message)`. + +use crate::cursor::_helpers::conversion; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// `CursorAlgebraError(message)`. +#[inline] +pub fn algebra_error(py: Python<'_>, message: &str) -> PyErr { + conversion::algebra_error(py, message) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/conversion/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/conversion/mod.rs new file mode 100644 index 0000000000..8327b9ff89 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/conversion/mod.rs @@ -0,0 +1 @@ +pub mod algebra_error; diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/mod.rs new file mode 100644 index 0000000000..ab6c53c122 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/mod.rs @@ -0,0 +1,4 @@ +pub mod parse_cursor_duration; +pub mod register_durations; +pub mod shift_by_cursor_duration; +pub mod shifted_moment; diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs new file mode 100644 index 0000000000..932140f66b --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs @@ -0,0 +1,16 @@ +//! `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. + +use crate::cursor::_helpers::durations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. +#[inline] +pub fn parse_cursor_duration<'py>( + py: Python<'py>, + value: &str, +) -> PyResult>> { + durations::parse_cursor_duration(py, value) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs new file mode 100644 index 0000000000..77d1ebdb29 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs @@ -0,0 +1,12 @@ +//! Register the durations functions and classes. + +use crate::cursor::_helpers::durations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the durations functions and classes. +#[inline] +pub fn register_durations(module: &Bound<'_, PyModule>) -> PyResult<()> { + durations::register(module) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs new file mode 100644 index 0000000000..9037da5d56 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs @@ -0,0 +1,16 @@ +//! `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. + +use crate::cursor::_helpers::durations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. +#[inline] +pub fn shift_by_cursor_duration<'py>( + moment: &Bound<'py, PyAny>, + duration: &Bound<'py, PyAny>, + direction: i64, +) -> PyResult> { + durations::shift_by_cursor_duration(moment, duration, direction) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs new file mode 100644 index 0000000000..c92de35127 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs @@ -0,0 +1,17 @@ +//! `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. + +use crate::cursor::_helpers::durations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_cursor::algebra::models::DurationSpan; + +/// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. +#[inline] +pub fn shifted_moment<'py>( + moment: &Bound<'py, PyAny>, + span: &DurationSpan, + direction: i64, +) -> PyResult> { + durations::shifted_moment(moment, span, direction) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs new file mode 100644 index 0000000000..549343ce56 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs @@ -0,0 +1,16 @@ +//! Raise `CursorAlgebraError` unless the ends form a valid aligned interval. + +use crate::cursor::_helpers::intervals; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. +#[inline] +pub fn interval_validation( + start: &Bound<'_, PyAny>, + end: &Bound<'_, PyAny>, + grain: &Bound<'_, PyAny>, +) -> PyResult<()> { + intervals::interval_validation(start, end, grain) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/mod.rs new file mode 100644 index 0000000000..1388ee87e4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/mod.rs @@ -0,0 +1,3 @@ +pub mod interval_validation; +pub mod partition; +pub mod register_intervals; diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs new file mode 100644 index 0000000000..7509cc126b --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs @@ -0,0 +1,15 @@ +//! The canonical partition containing an observed physical value. + +use crate::cursor::_helpers::intervals; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The canonical partition containing an observed physical value. +#[inline] +pub fn partition<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + intervals::partition(value, grain) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs new file mode 100644 index 0000000000..33e94ed6ad --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs @@ -0,0 +1,12 @@ +//! Register the intervals functions and classes. + +use crate::cursor::_helpers::intervals; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the intervals functions and classes. +#[inline] +pub fn register_intervals(module: &Bound<'_, PyModule>) -> PyResult<()> { + intervals::register(module) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/mod.rs new file mode 100644 index 0000000000..b90e1ad95e --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/mod.rs @@ -0,0 +1,6 @@ +pub mod conversion; +pub mod durations; +pub mod intervals; +pub mod overrides; +pub mod scalars; +pub mod values; diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs new file mode 100644 index 0000000000..a7fc01d88c --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs @@ -0,0 +1,12 @@ +//! An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. + +use crate::cursor::_helpers::overrides; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. +#[inline] +pub fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + overrides::cursor_integer(value) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs new file mode 100644 index 0000000000..193365f411 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs @@ -0,0 +1,12 @@ +//! A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. + +use crate::cursor::_helpers::overrides; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. +#[inline] +pub fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + overrides::cursor_timestamp(value) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/mod.rs new file mode 100644 index 0000000000..24e520f716 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/mod.rs @@ -0,0 +1,2 @@ +pub mod cursor_integer; +pub mod cursor_timestamp; diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs new file mode 100644 index 0000000000..56617ed205 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs @@ -0,0 +1,18 @@ +//! `raw` unchanged when it is already a typed value, else parsed. + +use crate::cursor::_helpers::scalars; +use crate::cursor::models::CursorSession; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_cursor::algebra::models::Scalar; + +/// `raw` unchanged when it is already a typed value, else parsed. +#[inline] +pub fn as_scalar<'py>( + session: &CursorSession<'py>, + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + scalars::as_scalar(session, raw, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs new file mode 100644 index 0000000000..d8a1465d67 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs @@ -0,0 +1,12 @@ +//! `-1`, `0` or `1` as `left` orders against `right`. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `-1`, `0` or `1` as `left` orders against `right`. +#[inline] +pub fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + scalars::compare_values(left, right) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/mod.rs new file mode 100644 index 0000000000..e4d7f7ee55 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/mod.rs @@ -0,0 +1,10 @@ +pub mod as_scalar; +pub mod compare_values; +pub mod parse; +pub mod parse_value; +pub mod register_scalars; +pub mod render_value; +pub mod sentinel_to_token; +pub mod to_exclusive; +pub mod to_inclusive; +pub mod try_parse; diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs new file mode 100644 index 0000000000..b9d0b5af14 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs @@ -0,0 +1,15 @@ +//! One cursor scalar, preserving date-ness and timestamp offsets. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One cursor scalar, preserving date-ness and timestamp offsets. +#[inline] +pub fn parse<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + scalars::parse(raw, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs new file mode 100644 index 0000000000..198f84f246 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs @@ -0,0 +1,15 @@ +//! One cursor value parsed from its boundary representation, keeping date-ness and offsets. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One cursor value parsed from its boundary representation, keeping date-ness and offsets. +#[inline] +pub fn parse_value<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + scalars::parse_value(raw, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs new file mode 100644 index 0000000000..5d54f46a04 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs @@ -0,0 +1,12 @@ +//! Register the scalars functions and classes. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scalars functions and classes. +#[inline] +pub fn register_scalars(module: &Bound<'_, PyModule>) -> PyResult<()> { + scalars::register(module) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs new file mode 100644 index 0000000000..cc7c06fb71 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs @@ -0,0 +1,12 @@ +//! A scalar in the existing cursor output format. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A scalar in the existing cursor output format. +#[inline] +pub fn render_value(value: &Bound<'_, PyAny>) -> PyResult { + scalars::render_value(value) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs new file mode 100644 index 0000000000..fd2abf4730 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs @@ -0,0 +1,12 @@ +//! A sentinel's token, or a scalar rendered. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A sentinel's token, or a scalar rendered. +#[inline] +pub fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { + scalars::sentinel_to_token(sentinel) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs new file mode 100644 index 0000000000..3f71c47689 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs @@ -0,0 +1,15 @@ +//! An operator-declared inclusive bound advanced by one cursor unit. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An operator-declared inclusive bound advanced by one cursor unit. +#[inline] +pub fn to_exclusive<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + scalars::to_exclusive(value, grain) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs new file mode 100644 index 0000000000..49da0d7d19 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs @@ -0,0 +1,15 @@ +//! An exclusive bound moved back by one cursor unit. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An exclusive bound moved back by one cursor unit. +#[inline] +pub fn to_inclusive<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + scalars::to_inclusive(value, grain) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs new file mode 100644 index 0000000000..aac8b2cfb1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs @@ -0,0 +1,15 @@ +//! Discovery values that may be dates or timestamps, or `None` when they do not parse. + +use crate::cursor::_helpers::scalars; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Discovery values that may be dates or timestamps, or `None` when they do not parse. +#[inline] +pub fn try_parse<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult>> { + scalars::try_parse(raw, cursor_type) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs new file mode 100644 index 0000000000..4a4809c85d --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs @@ -0,0 +1,13 @@ +//! The wall-clock fields of a `datetime`. + +use crate::cursor::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// The wall-clock fields of a `datetime`. +#[inline] +pub fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { + values::local_date_time(value) +} diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/mod.rs b/crates/sqlbuild-python-front/src/cursor/main/values/mod.rs new file mode 100644 index 0000000000..462025e030 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/values/mod.rs @@ -0,0 +1,2 @@ +pub mod local_date_time; +pub mod replaced; diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs new file mode 100644 index 0000000000..c6d6e8c946 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs @@ -0,0 +1,17 @@ +//! `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. + +use crate::cursor::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. +#[inline] +pub fn replaced<'py>( + template: &Bound<'py, PyAny>, + local: &LocalDateTime, + fold: bool, +) -> PyResult> { + values::replaced(template, local, fold) +} diff --git a/crates/sqlbuild-python-front/src/cursor/mod.rs b/crates/sqlbuild-python-front/src/cursor/mod.rs new file mode 100644 index 0000000000..b934be571e --- /dev/null +++ b/crates/sqlbuild-python-front/src/cursor/mod.rs @@ -0,0 +1,6 @@ +//! Cursor values, intervals, durations and overrides converted between Python and Rust. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python-front/src/cursor/models.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs rename to crates/sqlbuild-python-front/src/cursor/models.rs index 366ecd2dd1..7a7a17b513 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python-front/src/cursor/models.rs @@ -1,24 +1,35 @@ -//! One call's cursor values read from and built as the native cursor value records. - -use sqlbuild_python_base::native_host::main::host_error::host_error; -use std::cell::RefCell; +//! Public records of cursor. +use crate::cursor::_helpers::conversion::algebra_error; +use crate::cursor::_helpers::values::civil_date; +use crate::cursor::_helpers::values::local_date_time; +use crate::cursor::_helpers::values::naive_date_time; +use crate::cursor::_helpers::values::plain_date; +use crate::cursor::_helpers::values::replaced; +use crate::cursor::_helpers::values::timedelta_micros; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::constants::GRAIN_NAMES; use pyo3::IntoPyObject; -use pyo3::exceptions::{PyOverflowError, PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::exceptions::PyOverflowError; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::types::PyString; use pyo3::types::PyStringMethods; use sqlbuild_cursor::algebra::errors::CursorError; -use sqlbuild_cursor::algebra::models::{ - Grain, Interval, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, -}; +use sqlbuild_cursor::algebra::models::Grain; +use sqlbuild_cursor::algebra::models::Interval; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_cursor::algebra::models::Scalar; +use sqlbuild_cursor::algebra::models::ScalarValue; +use sqlbuild_cursor::algebra::models::Timestamp; +use sqlbuild_cursor::algebra::models::ZoneOffset; use sqlbuild_cursor::algebra::types::ZoneRules; -use sqlbuild_host::entries::models::FacadeHost; - -use crate::bindings::_helpers::cursor::values::{ - DATETIME_MODULE, civil_date, local_date_time, naive_date_time, plain_date, replaced, - timedelta_micros, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -26,36 +37,30 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; - -/// Cursor grain names, finest first. -pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ - ("second", Grain::Second), - ("minute", Grain::Minute), - ("hour", Grain::Hour), - ("day", Grain::Day), - ("month", Grain::Month), - ("year", Grain::Year), -]; - -/// `CursorAlgebraError(message)`. -pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - host_error(py, FacadeHost::CursorAlgebraError, message) -} +use std::cell::RefCell; /// The cursor values one native call reads, so unchanged inputs come back as the same objects. -pub(crate) struct CursorSession<'py> { - py: Python<'py>, - timestamp_class: Bound<'py, PyAny>, - date_class: Bound<'py, PyAny>, - integer_class: Bound<'py, PyAny>, +pub struct CursorSession<'py> { + pub(crate) py: Python<'py>, + pub(crate) timestamp_class: Bound<'py, PyAny>, + pub(crate) date_class: Bound<'py, PyAny>, + pub(crate) integer_class: Bound<'py, PyAny>, /// Each input value object and the `date`, `datetime` or `int` it holds. - inputs: RefCell, Bound<'py, PyAny>)>>, + pub(crate) inputs: RefCell, Bound<'py, PyAny>)>>, /// The Python error a zone lookup raised, re-raised in place of the algebra's error. - failure: RefCell>, + pub(crate) failure: RefCell>, +} + +impl std::fmt::Debug for CursorSession<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CursorSession") + .finish_non_exhaustive() + } } impl<'py> CursorSession<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { + pub fn new(py: Python<'py>) -> PyResult { Ok(Self { py, timestamp_class: record_class(py, RecordId::TimestampValue)?, @@ -66,18 +71,22 @@ impl<'py> CursorSession<'py> { }) } - pub(crate) fn py(&self) -> Python<'py> { + pub fn py(&self) -> Python<'py> { self.py } /// Whether `value` is one of the three typed cursor values. - pub(crate) fn scalar_typed(&self, value: &Bound<'py, PyAny>) -> PyResult { + pub fn scalar_typed(&self, value: &Bound<'py, PyAny>) -> PyResult { Ok(value.is_instance(&self.timestamp_class)? || value.is_instance(&self.date_class)? || value.is_instance(&self.integer_class)?) } - fn timestamp(&self, inner: &Bound<'py, PyAny>, handle: usize) -> PyResult { + pub(crate) fn timestamp( + &self, + inner: &Bound<'py, PyAny>, + handle: usize, + ) -> PyResult { let tzinfo: Bound<'py, PyAny> = native_field(inner, "tzinfo")?; let offset: ZoneOffset = if tzinfo.is_none() { ZoneOffset::Naive @@ -101,7 +110,7 @@ impl<'py> CursorSession<'py> { } /// A typed cursor value object as a scalar remembering the object. - pub(crate) fn scalar(&self, value: &Bound<'py, PyAny>) -> PyResult { + pub fn scalar(&self, value: &Bound<'py, PyAny>) -> PyResult { let inner: Bound<'py, PyAny> = native_field(value, "value")?; let handle: usize = self.inputs.borrow().len(); let typed: ScalarValue = if value.is_instance(&self.integer_class)? { @@ -124,7 +133,7 @@ impl<'py> CursorSession<'py> { } /// `CursorGrain | None` as a grain; an unknown name fails as `CursorGrain(name)` does. - pub(crate) fn grain(&self, grain: &Bound<'py, PyAny>) -> PyResult> { + pub fn grain(&self, grain: &Bound<'py, PyAny>) -> PyResult> { if grain.is_none() { return Ok(None); } @@ -139,7 +148,7 @@ impl<'py> CursorSession<'py> { } /// An `AlignedInterval` as an interval. - pub(crate) fn interval(&self, interval: &Bound<'py, PyAny>) -> PyResult { + pub fn interval(&self, interval: &Bound<'py, PyAny>) -> PyResult { Ok(Interval { start: self.scalar(&native_field(interval, "start")?)?, end: self.scalar(&native_field(interval, "end")?)?, @@ -147,14 +156,14 @@ impl<'py> CursorSession<'py> { }) } - fn input(&self, handle: usize, inner: bool) -> Bound<'py, PyAny> { + pub(crate) fn input(&self, handle: usize, inner: bool) -> Bound<'py, PyAny> { let inputs = self.inputs.borrow(); let (value, held) = &inputs[handle]; if inner { held.clone() } else { value.clone() } } /// The `datetime` holding `local` in the zone and type of input `template`. - fn date_time(&self, timestamp: &Timestamp) -> PyResult> { + pub(crate) fn date_time(&self, timestamp: &Timestamp) -> PyResult> { match timestamp.template { Some(template) => replaced( &self.input(template, true), @@ -166,7 +175,7 @@ impl<'py> CursorSession<'py> { } /// The input object for an unchanged value, else a new typed cursor value. - pub(crate) fn python_scalar(&self, scalar: &Scalar) -> PyResult> { + pub fn python_scalar(&self, scalar: &Scalar) -> PyResult> { if let Some(handle) = scalar.handle { return Ok(self.input(handle, false)); } @@ -184,7 +193,7 @@ impl<'py> CursorSession<'py> { } /// A grain as its `CursorGrain` member. - pub(crate) fn python_grain(&self, grain: Option) -> PyResult> { + pub fn python_grain(&self, grain: Option) -> PyResult> { let Some(grain) = grain else { return Ok(self.py.None().into_bound(self.py)); }; @@ -196,7 +205,7 @@ impl<'py> CursorSession<'py> { } /// A new `AlignedInterval`, validated as its constructor validates. - pub(crate) fn python_interval(&self, interval: &Interval) -> PyResult> { + pub fn python_interval(&self, interval: &Interval) -> PyResult> { new_record_with( self.py, RecordId::AlignedInterval, @@ -209,7 +218,7 @@ impl<'py> CursorSession<'py> { } /// The Python exception for a failed cursor operation. - pub(crate) fn error(&self, error: CursorError) -> PyErr { + pub fn error(&self, error: CursorError) -> PyErr { if let Some(failure) = self.failure.borrow_mut().take() { return failure; } @@ -222,11 +231,11 @@ impl<'py> CursorSession<'py> { } /// `result`, with a cursor error raised as its Python exception. - pub(crate) fn checked(&self, result: Result) -> PyResult { + pub fn checked(&self, result: Result) -> PyResult { result.map_err(|error| self.error(error)) } - fn zone_call( + pub(crate) fn zone_call( &self, template: usize, local: &LocalDateTime, @@ -235,7 +244,7 @@ impl<'py> CursorSession<'py> { replaced(&self.input(template, true), local, fold) } - fn remember(&self, error: PyErr) -> CursorError { + pub(crate) fn remember(&self, error: PyErr) -> CursorError { let message: String = error.to_string(); *self.failure.borrow_mut() = Some(error); CursorError::Algebra(message) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/mod.rs new file mode 100644 index 0000000000..516ceedfe5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod faults; +pub(crate) mod guidance; +pub(crate) mod raised; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs index c41e4f6ce8..51e2e48d0c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs @@ -9,7 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::models::{DiscoveryFailure, ReadFailure, StageFailure}; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::discovery::_helpers::project::project_files::path_object; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs index 9b3cbd0570..4fe5562215 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs @@ -15,9 +15,10 @@ use sqlbuild_model_config::declarations::models::{ use sqlbuild_model_config::sql_values::errors::SqlValueFailure; use sqlbuild_model_config::sql_values::models::RawSqlValue; -use crate::bindings::_helpers::authored_values::sql_values::{ - SqlValueClasses, raw_value, sql_value, sql_value_error, -}; +use crate::authored_values::main::raw_value::raw_value; +use crate::authored_values::main::sql_value::sql_value; +use crate::authored_values::main::sql_value_error::sql_value_error; +use crate::authored_values::models::SqlValueClasses; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_keys.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_keys.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs index 392338f405..d3418c38d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs @@ -5,9 +5,11 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::sqltext::model_headers::{ - ColumnLocation, ParsedModelHeader, header_failure, model_output_columns, parse_model_headers, -}; +use crate::sqltext::main::header_failure::header_failure; +use crate::sqltext::main::model_output_columns::model_output_columns; +use crate::sqltext::main::parse_model_headers::parse_model_headers; +use crate::sqltext::models::ColumnLocation; +use crate::sqltext::types::ParsedModelHeader; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/key_checks.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/key_checks.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/mod.rs new file mode 100644 index 0000000000..02655c3b04 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod header_declarations; +pub(crate) mod header_keys; +pub(crate) mod header_values; +pub(crate) mod hook_entries; +pub(crate) mod key_checks; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/mod.rs new file mode 100644 index 0000000000..217565b656 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod failures; +pub(crate) mod headers; +pub(crate) mod project; +pub(crate) mod python_files; +pub(crate) mod records; +pub(crate) mod snapshots; +pub(crate) mod sources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs index 74959af2a0..5e720d022e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs @@ -4,15 +4,14 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compile_driver::config_validation::{ - ProjectConfiguration, reject_invalid_connection_configuration, - reject_invalid_project_configuration, -}; -use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; -use crate::bindings::_helpers::discovery::project_config::{ - load_local_config_file, load_project_config_file, warn_deprecated_settings, -}; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; +use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; +use crate::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; +use crate::compile_driver::models::ProjectConfiguration; +use crate::discovery::_helpers::project::project_config::load_local_config_file; +use crate::discovery::_helpers::project::project_config::load_project_config_file; +use crate::discovery::_helpers::project::project_config::warn_deprecated_settings; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/mod.rs new file mode 100644 index 0000000000..f7641386da --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod configuration; +pub(crate) mod locations; +pub(crate) mod project_config; +pub(crate) mod project_facts; +pub(crate) mod project_files; +pub(crate) mod project_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs index a2dca5cebf..003e2a881e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs @@ -1,7 +1,8 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; -use pyo3::FromPyObject; +use crate::discovery::models::ModelFileTexts; +use crate::discovery::models::NativeModelFile; +use crate::discovery::models::NodeRow; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTypeMethods}; @@ -29,18 +30,7 @@ const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ const PRIVATE_SCOPE: &str = "private"; /// `(SQL resource name, owning node index, decorated name)` of one dependency. -type DependencyRow = (Option, Option, Option); - -/// A decorated Python node as the facade describes it. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct NodeRow { - kind: String, - name: String, - path: String, - decorator_config: bool, - depends_on: Vec, -} +pub(crate) type DependencyRow = (Option, Option, Option); /// The facts of `inputs`, with the Python nodes the facade has described. pub(crate) fn project_facts( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs index d591fd8e50..c60e8febc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs @@ -11,16 +11,17 @@ use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOpti use sqlbuild_discovery::tree::models::ProjectTree; use std::path::Path; -use crate::bindings::_helpers::discovery::declaration_records::source_locations; -use crate::bindings::_helpers::discovery::header_declarations::model_header_declarations; -use crate::bindings::_helpers::discovery::model_file_record::{ - DiscoveredPaths, ModelFileParts, NativeModelFile, PendingModelFile, -}; -use crate::bindings::_helpers::discovery::raised::failure_error; -use crate::bindings::_helpers::sqltext::authored_values::{ - map_to_python, optional_authored_value_to_python, -}; -use crate::bindings::models::{ModelTextRequest, NativeProjectTree}; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::headers::header_declarations::model_header_declarations; +use crate::discovery::_helpers::records::declaration_records::source_locations; +use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; +use crate::discovery::_helpers::records::model_file_record::ModelFileParts; +use crate::discovery::_helpers::records::model_file_record::PendingModelFile; +use crate::discovery::_helpers::snapshots::native_classes::ModelTextRequest; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::models::NativeModelFile; +use crate::sqltext::main::map_to_python::map_to_python; +use crate::sqltext::main::optional_authored_value_to_python::optional_authored_value_to_python; use sqlbuild_python_base::types::CompilerDetach; pub(crate) type PyObject = Py; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index f10ad200cf..9086c6bdb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -8,9 +8,11 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; -use crate::bindings::_helpers::discovery::project_facts::{NodeRow, project_facts}; -use crate::bindings::_helpers::discovery::raised::{failure_error, read_error}; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::project::project_facts::project_facts; +use crate::discovery::_helpers::sources::orchestration_files::orchestration_declarations; +use crate::discovery::models::NodeRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; const CSV_MODULE: &str = "csv"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs index 434dbf3a20..7e04bf4870 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs @@ -7,8 +7,8 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_e use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::discovery::locations::project_relative_path; -use crate::bindings::_helpers::discovery::python_module_files::discover_python_module_files; +use crate::discovery::_helpers::project::locations::project_relative_path; +use crate::discovery::_helpers::python_files::python_module_files::discover_python_module_files; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::event_exporting::main::current_scope::current_scope; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs index baa821e746..c592f4f592 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs @@ -9,12 +9,17 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; -use crate::bindings::_helpers::discovery::imported_modules::{ - ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, - provider_by_name, provider_usages, shim, tuple_of, -}; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; +use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; +use crate::discovery::_helpers::python_files::imported_modules::discovered; +use crate::discovery::_helpers::python_files::imported_modules::discovery_error; +use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; +use crate::discovery::_helpers::python_files::imported_modules::module_functions; +use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; +use crate::discovery::_helpers::python_files::imported_modules::provider_usages; +use crate::discovery::_helpers::python_files::imported_modules::shim; +use crate::discovery::_helpers::python_files::imported_modules::tuple_of; +use crate::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; +use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::compile_context::main::input_reads::record_settings_class_read::record_settings_class_read; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index b83d6e6cf2..68221f5848 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -9,12 +9,22 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::PYTHON_NODE_ROOT; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::discovery::imported_modules::{ - ProjectDir, definition, discovered, discovery_error, discovery_error_from, is_exception, - module_functions, provider_by_name, provider_usages, shim, tuple_of, -}; -use crate::bindings::_helpers::discovery::imported_providers; -use crate::bindings::_helpers::discovery::python_hook_files::discover_python_hook_files; +use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; +use crate::discovery::_helpers::python_files::imported_modules::definition; +use crate::discovery::_helpers::python_files::imported_modules::discovered; +use crate::discovery::_helpers::python_files::imported_modules::discovery_error; +use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; +use crate::discovery::_helpers::python_files::imported_modules::is_exception; +use crate::discovery::_helpers::python_files::imported_modules::module_functions; +use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; +use crate::discovery::_helpers::python_files::imported_modules::provider_usages; +use crate::discovery::_helpers::python_files::imported_modules::shim; +use crate::discovery::_helpers::python_files::imported_modules::tuple_of; +use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_functions; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::_helpers::python_files::python_hook_files::discover_python_hook_files; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -561,22 +571,13 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_provider_classes, module)?)?; module.add_function(wrap_pyfunction!( - imported_providers::discover_provider_classes, + discover_event_exporter_declarations, module )?)?; - module.add_function(wrap_pyfunction!( - imported_providers::discover_event_exporter_declarations, - module - )?)?; - module.add_function(wrap_pyfunction!( - imported_providers::bind_event_exporter_declarations, - module - )?)?; - module.add_function(wrap_pyfunction!( - imported_providers::discover_event_exporter_functions, - module - )?) + module.add_function(wrap_pyfunction!(bind_event_exporter_declarations, module)?)?; + module.add_function(wrap_pyfunction!(discover_event_exporter_functions, module)?) } fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/mod.rs new file mode 100644 index 0000000000..fee2f4e219 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod imported_modules; +pub(crate) mod imported_providers; +pub(crate) mod imported_python_nodes; +pub(crate) mod python_function_files; +pub(crate) mod python_hook_files; +pub(crate) mod python_module_files; +pub(crate) mod python_paths; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs index e175f76fe8..68aa0b7878 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs @@ -16,11 +16,15 @@ use sqlbuild_pysource::udf_functions::constants::UDF_KEYS; use sqlbuild_pysource::udf_functions::main::udf_function::udf_function; use sqlbuild_pysource::udf_functions::models::{UdfFunction, UdfShape}; -use crate::bindings::_helpers::discovery::declaration_records::path_class; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; -use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::models::NativeProjectTree; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::declaration_records::path_class; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_root; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs index 4caad201fe..ded9531391 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs @@ -9,13 +9,16 @@ use sqlbuild_discovery::declaration_files::models::{ }; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::bindings::_helpers::discovery::declaration_records::{ - add_scope_fields, discovery_session, path_class, -}; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{failure_error, stage_error}; -use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_tree}; -use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::declaration_records::add_scope_fields; +use crate::discovery::_helpers::records::declaration_records::discovery_session; +use crate::discovery::_helpers::records::declaration_records::path_class; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; /// Each public Python hook file as `file_path`, `relative_path` and its role's scope fields. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs index be655ab2fc..2afbf6ce18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs @@ -6,10 +6,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::python_module_files::python_module_files; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::stage_error; -use crate::bindings::_helpers::discovery::snapshot::project_tree; -use crate::bindings::models::NativeProjectTree; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; /// `sorted((project_dir / root).rglob("*.py"))` without `__init__` modules, or `_` parts too. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs index d5aafff79f..07873c43ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; -use crate::bindings::_helpers::discovery::raised::discovery_error; +use crate::discovery::_helpers::failures::raised::discovery_error; const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ python/, or move SQLBuild extensions under their documented extension path."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs index d3f4e5685f..0e0bb863b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs @@ -17,16 +17,21 @@ use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; -use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; -use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; -use crate::bindings::_helpers::discovery::snapshot::{ - in_snapshot, project_root, project_tree, snapshot_memo, -}; -use crate::bindings::_helpers::model_config::header_columns::schema_columns; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::headers::header_declarations::typed_constant_record; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_root; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; +use crate::discovery::_helpers::snapshots::snapshot::snapshot_memo; +use crate::model_config::main::headers::schema_columns::schema_columns; +use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::declaration_files::main::declaration_file_options::declaration_file_options; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs index 81870f4eda..8789e2ac0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs @@ -4,14 +4,13 @@ use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::discovery::declaration_records::discover_model_schema_files; -use crate::bindings::_helpers::discovery::model_records::{ - ModelFileSelection, discovered_model_files, -}; -use crate::bindings::_helpers::discovery::project_config::load_project_config_file; -use crate::bindings::_helpers::discovery::snapshot::in_snapshot; -use crate::bindings::_helpers::model_config::described_models::externally_described_models as described_models; -use crate::bindings::_helpers::render::declaration_indexes::public_model_schema_index; +use crate::discovery::_helpers::project::project_config::load_project_config_file; +use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; +use crate::discovery::_helpers::records::model_records::discovered_model_files; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::models::ModelFileSelection; +use crate::model_config::main::model_nodes::externally_described_models::externally_described_models as described_models; +use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/getters.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/getters.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/getters.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/getters.rs index 05142b07d6..08e5c1db33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/getters.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/getters.rs @@ -3,24 +3,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; -use crate::bindings::models::NativeDirectorySnapshot; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; +use crate::discovery::models::NativeModelFile; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_native_model_file, - }, - GetterEntry { - class: class_of::, - read: read_native_directory_snapshot, - }, -]; - -fn read_native_model_file<'py>( +pub(crate) fn read_native_model_file<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -58,7 +44,7 @@ fn read_native_model_file<'py>( }) } -fn read_native_directory_snapshot<'py>( +pub(crate) fn read_native_directory_snapshot<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/mod.rs new file mode 100644 index 0000000000..3437db58de --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod declaration_records; +pub(crate) mod description_inputs; +pub(crate) mod getters; +pub(crate) mod model_file_record; +pub(crate) mod model_records; +pub(crate) mod sql_test_records; +pub(crate) mod yaml_records; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs new file mode 100644 index 0000000000..7c0b3caf9b --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs @@ -0,0 +1,383 @@ +//! A discovered SQL model file as a native record whose Python attributes are built on demand. + +use crate::discovery::models::ModelFileTexts; +use crate::discovery::models::NativeModelFile; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::sqltext::main::map_to_python::map_to_python; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{FromPyObject, IntoPyObject}; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; +use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +pub(crate) const FIELD_NAMES: [&str; 10] = [ + "file_path", + "relative_path", + "contents", + "header_values", + "header_column_locations", + "output_column_locations", + "query_sql", + "enum_declarations", + "constant_declarations", + "extract_implicit_alias_columns", +]; +const STEM_ATTRIBUTE: &str = "stem"; + +/// Authored locations: native rows until Python reads them, then a cached dict. +pub(crate) struct Locations { + pub(crate) rows: Vec<(String, LineColumnSpan)>, + pub(crate) projected: PyOnceLock>, +} + +impl std::fmt::Debug for Locations { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Locations").finish_non_exhaustive() + } +} + +impl Locations { + pub(crate) fn from_rows(rows: Vec<(String, LineColumnSpan)>) -> Self { + Self { + rows, + projected: PyOnceLock::new(), + } + } + + /// These locations for another record: the rows, and the dict once Python has read it. + pub(crate) fn shared(&self, py: Python<'_>) -> Self { + let projected: PyOnceLock> = shared_projection(py, &self.projected); + Self { + rows: self.rows.clone(), + projected, + } + } + + pub(crate) fn from_dict(py: Python<'_>, dict: Py) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + let _ = projected.set(py, dict); + Self { + rows: Vec::new(), + projected, + } + } + + /// Python's `{name: SourceLocation(...)}`: a repeated name keeps its first position and last value. + pub(crate) fn dict<'py>(&self, path: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = path.py(); + Ok(self + .projected + .get_or_try_init(py, || { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, span) in &self.rows { + let location: Bound<'py, PyAny> = new_record( + py, + RecordId::SourceLocation, + (path, span.line, span.column, span.end_line, span.end_column), + None, + )?; + dict.set_item(name, location)?; + } + Ok::<_, pyo3::PyErr>(dict.unbind()) + })? + .bind(py) + .clone()) + } +} + +/// A model file's paths: native text, with the Python path objects built on first read. +pub(crate) struct ModelFilePaths { + pub(crate) native_file_path: PathBuf, + /// `None` when a path is not text Python can encode, so readers raise as Python does. + pub(crate) texts: Option, + /// `(file_path, relative_path)` as Python path objects. + pub(crate) objects: PyOnceLock<(Py, Py)>, + /// The relative path the objects are built from, unless they were given. + pub(crate) relative: Option, +} + +impl std::fmt::Debug for ModelFilePaths { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ModelFilePaths") + .finish_non_exhaustive() + } +} + +impl ModelFilePaths { + /// Paths given as Python objects; their text is read once, here. + pub(crate) fn given( + py: Python<'_>, + file_path: Py, + relative_path: Py, + ) -> PyResult { + let native_file_path: PathBuf = file_path.bind(py).extract()?; + let relative: PathBuf = relative_path.bind(py).extract()?; + let texts: Option = + native_file_path + .to_str() + .zip(relative.to_str()) + .map(|(file_text, relative_text)| ModelFileTexts { + name: model_file_stem(&posix(file_text)).to_owned(), + posix_path: posix(relative_text), + relative_path: relative_text.to_owned(), + file_path: file_text.to_owned(), + }); + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + let _ = objects.set(py, (file_path, relative_path)); + Ok(Self { + native_file_path, + texts, + objects, + relative: None, + }) + } + + /// Paths native discovery named; their Python path objects are built only when read. + pub(crate) fn discovered(relative_path: &str, file_path: String) -> Self { + let name: String = model_file_stem(relative_path).to_owned(); + Self { + native_file_path: PathBuf::from(&file_path), + texts: Some(ModelFileTexts { + name, + posix_path: relative_path.to_owned(), + relative_path: relative_path.replace('/', std::path::MAIN_SEPARATOR_STR), + file_path, + }), + objects: PyOnceLock::new(), + relative: Some(PathBuf::from(relative_path)), + } + } + + pub(crate) fn shared(&self, py: Python<'_>) -> Self { + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + if let Some((file_path, relative_path)) = self.objects.get(py) { + let _ = objects.set(py, (file_path.clone_ref(py), relative_path.clone_ref(py))); + } + Self { + native_file_path: self.native_file_path.clone(), + texts: self.texts.clone(), + objects, + relative: self.relative.clone(), + } + } + + /// `(file_path, relative_path)` as the `pathlib` objects discovery names the file by. + pub(crate) fn objects<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let (file_path, relative_path) = self.objects.get_or_try_init(py, || { + let Some(relative) = &self.relative else { + return Err(PyRuntimeError::new_err("a model file has no paths")); + }; + Ok::<_, pyo3::PyErr>(( + self.native_file_path.as_path().into_pyobject(py)?.unbind(), + relative.as_path().into_pyobject(py)?.unbind(), + )) + })?; + Ok((file_path.bind(py).clone(), relative_path.bind(py).clone())) + } + + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. + pub(crate) fn texts(&self, py: Python<'_>) -> PyResult<&ModelFileTexts> { + if let Some(texts) = &self.texts { + return Ok(texts); + } + let (file_path, relative_path) = self.objects(py)?; + for path in [file_path, relative_path] { + path.str()?.to_str()?; + } + Err(PyRuntimeError::new_err("a model file path is not text")) + } +} + +/// `text` with the platform's path separator spelled `/`, as `PurePath.as_posix` spells it. +fn posix(text: &str) -> String { + text.replace(std::path::MAIN_SEPARATOR, "/") +} + +/// `text` as the Python string `cache` shares with every reader, built on first read. +pub(crate) fn projected_text<'py>( + py: Python<'py>, + cache: &PyOnceLock>, + text: &str, +) -> Bound<'py, PyString> { + cache + .get_or_init(py, || PyString::new(py, text).unbind()) + .bind(py) + .clone() +} + +/// A model header's authored values: native entries, with the Python dict built on first read. +pub(crate) struct HeaderValues { + pub(crate) native: Option>, + pub(crate) projected: PyOnceLock>, +} + +impl std::fmt::Debug for HeaderValues { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HeaderValues") + .finish_non_exhaustive() + } +} + +impl HeaderValues { + pub(crate) fn native(values: Vec<(String, AuthoredValue)>) -> Self { + Self { + native: Some(values.into()), + projected: PyOnceLock::new(), + } + } + + pub(crate) fn given(py: Python<'_>, values: Py) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + let _ = projected.set(py, values); + Self { + native: None, + projected, + } + } + + pub(crate) fn shared(&self, py: Python<'_>) -> Self { + let projected: PyOnceLock> = shared_projection(py, &self.projected); + Self { + native: self.native.clone(), + projected, + } + } + + pub(crate) fn dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self + .projected + .get_or_try_init(py, || { + let values: &[(String, AuthoredValue)] = self.native.as_deref().unwrap_or_default(); + map_to_python(py, values)? + .into_bound(py) + .downcast_into::() + .map(Bound::unbind) + .map_err(pyo3::PyErr::from) + })? + .bind(py) + .clone()) + } +} + +/// What native discovery read for one model file. +pub(crate) struct ModelFileParts { + pub(crate) contents: String, + pub(crate) header_values: Vec<(String, AuthoredValue)>, + pub(crate) header_column_locations: Vec<(String, LineColumnSpan)>, + pub(crate) output_column_locations: Vec<(String, LineColumnSpan)>, + pub(crate) query_sql: String, + pub(crate) extract_implicit_alias_columns: bool, +} + +/// How discovery names a model file: natively, or by Python paths for names Python escapes. +pub(crate) enum DiscoveredPaths { + /// The `/`-separated relative path and `str(project_dir / relative_path)`. + Native(String, String), + /// `(file_path, relative_path)` built by Python. + Python(Py, Py), +} + +/// A model file before its header declarations are attached. +pub(crate) struct PendingModelFile { + paths: ModelFilePaths, + parts: ModelFileParts, +} + +impl PendingModelFile { + pub(crate) fn new( + py: Python<'_>, + paths: DiscoveredPaths, + parts: ModelFileParts, + ) -> PyResult { + let paths: ModelFilePaths = match paths { + DiscoveredPaths::Native(relative_path, file_path) => { + ModelFilePaths::discovered(&relative_path, file_path) + } + DiscoveredPaths::Python(file_path, relative_path) => { + ModelFilePaths::given(py, file_path, relative_path)? + } + }; + Ok(Self { paths, parts }) + } + + /// The model's name, its file stem. + pub(crate) fn name(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.name.clone()), + None => field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), + } + } + + /// The relative path as the Python path object the record keeps. + pub(crate) fn relative_path<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self.paths.objects(py)?.1) + } + + /// The authored header value of `key`. + pub(crate) fn header_value(&self, key: &str) -> Option<&AuthoredValue> { + self.parts + .header_values + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value) + } + + pub(crate) fn finished(self, declarations: (Py, Py)) -> NativeModelFile { + let (enum_declarations, constant_declarations) = declarations; + let parts: ModelFileParts = self.parts; + NativeModelFile { + paths: self.paths, + contents: parts.contents.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: HeaderValues::native(parts.header_values), + header_column_locations: Locations::from_rows(parts.header_column_locations), + output_column_locations: Locations::from_rows(parts.output_column_locations), + query_sql: parts.query_sql.into(), + enum_declarations, + constant_declarations, + extract_implicit_alias_columns: parts.extract_implicit_alias_columns, + } + } +} + +/// The keyword fields Python constructs a model file record with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ModelFileFields { + pub(crate) file_path: Py, + pub(crate) relative_path: Py, + pub(crate) contents: Py, + pub(crate) header_values: Py, + pub(crate) header_column_locations: Py, + pub(crate) output_column_locations: Py, + pub(crate) query_sql: Py, + #[pyo3(default)] + pub(crate) enum_declarations: Option>, + #[pyo3(default)] + pub(crate) constant_declarations: Option>, + #[pyo3(default = true)] + pub(crate) extract_implicit_alias_columns: bool, +} + +/// A new cache holding the dict `projected` already holds, if Python has read it. +fn shared_projection(py: Python<'_>, projected: &PyOnceLock>) -> PyOnceLock> { + let shared: PyOnceLock> = PyOnceLock::new(); + if let Some(dict) = projected.get(py) { + let _ = shared.set(py, dict.clone_ref(py)); + } + shared +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs index 9c8b60bc56..606e49bb74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs @@ -1,5 +1,6 @@ //! SQL model files discovered natively and built as Python discovery records. +use crate::discovery::models::ModelFileSelection; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -9,12 +10,18 @@ use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOpti use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use std::path::Path; -use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; -use crate::bindings::_helpers::discovery::model_file_record::DiscoveredPaths; -use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; -use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; -use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::models::NativeProjectTree; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::model_record; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_root; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -69,7 +76,7 @@ fn model_items<'py>( for file in files { let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); if !raw && let FileOutcome::Parsed(model) = file.outcome { - crate::bindings::_helpers::discovery::header_keys::record_header_keys( + crate::discovery::_helpers::headers::header_keys::record_header_keys( &file.relative_path, &model.header_values, ); @@ -100,12 +107,6 @@ fn model_items<'py>( }) } -/// Which discovered model files to build, and where failing files are reported. -pub(crate) struct ModelFileSelection<'py> { - pub(crate) selected_model_names: Option>, - pub(crate) on_fault: Option>, -} - /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. pub(crate) fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs index 5ce345c80d..1dd3ed910b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs @@ -16,17 +16,20 @@ use sqlbuild_discovery::sql_tests::models::{ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; -use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{ - discovery_error, failure_error, read_error, stage_error, -}; -use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; -use crate::bindings::_helpers::discovery::sql_test_blocks::{ - sql_test_block_records, validate_block_names, -}; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::NativeProjectTree; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::discovery_error; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_root; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; +use crate::discovery::_helpers::sources::sql_test_blocks::sql_test_block_records; +use crate::discovery::_helpers::sources::sql_test_blocks::validate_block_names; +use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::constants; use sqlbuild_discovery::sql_tests::main::sql_test_file_options::sql_test_file_options; use sqlbuild_python_base::boundary::main::imported::imported; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs index 5db87ecd59..964e001533 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs @@ -17,18 +17,21 @@ use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; use sqlbuild_python_base::compile_context::main::yaml_cache::retain_read_sources::retain_read_sources; use std::path::PathBuf; -use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{ - discovery_error, failure_error, read_error, stage_error, -}; -use crate::bindings::_helpers::discovery::snapshot::{ - escaped, in_snapshot, project_root, project_tree, -}; -use crate::bindings::_helpers::discovery::yaml_entries::{ - parse_schema_document, parse_sources_document, -}; -use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::discovery_error; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::native_classes::YamlDiscoveryRequest; +use crate::discovery::_helpers::snapshots::snapshot::escaped; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::snapshots::snapshot::project_root; +use crate::discovery::_helpers::snapshots::snapshot::project_tree; +use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; +use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/mod.rs new file mode 100644 index 0000000000..6fa2f0f6ac --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod native_classes; +pub(crate) mod scope_snapshot; +pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs new file mode 100644 index 0000000000..6d3dca266f --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs @@ -0,0 +1,55 @@ +//! Discovery's Python classes and the mapping requests its native calls read. + +use pyo3::{FromPyObject, pyclass}; + +/// The directory listings one discovery pass shares across its native calls. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeProjectTree { + pub(crate) inner: sqlbuild_discovery::tree::models::ProjectTree, +} + +/// One discovery pass of a project: the memo its native calls share, active inside its scope. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeDirectorySnapshot { + pub(crate) project_dir: pyo3::Py, + pub(crate) memo: pyo3::Py, +} + +/// One native YAML file discovery request; `kind` is `source` or `schema`. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct YamlDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) kind: String, +} + +/// One in-memory SQL model parse request; `file_path` is the path messages name. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct ModelTextRequest { + pub(crate) file_path: String, + pub(crate) supported_keys: Vec, + pub(crate) removed_keys: Vec, + pub(crate) extract_implicit_alias_columns: bool, + pub(crate) extract_output_column_locations: bool, +} + +/// The declaration collections one Python discovery pass read natively, kept for later stages. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct NativeDiscoverySession { + pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, +} + +/// One in-memory declaration file parse request; `kind` names the collection it belongs to. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct DeclarationTextRequest { + pub(crate) kind: String, + pub(crate) function_keys: Vec, + pub(crate) audit_keys: Vec, + pub(crate) hook_keys: Vec, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs index 4be9b1b1b5..ccffda6237 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs @@ -7,21 +7,22 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::discovery::declaration_records::{ - discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, - discover_model_schema_files, discover_sql_function_files, discover_sql_hook_files, path_class, -}; -use crate::bindings::_helpers::discovery::model_records::{ - ModelFileSelection, discovered_model_files, -}; -use crate::bindings::_helpers::discovery::project_config::{ - load_local_config_file, load_project_config_file, -}; -use crate::bindings::_helpers::discovery::snapshot::in_snapshot; -use crate::bindings::_helpers::discovery::sql_test_records::{ - discover_scenario_files, discover_test_files, -}; -use crate::bindings::_helpers::discovery::yaml_records::discover_source_files; +use crate::discovery::_helpers::project::project_config::load_local_config_file; +use crate::discovery::_helpers::project::project_config::load_project_config_file; +use crate::discovery::_helpers::records::declaration_records::discover_audit_files; +use crate::discovery::_helpers::records::declaration_records::discover_constant_files; +use crate::discovery::_helpers::records::declaration_records::discover_enum_files; +use crate::discovery::_helpers::records::declaration_records::discover_macro_files; +use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; +use crate::discovery::_helpers::records::declaration_records::discover_sql_function_files; +use crate::discovery::_helpers::records::declaration_records::discover_sql_hook_files; +use crate::discovery::_helpers::records::declaration_records::path_class; +use crate::discovery::_helpers::records::model_records::discovered_model_files; +use crate::discovery::_helpers::records::sql_test_records::discover_scenario_files; +use crate::discovery::_helpers::records::sql_test_records::discover_test_files; +use crate::discovery::_helpers::records::yaml_records::discover_source_files; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::models::ModelFileSelection; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs index 82b5c7185e..a8d137b27b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs @@ -11,10 +11,10 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_python_base::boundary::main::context_variable::context_variable; use std::path::PathBuf; -use crate::bindings::_helpers::discovery::raised::discovery_error; -use crate::bindings::models::{ - NativeDirectorySnapshot, NativeDirectorySnapshotScope, NativeProjectTree, -}; +use crate::discovery::_helpers::failures::raised::discovery_error; +use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::models::NativeDirectorySnapshotScope; const SNAPSHOT_VARIABLE: &str = "sqlbuild_directory_snapshot"; const TREE_MEMO_KEY: &str = "native_project_tree"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs index 97cd48cae0..029e093f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs @@ -9,7 +9,8 @@ use sqlbuild_discovery::declaration_files::models::{ }; use sqlbuild_discovery::models::FileOutcome; -use crate::bindings::models::{DeclarationTextRequest, NativeDiscoverySession}; +use crate::discovery::_helpers::snapshots::native_classes::DeclarationTextRequest; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; use sqlbuild_python_base::types::CompilerDetach; const PARSED_TAG: &str = "ok"; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/mod.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/mod.rs new file mode 100644 index 0000000000..73b1922f12 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declaration_files; +pub(crate) mod orchestration_files; +pub(crate) mod orchestration_values; +pub(crate) mod provider_secrets; +pub(crate) mod sql_test_blocks; +pub(crate) mod yaml_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs index 91ec165356..43e1733cdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs @@ -9,10 +9,10 @@ use sqlbuild_discovery::orchestration_files::models::{ }; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::bindings::_helpers::discovery::orchestration_values::check_orchestration_values; -use crate::bindings::_helpers::discovery::raised::{ - failure_error, relative_read_error, stage_error, -}; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::failures::raised::relative_read_error; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; /// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). pub(crate) fn orchestration_declarations( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs index e80a1f7e5a..2b83ef870b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs @@ -2,7 +2,6 @@ use std::cell::Cell; -use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_secrets; @@ -15,26 +14,7 @@ const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; thread_local! { /// Whether the invocation running on this thread passed `--allow-missing-secrets`. - static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; -} - -/// `--allow-missing-secrets` for one invocation, restored when the invocation ends. -pub(crate) struct MissingSecretsScope { - previous: bool, -} - -impl MissingSecretsScope { - pub(crate) fn enter(allowed: bool) -> Self { - Self { - previous: INVOCATION_ALLOWS_MISSING_SECRETS.replace(allowed), - } - } -} - -impl Drop for MissingSecretsScope { - fn drop(&mut self) { - INVOCATION_ALLOWS_MISSING_SECRETS.set(self.previous); - } + pub(crate) static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; } /// Whether the running invocation passed `--allow-missing-secrets`. @@ -42,35 +22,6 @@ pub(crate) fn invocation_allows_missing_secrets() -> bool { INVOCATION_ALLOWS_MISSING_SECRETS.get() } -/// What discovery does when a provider's required secrets are unset. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum SecretPolicy { - /// Fail with D012: static commands by default. - Require, - /// Continue and list the providers on stderr: static commands that opted out. - Allow, - /// Continue silently; the command checks the providers it uses before running anything. - Defer, -} - -impl SecretPolicy { - /// The keyword value that selects the policy. - pub(crate) const fn name(self) -> &'static str { - match self { - Self::Require => "require", - Self::Allow => "allow", - Self::Defer => "defer", - } - } - - pub(crate) fn parse(value: &str) -> PyResult { - [Self::Require, Self::Allow, Self::Defer] - .into_iter() - .find(|policy| policy.name() == value) - .ok_or_else(|| PyValueError::new_err(format!("unknown secret policy '{value}'"))) - } -} - /// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. pub(crate) fn provider_uses( usages: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs index 935e061d38..927a0d89ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs @@ -14,9 +14,9 @@ use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_nam use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue}; -use crate::bindings::_helpers::authored_values::sql_values::{ - SqlValueClasses, raw_value, sql_value, -}; +use crate::authored_values::main::raw_value::raw_value; +use crate::authored_values::main::sql_value::sql_value; +use crate::authored_values::models::SqlValueClasses; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::error_class::error_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs rename to crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs index e04bf5ca27..74a8d6921f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs @@ -13,9 +13,10 @@ use sqlbuild_discovery::yaml_entries::models::{ }; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::contract_classes::yaml_entry_classes; -use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; +use crate::model_config::main::headers::audit_tuple::audit_tuple; +use crate::model_config::main::model_nodes::yaml_entry_classes::yaml_entry_classes; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::PyNode; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-front/src/discovery/constants.rs b/crates/sqlbuild-python-front/src/discovery/constants.rs new file mode 100644 index 0000000000..0f5b228000 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/constants.rs @@ -0,0 +1,42 @@ +//! Public constants of discovery. + +use crate::discovery::_helpers::records::getters::read_native_directory_snapshot; +use crate::discovery::_helpers::records::getters::read_native_model_file; +use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; +use crate::discovery::models::NativeModelFile; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The fields a SQL hook entry reports, in output order. +pub const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ + "statement", + "name", + "relative_path", + "definition_sql", + "kwargs", + "description", +]; + +/// The flag of the static commands that lets them compile without provider secrets. +pub(crate) const ALLOW_MISSING_SECRETS_FLAG: &str = "--allow-missing-secrets"; + +/// The environment variable that sets `--allow-missing-secrets`. +pub const ALLOW_MISSING_SECRETS_ENV: &str = "SQLBUILD_ALLOW_MISSING_SECRETS"; + +/// The `[settings]` key of `sqlbuild_local.toml` that sets `--allow-missing-secrets`. +pub const ALLOW_MISSING_SECRETS_SETTING: &str = "allow_missing_secrets"; + +/// Environment values that switch a boolean `SQLBUILD_*` variable on. +pub const TRUE_ENV_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_native_model_file, + }, + GetterEntry { + class: class_of::, + read: read_native_directory_snapshot, + }, +]; diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs new file mode 100644 index 0000000000..9fbbcfeb55 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs @@ -0,0 +1,9 @@ +//! Explicit references help. + +use crate::discovery::_helpers::failures::guidance; + +/// Explicit references help. +#[inline] +pub fn explicit_references_help(allowed: &str) -> String { + guidance::explicit_references_help(allowed) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs new file mode 100644 index 0000000000..6b6a9694f2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs @@ -0,0 +1,9 @@ +//! Microbatch guidance. + +use crate::discovery::_helpers::failures::guidance; + +/// Microbatch guidance. +#[inline] +pub fn microbatch_guidance() -> (String, String) { + guidance::microbatch_guidance() +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/mod.rs new file mode 100644 index 0000000000..c845fc4dea --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/mod.rs @@ -0,0 +1,6 @@ +pub mod explicit_references_help; +pub mod microbatch_guidance; +pub mod register_guidance; +pub mod register_raised; +pub mod setting_help; +pub mod sql_analysis_guidance; diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs new file mode 100644 index 0000000000..1de4ee6af6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs @@ -0,0 +1,12 @@ +//! Register the guidance functions and classes. + +use crate::discovery::_helpers::failures::guidance; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the guidance functions and classes. +#[inline] +pub fn register_guidance(module: &Bound<'_, PyModule>) -> PyResult<()> { + guidance::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs new file mode 100644 index 0000000000..923c3390b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs @@ -0,0 +1,12 @@ +//! Register the raised functions and classes. + +use crate::discovery::_helpers::failures::raised; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the raised functions and classes. +#[inline] +pub fn register_raised(module: &Bound<'_, PyModule>) -> PyResult<()> { + raised::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs new file mode 100644 index 0000000000..e63a8de597 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs @@ -0,0 +1,13 @@ +//! Render `, set this in :` followed by the exact TOML section and key. + +use crate::discovery::_helpers::failures::guidance; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Render `, set this in :` followed by the exact TOML section and key. +#[inline] +pub fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { + guidance::setting_help(py, options) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs new file mode 100644 index 0000000000..4073b59569 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs @@ -0,0 +1,15 @@ +//! The note and help for turning SQL analysis on, or `None` when it is effectively on. + +use crate::discovery::_helpers::failures::guidance; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The note and help for turning SQL analysis on, or `None` when it is effectively on. +#[inline] +pub fn sql_analysis_guidance( + discovered_inputs: &Bound<'_, PyAny>, + purpose: &str, +) -> PyResult> { + guidance::sql_analysis_guidance(discovered_inputs, purpose) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs new file mode 100644 index 0000000000..5467e5d106 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs @@ -0,0 +1,18 @@ +//! Serialize SQL and Python hook entries; other values serialize to no hooks. + +use crate::discovery::_helpers::headers::hook_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyList; + +/// Serialize SQL and Python hook entries; other values serialize to no hooks. +#[inline] +pub fn hook_entries<'py>( + value: &Bound<'py, PyAny>, + sql_fields: &[String], + python_hook_fields: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + hook_entries::hook_entries(value, sql_fields, python_hook_fields) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/mod.rs new file mode 100644 index 0000000000..e0c664d7db --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/mod.rs @@ -0,0 +1,5 @@ +pub mod hook_entries; +pub mod register_header_declarations; +pub mod register_header_values; +pub mod register_hook_entries; +pub mod register_key_checks; diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs new file mode 100644 index 0000000000..fda14402b4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs @@ -0,0 +1,12 @@ +//! Register the header declarations functions and classes. + +use crate::discovery::_helpers::headers::header_declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the header declarations functions and classes. +#[inline] +pub fn register_header_declarations(module: &Bound<'_, PyModule>) -> PyResult<()> { + header_declarations::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs new file mode 100644 index 0000000000..ae668cb5c5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs @@ -0,0 +1,12 @@ +//! Register the header values functions and classes. + +use crate::discovery::_helpers::headers::header_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the header values functions and classes. +#[inline] +pub fn register_header_values(module: &Bound<'_, PyModule>) -> PyResult<()> { + header_values::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs new file mode 100644 index 0000000000..cd409be58a --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs @@ -0,0 +1,12 @@ +//! Register the hook entries functions and classes. + +use crate::discovery::_helpers::headers::hook_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the hook entries functions and classes. +#[inline] +pub fn register_hook_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + hook_entries::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs new file mode 100644 index 0000000000..c09f4fc734 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs @@ -0,0 +1,12 @@ +//! Register the key checks functions and classes. + +use crate::discovery::_helpers::headers::key_checks; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the key checks functions and classes. +#[inline] +pub fn register_key_checks(module: &Bound<'_, PyModule>) -> PyResult<()> { + key_checks::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/mod.rs new file mode 100644 index 0000000000..6955d56242 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/mod.rs @@ -0,0 +1,7 @@ +pub mod failures; +pub mod headers; +pub mod project; +pub mod python_files; +pub mod records; +pub mod snapshots; +pub mod sources; diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs new file mode 100644 index 0000000000..6c6a3da67e --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs @@ -0,0 +1,12 @@ +//! Column name. + +use crate::discovery::_helpers::project::project_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Column name. +#[inline] +pub fn column_name(column: &Bound<'_, PyAny>) -> PyResult { + project_facts::column_name(column) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs new file mode 100644 index 0000000000..c647110437 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs @@ -0,0 +1,13 @@ +//! `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_config::project::models::ChangesConfig; + +/// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. +#[inline] +pub fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { + project_config::configured_changes(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs new file mode 100644 index 0000000000..453f1ecffe --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs @@ -0,0 +1,12 @@ +//! `[git] base_ref` of the project at `project_dir`, which `--since @base` names. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. +#[inline] +pub fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { + project_config::configured_git_base_ref(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs new file mode 100644 index 0000000000..37b4f3abc4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs @@ -0,0 +1,13 @@ +//! `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_config::project::models::ReviewConfig; + +/// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. +#[inline] +pub fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { + project_config::configured_review(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs new file mode 100644 index 0000000000..5a7dc7607d --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs @@ -0,0 +1,18 @@ +//! The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. + +use crate::discovery::_helpers::project::configuration; +use crate::discovery::models::SecretPolicy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. +#[inline] +pub fn discover_configured_providers<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + secret_policy: SecretPolicy, +) -> PyResult> { + configuration::discover_configured_providers(py, project_dir, secret_policy) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs new file mode 100644 index 0000000000..72edfe0943 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs @@ -0,0 +1,16 @@ +//! The validated configuration without a discovery lifecycle, which reading two files never needs. + +use crate::discovery::_helpers::project::configuration; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The validated configuration without a discovery lifecycle, which reading two files never needs. +#[inline] +pub fn discover_connection_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration::discover_connection_configuration(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs new file mode 100644 index 0000000000..cec76516c8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs @@ -0,0 +1,16 @@ +//! `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. + +use crate::discovery::_helpers::project::configuration; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. +#[inline] +pub fn discover_project_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration::discover_project_configuration(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/each.rs b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs new file mode 100644 index 0000000000..55055191f5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs @@ -0,0 +1,16 @@ +//! `read` applied to each item of the iterable attribute `name` of `owner`. + +use crate::discovery::_helpers::project::project_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `read` applied to each item of the iterable attribute `name` of `owner`. +#[inline] +pub fn each( + owner: &Bound<'_, PyAny>, + name: &str, + read: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult> { + project_facts::each(owner, name, read) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs new file mode 100644 index 0000000000..cf145a6ad4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs @@ -0,0 +1,17 @@ +//! `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. +#[inline] +pub fn load_local_config_file( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult> { + project_config::load_local_config_file(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs new file mode 100644 index 0000000000..e72087f8ec --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs @@ -0,0 +1,17 @@ +//! `load_project_config`: the validated `ProjectConfig` of `project_dir`. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `load_project_config`: the validated `ProjectConfig` of `project_dir`. +#[inline] +pub fn load_project_config_file( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult> { + project_config::load_project_config_file(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs new file mode 100644 index 0000000000..00673e75cd --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -0,0 +1,23 @@ +pub mod column_name; +pub mod configured_changes; +pub mod configured_git_base_ref; +pub mod configured_review; +pub mod discover_configured_providers; +pub mod discover_connection_configuration; +pub mod discover_project_configuration; +pub mod each; +pub mod load_local_config_file; +pub mod load_project_config_file; +pub mod named_node_rows; +pub mod project_facts; +pub mod register_locations; +pub mod register_project_config; +pub mod register_project_files; +pub mod register_project_validation; +pub mod resolve_config_file_path; +pub mod seed_csv_settings; +pub mod stem; +pub mod validate_local_sql_analysis; +pub mod validation_error; +pub mod warn_deprecated_settings; +pub mod yaml_entry_line; diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs new file mode 100644 index 0000000000..c7291aa22e --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs @@ -0,0 +1,13 @@ +//! The discovered Python nodes by kind, name and path only, as the resource-name check reads them. + +use crate::discovery::_helpers::project::project_facts; +use crate::discovery::models::NodeRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. +#[inline] +pub fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { + project_facts::named_node_rows(inputs) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs new file mode 100644 index 0000000000..4cd226e6e7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs @@ -0,0 +1,19 @@ +//! The facts of `inputs`, with the Python nodes the facade has described. + +use crate::discovery::_helpers::project::project_facts; +use crate::discovery::models::NodeRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_discovery::project_validation::models::ProjectFacts; + +/// The facts of `inputs`, with the Python nodes the facade has described. +#[inline] +pub fn project_facts( + py: Python<'_>, + inputs: &Bound<'_, PyAny>, + nodes: Vec, +) -> PyResult { + project_facts::project_facts(py, inputs, nodes) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs new file mode 100644 index 0000000000..8765670175 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs @@ -0,0 +1,12 @@ +//! Register the locations functions and classes. + +use crate::discovery::_helpers::project::locations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the locations functions and classes. +#[inline] +pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { + locations::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs new file mode 100644 index 0000000000..065654f6f6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs @@ -0,0 +1,12 @@ +//! Register the project config functions and classes. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the project config functions and classes. +#[inline] +pub fn register_project_config(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_config::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs new file mode 100644 index 0000000000..117895dbe5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs @@ -0,0 +1,12 @@ +//! Whether native discovery reproduces the string semantics of this Python and Unicode version. + +use crate::discovery::_helpers::project::project_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Whether native discovery reproduces the string semantics of this Python and Unicode version. +#[inline] +pub fn register_project_files(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_files::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs new file mode 100644 index 0000000000..b2f59cb128 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs @@ -0,0 +1,12 @@ +//! Register the project validation functions and classes. + +use crate::discovery::_helpers::project::project_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the project validation functions and classes. +#[inline] +pub fn register_project_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_validation::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs new file mode 100644 index 0000000000..72d5f0368a --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs @@ -0,0 +1,17 @@ +//! The project or local configuration file path, rejecting an unconverted legacy file. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The project or local configuration file path, rejecting an unconverted legacy file. +#[inline] +pub fn resolve_config_file_path<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + local: bool, +) -> PyResult> { + project_config::resolve_config_file_path(py, project_dir, local) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs new file mode 100644 index 0000000000..c8f03c42c9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs @@ -0,0 +1,13 @@ +//! The CSV reader settings a seed declaration authors, without the dialect check. + +use crate::discovery::_helpers::project::project_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_discovery::project_validation::models::CsvSettings; + +/// The CSV reader settings a seed declaration authors, without the dialect check. +#[inline] +pub fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { + project_facts::seed_csv_settings(entry) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs new file mode 100644 index 0000000000..aadcc56de9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs @@ -0,0 +1,12 @@ +//! `str(Path(...).stem)` of the path attribute `name`. + +use crate::discovery::_helpers::project::project_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `str(Path(...).stem)` of the path attribute `name`. +#[inline] +pub fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { + project_facts::stem(owner, name) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs new file mode 100644 index 0000000000..9b0eb51bbd --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs @@ -0,0 +1,17 @@ +//! `validate_local_sql_analysis_policy` from the project requirement and the local setting. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// `validate_local_sql_analysis_policy` from the project requirement and the local setting. +#[inline] +pub fn validate_local_sql_analysis( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, + flags: (bool, bool, bool), +) -> Result<(), PyErr> { + project_config::validate_local_sql_analysis(py, project_dir, flags) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs new file mode 100644 index 0000000000..f8785d84ce --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs @@ -0,0 +1,12 @@ +//! The Python exception a failed project validation raises. + +use crate::discovery::_helpers::project::project_validation; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; +use sqlbuild_discovery::project_validation::models::ValidationFailure; + +/// The Python exception a failed project validation raises. +#[inline] +pub fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { + project_validation::validation_error(py, failure) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs new file mode 100644 index 0000000000..d3b3a3bf12 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs @@ -0,0 +1,13 @@ +//! Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. + +use crate::discovery::_helpers::project::project_config; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. +#[inline] +pub fn warn_deprecated_settings(py: Python<'_>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { + project_config::warn_deprecated_settings(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs new file mode 100644 index 0000000000..b524063a91 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs @@ -0,0 +1,9 @@ +//! The 1-based line of the outermost `- name: ` entry, if present. + +use crate::discovery::_helpers::project::locations; + +/// The 1-based line of the outermost `- name: ` entry, if present. +#[inline] +pub fn yaml_entry_line(contents: &str, name: &str) -> Option { + locations::yaml_entry_line(contents, name) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs new file mode 100644 index 0000000000..acb768a560 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs @@ -0,0 +1,15 @@ +//! Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. + +use crate::discovery::_helpers::python_files::imported_providers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. +#[inline] +pub fn discover_event_exporter_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + imported_providers::discover_event_exporter_functions(project_dir, providers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs new file mode 100644 index 0000000000..f046e4db10 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs @@ -0,0 +1,15 @@ +//! Decorated model lifecycle hook functions under hooks/python/. + +use crate::discovery::_helpers::python_files::imported_python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Decorated model lifecycle hook functions under hooks/python/. +#[inline] +pub fn discover_hook_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + imported_python_nodes::discover_hook_functions(project_dir, providers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs new file mode 100644 index 0000000000..2b665bfcc1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs @@ -0,0 +1,15 @@ +//! Custom materialization Python files under materializations/. + +use crate::discovery::_helpers::python_files::imported_python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Custom materialization Python files under materializations/. +#[inline] +pub fn discover_materialization_files<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + imported_python_nodes::discover_materialization_files(project_dir, providers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs new file mode 100644 index 0000000000..c261a096ca --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs @@ -0,0 +1,15 @@ +//! Provider classes under providers/. + +use crate::discovery::_helpers::python_files::imported_providers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Provider classes under providers/. +#[inline] +pub fn discover_provider_classes<'py>( + project_dir: Bound<'py, PyAny>, + secret_policy: &str, +) -> PyResult> { + imported_providers::discover_provider_classes(project_dir, secret_policy) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs new file mode 100644 index 0000000000..89093f5de4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs @@ -0,0 +1,15 @@ +//! Discover the `@udf(...)` Python function files under `functions/python/`. + +use crate::discovery::_helpers::python_files::python_function_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover the `@udf(...)` Python function files under `functions/python/`. +#[inline] +pub fn discover_python_function_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + python_function_files::discover_python_function_files(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs new file mode 100644 index 0000000000..ab6f9875ce --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs @@ -0,0 +1,15 @@ +//! Decorated Python DAG node functions under python/. + +use crate::discovery::_helpers::python_files::imported_python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Decorated Python DAG node functions under python/. +#[inline] +pub fn discover_python_node_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + imported_python_nodes::discover_python_node_functions(project_dir, providers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs new file mode 100644 index 0000000000..7cfe250281 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs @@ -0,0 +1,11 @@ +//! Install the `RecallWriter` class sink parameters are recognised by, once, at registration. + +use crate::discovery::_helpers::python_files::imported_providers; +use pyo3::prelude::Bound; +use pyo3::types::PyType; + +/// Install the `RecallWriter` class sink parameters are recognised by, once, at registration. +#[inline] +pub fn install_recall_writer(writer: &Bound<'_, PyType>) { + imported_providers::install_recall_writer(writer) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/mod.rs new file mode 100644 index 0000000000..4b4b592968 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/mod.rs @@ -0,0 +1,15 @@ +pub mod discover_event_exporter_functions; +pub mod discover_hook_functions; +pub mod discover_materialization_files; +pub mod discover_provider_classes; +pub mod discover_python_function_files; +pub mod discover_python_node_functions; +pub mod install_recall_writer; +pub mod node_identity; +pub mod provider_by_name; +pub mod provider_usages; +pub mod register_imported_python_nodes; +pub mod register_python_function_files; +pub mod register_python_hook_files; +pub mod register_python_module_files; +pub mod reject_unsupported_python_paths; diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs new file mode 100644 index 0000000000..49f66190e2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs @@ -0,0 +1,17 @@ +//! The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. + +use crate::discovery::_helpers::python_files::imported_python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. +#[inline] +pub fn node_identity<'py>( + py: Python<'py>, + (kind, documented): (&str, bool), + get: &dyn Fn(&str) -> PyResult>, +) -> PyResult> { + imported_python_nodes::node_identity(py, (kind, documented), get) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs new file mode 100644 index 0000000000..24098feb21 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs @@ -0,0 +1,13 @@ +//! `{provider.name: provider for provider in providers}`. + +use crate::discovery::_helpers::python_files::imported_modules; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// `{provider.name: provider for provider in providers}`. +#[inline] +pub fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { + imported_modules::provider_by_name(providers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs new file mode 100644 index 0000000000..5685f5b30d --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs @@ -0,0 +1,16 @@ +//! The provider parameters of one loaded user function. + +use crate::discovery::_helpers::python_files::imported_modules; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The provider parameters of one loaded user function. +#[inline] +pub fn provider_usages<'py>( + function: &Bound<'py, PyAny>, + provider_by_name: &Bound<'py, PyDict>, +) -> PyResult> { + imported_modules::provider_usages(function, provider_by_name) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs new file mode 100644 index 0000000000..ea02864cab --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs @@ -0,0 +1,12 @@ +//! Register the imported python nodes functions and classes. + +use crate::discovery::_helpers::python_files::imported_python_nodes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the imported python nodes functions and classes. +#[inline] +pub fn register_imported_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { + imported_python_nodes::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs new file mode 100644 index 0000000000..8913e46dae --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs @@ -0,0 +1,12 @@ +//! Register the python function files functions and classes. + +use crate::discovery::_helpers::python_files::python_function_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python function files functions and classes. +#[inline] +pub fn register_python_function_files(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_function_files::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs new file mode 100644 index 0000000000..3c7f62435a --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs @@ -0,0 +1,12 @@ +//! Register the python hook files functions and classes. + +use crate::discovery::_helpers::python_files::python_hook_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python hook files functions and classes. +#[inline] +pub fn register_python_hook_files(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_hook_files::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs new file mode 100644 index 0000000000..8939ee1aec --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs @@ -0,0 +1,12 @@ +//! Register the python module files functions and classes. + +use crate::discovery::_helpers::python_files::python_module_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python module files functions and classes. +#[inline] +pub fn register_python_module_files(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_module_files::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs new file mode 100644 index 0000000000..af1736586b --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs @@ -0,0 +1,12 @@ +//! Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. + +use crate::discovery::_helpers::python_files::python_paths; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. +#[inline] +pub fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { + python_paths::reject_unsupported_python_paths(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs new file mode 100644 index 0000000000..e4962b97a9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs @@ -0,0 +1,12 @@ +//! Detect a project-level adapter.py without importing it. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Detect a project-level adapter.py without importing it. +#[inline] +pub fn discover_adapter_file<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + declaration_records::discover_adapter_file(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs new file mode 100644 index 0000000000..19af629ec1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs @@ -0,0 +1,16 @@ +//! Discover generic and singular audit SQL files in their project-wide and scoped roles. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover generic and singular audit SQL files in their project-wide and scoped roles. +#[inline] +pub fn discover_audit_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + declaration_records::discover_audit_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs new file mode 100644 index 0000000000..05ec7b9656 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs @@ -0,0 +1,17 @@ +//! Discover global and scoped constant declaration files. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover global and scoped constant declaration files. +#[inline] +pub fn discover_constant_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + declaration_records::discover_constant_files(project_dir, on_fault, isolate_declaration_kind) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs new file mode 100644 index 0000000000..d9eacb8305 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs @@ -0,0 +1,17 @@ +//! Discover global and scoped enum declaration files. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover global and scoped enum declaration files. +#[inline] +pub fn discover_enum_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, + isolate_declaration_kind: bool, +) -> PyResult> { + declaration_records::discover_enum_files(project_dir, on_fault, isolate_declaration_kind) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs new file mode 100644 index 0000000000..fbd7a0a5da --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs @@ -0,0 +1,16 @@ +//! Discover global and scoped project macro Python files. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover global and scoped project macro Python files. +#[inline] +pub fn discover_macro_files<'py>( + project_dir: &Bound<'py, PyAny>, + isolate_declaration_kind: bool, +) -> PyResult> { + declaration_records::discover_macro_files(project_dir, isolate_declaration_kind) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs new file mode 100644 index 0000000000..2ac5e14538 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs @@ -0,0 +1,17 @@ +//! Discover the (selected) model files natively, reporting failing files to `on_fault`. + +use crate::discovery::_helpers::records::model_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Discover the (selected) model files natively, reporting failing files to `on_fault`. +#[inline] +pub fn discover_model_files<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + model_records::discover_model_files(py, options) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs new file mode 100644 index 0000000000..9c886cd419 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs @@ -0,0 +1,16 @@ +//! Discover project-wide and scoped reusable model schemas. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover project-wide and scoped reusable model schemas. +#[inline] +pub fn discover_model_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + declaration_records::discover_model_schema_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs new file mode 100644 index 0000000000..1dd47a45dc --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs @@ -0,0 +1,16 @@ +//! Discover scenario files natively, reporting failing files to `on_fault`. + +use crate::discovery::_helpers::records::sql_test_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover scenario files natively, reporting failing files to `on_fault`. +#[inline] +pub fn discover_scenario_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + sql_test_records::discover_scenario_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs new file mode 100644 index 0000000000..3481d9f5bd --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs @@ -0,0 +1,15 @@ +//! Discover model schema.yml and seed declaration files natively. + +use crate::discovery::_helpers::records::yaml_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover model schema.yml and seed declaration files natively. +#[inline] +pub fn discover_schema_files<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + yaml_records::discover_schema_files(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs new file mode 100644 index 0000000000..89c3cc5a60 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs @@ -0,0 +1,13 @@ +//! Discover seed CSV files under seeds/. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover seed CSV files under seeds/. +#[inline] +pub fn discover_seed_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + declaration_records::discover_seed_files(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs new file mode 100644 index 0000000000..bd3e3d36ef --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs @@ -0,0 +1,16 @@ +//! Discover source YAML files natively, reporting failing files to `on_fault`. + +use crate::discovery::_helpers::records::yaml_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover source YAML files natively, reporting failing files to `on_fault`. +#[inline] +pub fn discover_source_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + yaml_records::discover_source_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs new file mode 100644 index 0000000000..2cb6438195 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs @@ -0,0 +1,16 @@ +//! Discover SQL function files under functions/sql/. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover SQL function files under functions/sql/. +#[inline] +pub fn discover_sql_function_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + declaration_records::discover_sql_function_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs new file mode 100644 index 0000000000..6d970cbb61 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs @@ -0,0 +1,16 @@ +//! Discover named SQL lifecycle hooks in their project-wide and scoped roles. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover named SQL lifecycle hooks in their project-wide and scoped roles. +#[inline] +pub fn discover_sql_hook_files<'py>( + project_dir: &Bound<'py, PyAny>, + on_fault: Option>, +) -> PyResult> { + declaration_records::discover_sql_hook_files(project_dir, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs new file mode 100644 index 0000000000..3f73ba9034 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs @@ -0,0 +1,17 @@ +//! Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. + +use crate::discovery::_helpers::records::sql_test_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. +#[inline] +pub fn discover_test_files<'py>( + project_dir: &Bound<'py, PyAny>, + selected_paths: Option>, + on_fault: Option>, +) -> PyResult> { + sql_test_records::discover_test_files(project_dir, selected_paths, on_fault) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs new file mode 100644 index 0000000000..9df1381eaf --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs @@ -0,0 +1,18 @@ +//! The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. + +use crate::discovery::_helpers::records::model_records; +use crate::discovery::models::ModelFileSelection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. +#[inline] +pub fn discovered_model_files<'py>( + project_dir: &Bound<'py, PyAny>, + extraction: (bool, bool), + selection: ModelFileSelection<'py>, +) -> PyResult> { + model_records::discovered_model_files(project_dir, extraction, selection) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs new file mode 100644 index 0000000000..fd6cd14803 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs @@ -0,0 +1,15 @@ +//! Model files that `[defaults]`, `[path_defaults]` or a model schema describe. + +use crate::discovery::_helpers::records::description_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. +#[inline] +pub fn externally_described_models<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + description_inputs::externally_described_models(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs new file mode 100644 index 0000000000..3e85c0fcd2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs @@ -0,0 +1,24 @@ +pub mod discover_adapter_file; +pub mod discover_audit_files; +pub mod discover_constant_files; +pub mod discover_enum_files; +pub mod discover_macro_files; +pub mod discover_model_files; +pub mod discover_model_schema_files; +pub mod discover_scenario_files; +pub mod discover_schema_files; +pub mod discover_seed_files; +pub mod discover_source_files; +pub mod discover_sql_function_files; +pub mod discover_sql_hook_files; +pub mod discover_test_files; +pub mod discovered_model_files; +pub mod externally_described_models; +pub mod parse_sql_test_file; +pub mod projected_text; +pub mod register_declaration_records; +pub mod register_description_inputs; +pub mod register_model_records; +pub mod register_sql_test_records; +pub mod register_yaml_records; +pub mod retained_discovery_session; diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs new file mode 100644 index 0000000000..c10bd475da --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs @@ -0,0 +1,16 @@ +//! Parse in-memory SQL test contents into ordered blocks, as test discovery does. + +use crate::discovery::_helpers::records::sql_test_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Parse in-memory SQL test contents into ordered blocks, as test discovery does. +#[inline] +pub fn parse_sql_test_file<'py>( + contents: String, + file_path: &Bound<'py, PyAny>, +) -> PyResult> { + sql_test_records::parse_sql_test_file(contents, file_path) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs new file mode 100644 index 0000000000..2c1eadeda9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs @@ -0,0 +1,18 @@ +//! `text` as the Python string `cache` shares with every reader, built on first read. + +use crate::discovery::_helpers::records::model_file_record; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::Python; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyString; + +/// `text` as the Python string `cache` shares with every reader, built on first read. +#[inline] +pub fn projected_text<'py>( + py: Python<'py>, + cache: &PyOnceLock>, + text: &str, +) -> Bound<'py, PyString> { + model_file_record::projected_text(py, cache, text) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs new file mode 100644 index 0000000000..19486f7f7c --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs @@ -0,0 +1,12 @@ +//! Register the declaration records functions and classes. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the declaration records functions and classes. +#[inline] +pub fn register_declaration_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_records::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs new file mode 100644 index 0000000000..f633072057 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs @@ -0,0 +1,12 @@ +//! Register the description inputs functions and classes. + +use crate::discovery::_helpers::records::description_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the description inputs functions and classes. +#[inline] +pub fn register_description_inputs(module: &Bound<'_, PyModule>) -> PyResult<()> { + description_inputs::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs new file mode 100644 index 0000000000..1ca1846266 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs @@ -0,0 +1,12 @@ +//! Register the model records functions and classes. + +use crate::discovery::_helpers::records::model_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model records functions and classes. +#[inline] +pub fn register_model_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_records::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs new file mode 100644 index 0000000000..ab0033786d --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs @@ -0,0 +1,12 @@ +//! Register the sql test records functions and classes. + +use crate::discovery::_helpers::records::sql_test_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sql test records functions and classes. +#[inline] +pub fn register_sql_test_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + sql_test_records::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs new file mode 100644 index 0000000000..9f117e1b44 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs @@ -0,0 +1,12 @@ +//! Register the yaml records functions and classes. + +use crate::discovery::_helpers::records::yaml_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the yaml records functions and classes. +#[inline] +pub fn register_yaml_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + yaml_records::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs new file mode 100644 index 0000000000..75b0fd3daf --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs @@ -0,0 +1,14 @@ +//! The session this discovery pass read its declaration files with, if any. + +use crate::discovery::_helpers::records::declaration_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The session this discovery pass read its declaration files with, if any. +#[inline] +pub fn retained_discovery_session<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + declaration_records::retained_discovery_session(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs new file mode 100644 index 0000000000..611b9a2b24 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs @@ -0,0 +1,12 @@ +//! Share one directory snapshot across every discovery call for `project_dir` inside the scope. + +use crate::discovery::_helpers::snapshots::snapshot; +use crate::discovery::models::NativeDirectorySnapshotScope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Share one directory snapshot across every discovery call for `project_dir` inside the scope. +#[inline] +pub fn directory_snapshot_scope(project_dir: Bound<'_, PyAny>) -> NativeDirectorySnapshotScope { + snapshot::directory_snapshot_scope(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs new file mode 100644 index 0000000000..bbd45acb50 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs @@ -0,0 +1,14 @@ +//! Parse the scope-relevant authored roots while retaining independent file faults. + +use crate::discovery::_helpers::snapshots::scope_snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Parse the scope-relevant authored roots while retaining independent file faults. +#[inline] +pub fn discover_scope_snapshot<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + scope_snapshot::discover_scope_snapshot(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs new file mode 100644 index 0000000000..860c66ec23 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs @@ -0,0 +1,12 @@ +//! Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. + +use crate::discovery::_helpers::snapshots::snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; + +/// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. +#[inline] +pub fn escaped(text: &Bound<'_, PyString>) -> PyResult { + snapshot::escaped(text) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs new file mode 100644 index 0000000000..c21854391c --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs @@ -0,0 +1,15 @@ +//! Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. + +use crate::discovery::_helpers::snapshots::snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. +#[inline] +pub fn in_snapshot<'py, T>( + project_dir: &Bound<'py, PyAny>, + body: impl FnOnce() -> PyResult, +) -> PyResult { + snapshot::in_snapshot(project_dir, body) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/mod.rs new file mode 100644 index 0000000000..f4f42321ca --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/mod.rs @@ -0,0 +1,6 @@ +pub mod directory_snapshot_scope; +pub mod discover_scope_snapshot; +pub mod escaped; +pub mod in_snapshot; +pub mod register_scope_snapshot; +pub mod register_snapshot; diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs new file mode 100644 index 0000000000..de088e3cb1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs @@ -0,0 +1,12 @@ +//! Register the scope snapshot functions and classes. + +use crate::discovery::_helpers::snapshots::scope_snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope snapshot functions and classes. +#[inline] +pub fn register_scope_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_snapshot::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs new file mode 100644 index 0000000000..e29fc29827 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs @@ -0,0 +1,12 @@ +//! Register the snapshot functions and classes. + +use crate::discovery::_helpers::snapshots::snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the snapshot functions and classes. +#[inline] +pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { + snapshot::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs new file mode 100644 index 0000000000..bf17b61bf6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs @@ -0,0 +1,12 @@ +//! `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. + +use crate::discovery::_helpers::sources::provider_secrets; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. +#[inline] +pub fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { + provider_secrets::consumer_uses(consumers) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs new file mode 100644 index 0000000000..95d0bc0f90 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs @@ -0,0 +1,9 @@ +//! Whether the running invocation passed `--allow-missing-secrets`. + +use crate::discovery::_helpers::sources::provider_secrets; + +/// Whether the running invocation passed `--allow-missing-secrets`. +#[inline] +pub fn invocation_allows_missing_secrets() -> bool { + provider_secrets::invocation_allows_missing_secrets() +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs new file mode 100644 index 0000000000..a23aea17c6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs @@ -0,0 +1,8 @@ +pub mod consumer_uses; +pub mod invocation_allows_missing_secrets; +pub mod orchestration_declarations; +pub mod provider_uses; +pub mod register_declaration_files; +pub mod register_yaml_entries; +pub mod require_used_secrets; +pub mod sink_uses; diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs new file mode 100644 index 0000000000..915c41e145 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -0,0 +1,15 @@ +//! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). + +use crate::discovery::_helpers::sources::orchestration_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; + +/// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). +#[inline] +pub fn orchestration_declarations( + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + orchestration_files::orchestration_declarations(project_dir) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs new file mode 100644 index 0000000000..872118c52d --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs @@ -0,0 +1,15 @@ +//! `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. + +use crate::discovery::_helpers::sources::provider_secrets; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. +#[inline] +pub fn provider_uses( + usages: &Bound<'_, PyAny>, + consumer: Option<&str>, +) -> PyResult> { + provider_secrets::provider_uses(usages, consumer) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs new file mode 100644 index 0000000000..407e5eb323 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs @@ -0,0 +1,12 @@ +//! Register the declaration files functions and classes. + +use crate::discovery::_helpers::sources::declaration_files; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the declaration files functions and classes. +#[inline] +pub fn register_declaration_files(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_files::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs new file mode 100644 index 0000000000..37f5ed9420 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs @@ -0,0 +1,12 @@ +//! Register the yaml entries functions and classes. + +use crate::discovery::_helpers::sources::yaml_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the yaml entries functions and classes. +#[inline] +pub fn register_yaml_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + yaml_entries::register(module) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs new file mode 100644 index 0000000000..7342258da8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs @@ -0,0 +1,15 @@ +//! Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. + +use crate::discovery::_helpers::sources::provider_secrets; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. +#[inline] +pub fn require_used_secrets( + discovered: &Bound<'_, PyAny>, + uses: &[(String, String)], +) -> PyResult<()> { + provider_secrets::require_used_secrets(discovered, uses) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs new file mode 100644 index 0000000000..5d4fc617cf --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs @@ -0,0 +1,12 @@ +//! The providers the project's lifecycle and command-output sinks use, which run with every command. + +use crate::discovery::_helpers::sources::provider_secrets; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The providers the project's lifecycle and command-output sinks use, which run with every command. +#[inline] +pub fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { + provider_secrets::sink_uses(discovered) +} diff --git a/crates/sqlbuild-python-front/src/discovery/mod.rs b/crates/sqlbuild-python-front/src/discovery/mod.rs new file mode 100644 index 0000000000..e3636f2cd8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/mod.rs @@ -0,0 +1,6 @@ +//! Project discovery: configuration, declaration files, providers, Python nodes and the project facts read from them. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/discovery/models.rs b/crates/sqlbuild-python-front/src/discovery/models.rs new file mode 100644 index 0000000000..df8f05bc3b --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/models.rs @@ -0,0 +1,391 @@ +//! Public records of discovery. + +use crate::discovery::_helpers::project::project_facts::DependencyRow; +use crate::discovery::_helpers::records::model_file_record::FIELD_NAMES; +use crate::discovery::_helpers::records::model_file_record::HeaderValues; +use crate::discovery::_helpers::records::model_file_record::Locations; +use crate::discovery::_helpers::records::model_file_record::ModelFileFields; +use crate::discovery::_helpers::records::model_file_record::ModelFilePaths; +use crate::discovery::_helpers::records::model_file_record::projected_text; +use crate::discovery::_helpers::sources::provider_secrets::INVOCATION_ALLOWS_MISSING_SECRETS; +use pyo3::FromPyObject; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use pyo3::types::PyTuple; +use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::path::PathBuf; +use std::sync::Arc; + +/// The names a model file is known by, as Python's `pathlib` spells them. +#[derive(Debug, Clone)] +pub struct ModelFileTexts { + /// `file_path.stem`, the model's name. + pub name: String, + /// `relative_path.as_posix()`. + pub posix_path: String, + /// `str(relative_path)`. + pub relative_path: String, + /// `str(file_path)`. + pub file_path: String, +} + +/// A discovered SQL model file and its raw contents. +#[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] +pub struct NativeModelFile { + pub(crate) paths: ModelFilePaths, + pub(crate) contents: Arc, + /// `contents` and `query_sql` as the Python strings readers share, built on first read. + pub(crate) projected_texts: (PyOnceLock>, PyOnceLock>), + pub(crate) header_values: HeaderValues, + pub(crate) header_column_locations: Locations, + pub(crate) output_column_locations: Locations, + pub(crate) query_sql: Arc, + pub(crate) enum_declarations: Py, + pub(crate) constant_declarations: Py, + pub(crate) extract_implicit_alias_columns: bool, +} + +impl std::fmt::Debug for NativeModelFile { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeModelFile") + .finish_non_exhaustive() + } +} + +impl NativeModelFile { + pub fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_column_locations + .dict(&self.paths.objects(py)?.1) + } + + /// The header's authored values as the Python dict config builds read. + pub fn header_value_dict<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_values.dict(py) + } + + /// The header columns' authored locations, unless Python constructed this record. + pub fn header_location_rows(&self) -> &[(String, LineColumnSpan)] { + &self.header_column_locations.rows + } + + /// The header's native authored values, unless Python constructed this record. + pub fn native_header_values(&self) -> Option<&Arc<[(String, AuthoredValue)]>> { + self.header_values.native.as_ref() + } + + pub fn declaration_tuples<'py>( + &self, + py: Python<'py>, + ) -> (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>) { + ( + self.enum_declarations.bind(py), + self.constant_declarations.bind(py), + ) + } + + /// The file path as a native path. + pub fn file_path_buf(&self) -> PathBuf { + self.paths.native_file_path.clone() + } + + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. + pub fn path_texts(&self, py: Python<'_>) -> PyResult { + self.paths.texts(py).cloned() + } + + /// `relative_path.as_posix()`. + pub fn posix_path(&self, py: Python<'_>) -> PyResult { + Ok(self.paths.texts(py)?.posix_path.clone()) + } + + /// `str(relative_path)`. + pub fn relative_text(&self, py: Python<'_>) -> PyResult { + Ok(self.paths.texts(py)?.relative_path.clone()) + } + + /// `str(relative_path.parent)`, the folder project graphs group the model by. + pub fn folder(&self, py: Python<'_>) -> PyResult { + Ok(self + .paths + .texts(py)? + .relative_path + .rsplit_once(std::path::MAIN_SEPARATOR) + .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())) + } + + /// `(file_path, relative_path)` as Python path objects. + pub fn path_objects<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + self.paths.objects(py) + } + + /// The authored query below the header. + pub fn query_text(&self) -> &Arc { + &self.query_sql + } + + /// The whole authored file. + pub fn contents_text(&self) -> &Arc { + &self.contents + } + + pub(crate) fn fields<'py>(&self, py: Python<'py>) -> PyResult>> { + let (file_path, relative_path) = self.paths.objects(py)?; + Ok(vec![ + file_path, + relative_path.clone(), + self.contents(py).into_any(), + self.header_values.dict(py)?.into_any(), + self.header_column_locations + .dict(&relative_path)? + .into_any(), + self.output_column_locations + .dict(&relative_path)? + .into_any(), + self.query_sql(py).into_any(), + self.enum_declarations.bind(py).clone().into_any(), + self.constant_declarations.bind(py).clone().into_any(), + pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) + .to_owned() + .into_any(), + ]) + } +} + +#[pymethods] +impl NativeModelFile { + #[new] + #[pyo3(signature = (**fields))] + pub(crate) fn new(py: Python<'_>, fields: Option>) -> PyResult { + let Some(fields) = fields else { + return Err(PyTypeError::new_err( + "DiscoveredSqlModelFile requires its fields", + )); + }; + for key in fields.keys() { + let key: String = key.extract()?; + if !FIELD_NAMES.contains(&key.as_str()) { + return Err(PyTypeError::new_err(format!( + "DiscoveredSqlModelFile got an unexpected keyword argument '{key}'" + ))); + } + } + let fields: ModelFileFields = fields.extract()?; + Ok(Self { + paths: ModelFilePaths::given(py, fields.file_path, fields.relative_path)?, + contents: fields.contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: HeaderValues::given(py, fields.header_values), + header_column_locations: Locations::from_dict(py, fields.header_column_locations), + output_column_locations: Locations::from_dict(py, fields.output_column_locations), + query_sql: fields.query_sql.bind(py).to_str()?.into(), + enum_declarations: fields + .enum_declarations + .unwrap_or_else(|| PyTuple::empty(py).unbind()), + constant_declarations: fields + .constant_declarations + .unwrap_or_else(|| PyTuple::empty(py).unbind()), + extract_implicit_alias_columns: fields.extract_implicit_alias_columns, + }) + } + + /// This file with edited contents and query, as `dataclasses.replace` would make it. + pub fn with_text( + &self, + py: Python<'_>, + contents: Py, + query_sql: Py, + ) -> PyResult { + Ok(Self { + paths: self.paths.shared(py), + contents: contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: self.header_values.shared(py), + header_column_locations: self.header_column_locations.shared(py), + output_column_locations: self.output_column_locations.shared(py), + query_sql: query_sql.bind(py).to_str()?.into(), + enum_declarations: self.enum_declarations.clone_ref(py), + constant_declarations: self.constant_declarations.clone_ref(py), + extract_implicit_alias_columns: self.extract_implicit_alias_columns, + }) + } + + #[getter] + pub fn file_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.0.unbind()) + } + + #[getter] + pub fn relative_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.1.unbind()) + } + + #[getter] + pub fn contents<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + projected_text(py, &self.projected_texts.0, &self.contents) + } + + #[getter] + pub fn header_values(&self, py: Python<'_>) -> PyResult> { + Ok(self.header_values.dict(py)?.unbind()) + } + + #[getter] + pub fn header_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_locations(py) + } + + #[getter] + pub fn output_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.output_column_locations + .dict(&self.paths.objects(py)?.1) + } + + #[getter] + pub fn query_sql<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + projected_text(py, &self.projected_texts.1, &self.query_sql) + } + + #[getter] + pub fn enum_declarations(&self, py: Python<'_>) -> Py { + self.enum_declarations.clone_ref(py) + } + + #[getter] + pub fn constant_declarations(&self, py: Python<'_>) -> Py { + self.constant_declarations.clone_ref(py) + } + + #[getter] + pub fn extract_implicit_alias_columns(&self) -> bool { + self.extract_implicit_alias_columns + } + + pub(crate) fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult> { + let Ok(other) = other.downcast::() else { + return Ok(py.NotImplemented()); + }; + let other: &NativeModelFile = other.get(); + for (left, right) in self.fields(py)?.iter().zip(other.fields(py)?) { + if !left.eq(right)? { + return false.into_py_any(py); + } + } + true.into_py_any(py) + } + + pub(crate) fn __repr__(&self, py: Python<'_>) -> PyResult { + let fields: Vec = FIELD_NAMES + .iter() + .zip(self.fields(py)?) + .map(|(name, value)| Ok(format!("{name}={}", value.repr()?))) + .collect::>()?; + Ok(format!("DiscoveredSqlModelFile({})", fields.join(", "))) + } + + /// The keyword fields `copy` and `pickle` rebuild this record from. + pub(crate) fn __getnewargs_ex__<'py>( + slf: &Bound<'py, Self>, + ) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyDict>)> { + let py: Python<'py> = slf.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in FIELD_NAMES.iter().zip(slf.get().fields(py)?) { + keywords.set_item(name, value)?; + } + Ok((PyTuple::empty(py), keywords)) + } +} + +/// Which discovered model files to build, and where failing files are reported. +#[derive(Debug)] +pub struct ModelFileSelection<'py> { + pub selected_model_names: Option>, + pub on_fault: Option>, +} + +/// A decorated Python node as the facade describes it. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct NodeRow { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) path: String, + pub(crate) decorator_config: bool, + pub(crate) depends_on: Vec, +} + +/// `--allow-missing-secrets` for one invocation, restored when the invocation ends. +#[derive(Debug)] +pub struct MissingSecretsScope { + pub(crate) previous: bool, +} + +impl MissingSecretsScope { + pub fn enter(allowed: bool) -> Self { + Self { + previous: INVOCATION_ALLOWS_MISSING_SECRETS.replace(allowed), + } + } +} + +impl Drop for MissingSecretsScope { + fn drop(&mut self) { + INVOCATION_ALLOWS_MISSING_SECRETS.set(self.previous); + } +} + +/// What discovery does when a provider's required secrets are unset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SecretPolicy { + /// Fail with D012: static commands by default. + Require, + /// Continue and list the providers on stderr: static commands that opted out. + Allow, + /// Continue silently; the command checks the providers it uses before running anything. + Defer, +} + +impl SecretPolicy { + /// The keyword value that selects the policy. + pub const fn name(self) -> &'static str { + match self { + Self::Require => "require", + Self::Allow => "allow", + Self::Defer => "defer", + } + } + + pub fn parse(value: &str) -> PyResult { + [Self::Require, Self::Allow, Self::Defer] + .into_iter() + .find(|policy| policy.name() == value) + .ok_or_else(|| PyValueError::new_err(format!("unknown secret policy '{value}'"))) + } +} + +/// The context manager that makes one snapshot active for a project until it exits. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeDirectorySnapshotScope { + pub project_dir: pyo3::Py, + /// The context-variable token of the snapshot this scope activated, if it activated one. + pub token: Option>, +} diff --git a/crates/sqlbuild-python-front/src/lib.rs b/crates/sqlbuild-python-front/src/lib.rs new file mode 100644 index 0000000000..b0070dbde8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/lib.rs @@ -0,0 +1,21 @@ +//! The front layer of the `sqlbuild._native` bindings: discovery, model configuration, scopes, rendering and SQL text. + +#![forbid(unsafe_code)] + +pub mod attachments; +pub mod authored_values; +pub mod command_runtime; +pub mod compile_attachment; +pub mod compile_driver; +pub mod contracts; +pub mod cursor; +pub mod discovery; +pub mod macro_bridge; +pub mod model_config; +pub mod native_host; +pub mod refactor; +pub mod render; +pub mod scopes; +pub mod sql_test_glue; +pub mod sqltext; +pub mod state_rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs rename to crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/mod.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/mod.rs new file mode 100644 index 0000000000..45adffdb4c --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod first_compile; +pub(crate) mod module_digests; +pub(crate) mod store_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs rename to crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs index dd34d77fae..45cc26a97a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs @@ -10,7 +10,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::module_digests::main::module_digests_metadata::module_digests_metadata as metadata_text; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::macro_bridge::store_keys::text; +use crate::macro_bridge::_helpers::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::main::digest_files::digest_files; use sqlbuild_python_base::frontier::main::emit_error::emit_error; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs rename to crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs new file mode 100644 index 0000000000..be5b01f271 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs @@ -0,0 +1,16 @@ +//! Digest the whole context a macro may read, or None when a value has no stable text. + +use crate::macro_bridge::_helpers::store_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Digest the whole context a macro may read, or None when a value has no stable text. +#[inline] +pub fn context_store_token<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + store_keys::context_store_token(py, options) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs new file mode 100644 index 0000000000..f2e88dec30 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs @@ -0,0 +1,9 @@ +//! Keep the delta compile's claim: it finished, so no full compile of this invocation follows. + +use crate::macro_bridge::_helpers::first_compile; + +/// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. +#[inline] +pub fn keep_delta_claim() { + first_compile::keep_delta_claim() +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs new file mode 100644 index 0000000000..1219b76282 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs @@ -0,0 +1,16 @@ +//! Name a macro by its name, file, identity and exact source, or None when it has no key. + +use crate::macro_bridge::_helpers::store_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Name a macro by its name, file, identity and exact source, or None when it has no key. +#[inline] +pub fn macro_store_token<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + store_keys::macro_store_token(py, options) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/mod.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/mod.rs new file mode 100644 index 0000000000..b4f017fd06 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/mod.rs @@ -0,0 +1,7 @@ +pub mod context_store_token; +pub mod keep_delta_claim; +pub mod macro_store_token; +pub mod register_first_compile; +pub mod register_module_digests; +pub mod register_store_keys; +pub mod text; diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs new file mode 100644 index 0000000000..d255b60486 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs @@ -0,0 +1,12 @@ +//! Register the first compile functions and classes. + +use crate::macro_bridge::_helpers::first_compile; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the first compile functions and classes. +#[inline] +pub fn register_first_compile(module: &Bound<'_, PyModule>) -> PyResult<()> { + first_compile::register(module) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs new file mode 100644 index 0000000000..5c4403757a --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs @@ -0,0 +1,12 @@ +//! Register the module digests functions and classes. + +use crate::macro_bridge::_helpers::module_digests; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the module digests functions and classes. +#[inline] +pub fn register_module_digests(module: &Bound<'_, PyModule>) -> PyResult<()> { + module_digests::register(module) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs new file mode 100644 index 0000000000..8ec80a3394 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs @@ -0,0 +1,12 @@ +//! Register the store keys functions and classes. + +use crate::macro_bridge::_helpers::store_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the store keys functions and classes. +#[inline] +pub fn register_store_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { + store_keys::register(module) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs new file mode 100644 index 0000000000..4a1a649ee5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs @@ -0,0 +1,13 @@ +//! A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. + +use crate::macro_bridge::_helpers::store_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; +use sqlbuild_core::json::models::JsonValue; + +/// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. +#[inline] +pub fn text(value: &Bound<'_, PyString>) -> PyResult { + store_keys::text(value) +} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/mod.rs b/crates/sqlbuild-python-front/src/macro_bridge/mod.rs new file mode 100644 index 0000000000..53d5f87ee2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/macro_bridge/mod.rs @@ -0,0 +1,4 @@ +//! Keys and digests the macro bridge uses to reuse first-compile macro results. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs index a869bc2bcd..94efd6949a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs @@ -1,38 +1,20 @@ //! The Python classes a model's `CompileModelConfig` is built from, and building one from a native build. +use crate::model_config::models::ConfigClasses; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject}; use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; -use crate::bindings::_helpers::model_config::model_config_builder::{ - BuiltConfig, NativeModelConfigBuilder, -}; -use crate::bindings::_helpers::model_config::typed_config::{BuiltConfigValues, retain_values}; +use crate::model_config::_helpers::config::typed_config::retain_values; +use crate::model_config::models::BuiltConfig; +use crate::model_config::types::BuiltConfigValues; const MATERIALIZED_KEY: &str = "materialized"; const VALUES_FIELD: &str = "values"; const PERMANENT_TABLE_TYPE: &str = "permanent"; const TRANSIENT_TABLE_TYPE: &str = "transient"; -/// The builder, storage defaults and Python classes a model's `CompileModelConfig` is made of. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ConfigClasses<'py> { - pub(crate) builder: Bound<'py, NativeModelConfigBuilder>, - /// Inherited `(retention, table type)` by table-backed materialization, or `None`. - pub(crate) inherited_storage: Bound<'py, PyDict>, - pub(crate) table_backed: Vec, - pub(crate) compile_model_config: Bound<'py, PyAny>, - pub(crate) resolved_retention: Bound<'py, PyAny>, - pub(crate) retention_source: Bound<'py, PyAny>, - pub(crate) resolved_table_type: Bound<'py, PyAny>, - pub(crate) table_type: Bound<'py, PyAny>, - pub(crate) table_type_source: Bound<'py, PyAny>, - /// `record_template_reads`, the compile-reuse sink of template reads. - pub(crate) record_template_reads: Bound<'py, PyAny>, -} - /// The model's `CompileModelConfig`, once its template reads are recorded. pub(crate) fn built_config<'py>( classes: &ConfigClasses<'py>, @@ -112,7 +94,7 @@ fn storage_key<'py>( Ok(py.None().into_bound(py)) } -fn resolved_retention<'py>( +pub(crate) fn resolved_retention<'py>( classes: &ConfigClasses<'py>, desired_days: Bound<'py, PyAny>, unmanaged: bool, @@ -124,7 +106,7 @@ fn resolved_retention<'py>( classes.resolved_retention.call((), Some(&fields)) } -fn resolved_table_type<'py>( +pub(crate) fn resolved_table_type<'py>( classes: &ConfigClasses<'py>, value: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs index 043840ee5a..1ff777cbd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs @@ -5,8 +5,8 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{Py, PyErr, pymethods}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; -use crate::bindings::_helpers::model_config::config_values::config_value_type_error; -use crate::bindings::_helpers::model_config::errors::NativeConfigError; +use crate::model_config::_helpers::config::config_values::config_value_type_error; +use crate::model_config::errors::NativeConfigError; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs index c8e79d0330..33b0181197 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_nodes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs @@ -1,10 +1,12 @@ //! Python config values converted once into `ConfigNode`s, and projected back into Python objects. +use crate::model_config::models::HostHandle; +use crate::model_config::types::PyConfigNode; use std::sync::Arc; use pyo3::IntoPyObject; use pyo3::exceptions::{PyOverflowError, PyUnicodeEncodeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, @@ -12,10 +14,9 @@ use pyo3::types::{ use sqlbuild_model_config::config_values::models::{ ConfigNode, HostChildren, HostFacts, HostValue, }; -use sqlbuild_model_config::config_values::types::HostObject; use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::model_config::models::PyNode; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::types::RecordId; @@ -26,34 +27,6 @@ const HOOK_ENTRY_CLASSES: [RecordId; 3] = [ RecordId::PythonHookEntry, ]; -/// The Python object an opaque config value stands for. -#[derive(Clone, Debug)] -pub(crate) struct HostHandle(pub(crate) Arc>); - -impl HostObject for HostHandle { - fn is_date_like(&self) -> bool { - Python::attach(|py| PyNode(self.0.bind(py).clone()).is_date_like()) - } - - /// Python's text of the projected value; a `str()` or `repr()` that raises reads as empty, as `PyNode`. - fn display(node: &ConfigNode, repr: bool) -> String { - Python::attach(|py| { - config_object(py, node).map_or_else( - |_| String::new(), - |value| { - let value: PyNode<'_> = PyNode(value); - if repr { - value.python_repr() - } else { - value.python_str() - } - }, - ) - }) - } -} -/// A config value converted from Python. -pub(crate) type PyConfigNode = ConfigNode; /// A converted config mapping's entries. pub(crate) type PyConfigEntries = Vec<(PyConfigNode, PyConfigNode)>; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs new file mode 100644 index 0000000000..759e89e53f --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs @@ -0,0 +1,211 @@ +//! `${...}` templates in authored Python config values and effective vars, expanded natively. + +use crate::model_config::models::PythonHost; +use crate::model_config::models::Stop; +use crate::model_config::models::TemplateSources; +use std::cell::RefCell; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, +}; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::templates::errors::TemplateError; +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::main::template_error_message::template_error_message; +use sqlbuild_model_config::templates::models::{StringExpansion, TemplateOptions}; +use sqlbuild_model_config::templates::types::TemplateHost; + +use crate::model_config::_helpers::config::config_errors::native_config_error; +use crate::model_config::errors::NativeConfigError; + +const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const ENVIRONMENT_READ: &str = "env"; +pub(crate) const CONTEXT_READ: &str = "ctx"; +pub(crate) const EFFECTIVE_VARS_LABEL: &str = "effective vars"; +const VALUE_PATH: &str = "its value"; +const BUILTINS_MODULE: &str = "builtins"; +const ASCII_FUNCTION: &str = "ascii"; +const ENCODE: &str = "encode"; +const UTF32_LE: &str = "utf-32-le"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// Effective vars resolved on demand, as Python's `_EffectiveVarsResolver` resolves them. +#[derive(Debug)] +pub(crate) struct EffectiveVars<'py> { + pub(crate) resolved: Bound<'py, PyDict>, + pub(crate) resolving: RefCell>, +} + +/// The `CompileInputError` Python raises for a template error in the context `label`. +pub(crate) fn template_error(error: &TemplateError, label: &str) -> ConfigError { + ConfigError::compile(template_error_message(error, label)) +} + +/// The expanded value, or the native error expansion stopped at, and the reads in order. +pub(crate) type TemplateOutcome<'py> = ( + Result, Py>, + Vec<(&'static str, String)>, +); + +/// `value` with every template expanded under `options`, or the error, with its reads. +pub(crate) fn expand_config_templates<'py>( + py: Python<'py>, + value: &Bound<'py, PyAny>, + sources: TemplateSources<'py>, + (options, label): (TemplateOptions, &str), +) -> PyResult> { + let host = PythonHost::new(py, sources, label); + let result = expanded(&host, value, options); + outcome(py, host, result, label) +} + +/// The first `(var name, value path, UTF-8 byte offset)` of a lone surrogate in raw vars. +pub(crate) fn lone_surrogate_site<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult, String, usize)>> { + for (key, value) in raw_values.iter() { + if let Some((path, offset)) = surrogate_path(&value, VALUE_PATH.to_owned())? { + return Ok(Some((key.str()?, path, offset))); + } + } + Ok(None) +} + +/// Python's former `_reject_lone_surrogates` walk: dict keys then items, lists and tuples. +fn surrogate_path(value: &Bound<'_, PyAny>, path: String) -> PyResult> { + if let Ok(mapping) = value.downcast::() { + for (key, item) in mapping.iter() { + if let Some(site) = surrogate_path(&key, format!("{path} key"))? { + return Ok(Some(site)); + } + let key_ascii = PyModule::import(value.py(), BUILTINS_MODULE)? + .getattr(ASCII_FUNCTION)? + .call1((&key,))? + .str()? + .to_string_lossy() + .into_owned(); + if let Some(site) = surrogate_path(&item, format!("{path}[{key_ascii}]"))? { + return Ok(Some(site)); + } + } + return Ok(None); + } + if value.is_instance_of::() || value.is_instance_of::() { + for (index, item) in value.try_iter()?.enumerate() { + if let Some(site) = surrogate_path(&item?, format!("{path}[{index}]"))? { + return Ok(Some(site)); + } + } + return Ok(None); + } + let Ok(text) = value.downcast::() else { + return Ok(None); + }; + Ok(first_surrogate_offset(text)?.map(|offset| (path, offset))) +} + +/// The UTF-8 byte offset of the first lone surrogate in `text`. +fn first_surrogate_offset(text: &Bound<'_, PyString>) -> PyResult> { + let encoded = text.call_method1(ENCODE, (UTF32_LE, SURROGATE_PASS))?; + let mut offset = 0; + for unit in encoded.extract::>()?.as_chunks::<4>().0 { + let code_point = u32::from_le_bytes([unit[0], unit[1], unit[2], unit[3]]); + match char::from_u32(code_point) { + Some(character) => offset += character.len_utf8(), + None => return Ok(Some(offset)), + } + } + Ok(None) +} + +/// The resolved effective vars, or the native error resolution stopped at, with the reads. +pub(crate) fn resolve_effective_vars<'py>( + py: Python<'py>, + raw_values: &Bound<'py, PyDict>, + environment: Bound<'py, PyAny>, +) -> PyResult> { + let mut host = PythonHost::new( + py, + TemplateSources(raw_values.clone(), environment, PyDict::new(py)), + EFFECTIVE_VARS_LABEL, + ); + let resolved = PyDict::new(py); + host.effective = Some(EffectiveVars { + resolved: resolved.clone(), + resolving: RefCell::new(Vec::new()), + }); + let mut result: Result, Stop> = Ok(resolved.clone().into_any()); + for key in raw_values.keys() { + let name = key.str()?.to_str()?.to_owned(); + if let Err(failure) = host.variable(&name) { + result = Err(host.stop(failure)); + break; + } + } + outcome(py, host, result, EFFECTIVE_VARS_LABEL) +} + +fn outcome<'py>( + py: Python<'py>, + host: PythonHost<'py>, + result: Result, Stop>, + label: &str, +) -> PyResult> { + let value = match result { + Ok(value) => Ok(value), + Err(Stop::Failure(error)) => Err(native_config_error(py, template_error(&error, label))?), + Err(Stop::Python(error)) => return Err(error), + }; + Ok((value, host.into_reads())) +} + +/// Expand every template in `value` and the containers inside it. +pub(crate) fn expanded<'py>( + host: &PythonHost<'py>, + value: &Bound<'py, PyAny>, + options: TemplateOptions, +) -> Result, Stop> { + let py = host.py; + if let Ok(text) = value.downcast::() { + let text = text.to_str().map_err(Stop::Python)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + return match expand_template_string(host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), + Ok(StringExpansion::Value(result)) => Ok(result), + Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), + Err(failure) => Err(host.stop(failure)), + }; + } + if let Ok(mapping) = value.downcast::() { + let result = PyDict::new(py); + for (key, item) in mapping.iter() { + result + .set_item(key, expanded(host, &item, options)?) + .map_err(Stop::Python)?; + } + return Ok(result.into_any()); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyList::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyTuple::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + Ok(value.clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs index 158d716c7e..8cf5b4efed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs @@ -10,8 +10,9 @@ use sqlbuild_model_config::config_values::main::checked_text::checked_text; use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::CursorBound; -use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_node}; -use crate::bindings::_helpers::model_config::typed_config::type_error; +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/getters.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/getters.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/getters.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/getters.rs index 4ba23b2a87..041cade44f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/getters.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::model_config::errors::NativeConfigError; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::model_config::errors::NativeConfigError; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_native_config_error, -}]; - -fn read_native_config_error<'py>( +pub(crate) fn read_native_config_error<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/mod.rs new file mode 100644 index 0000000000..e9893f192c --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod config_classes; +pub(crate) mod config_errors; +pub(crate) mod config_nodes; +pub(crate) mod config_templates; +pub(crate) mod config_values; +pub(crate) mod getters; +pub(crate) mod template_data; +pub(crate) mod typed_config; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs index 48273a287f..5a8c4bb375 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs @@ -10,12 +10,13 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::model_config::config_errors::config_exception; -use crate::bindings::_helpers::model_config::config_templates::{ - TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, - resolve_effective_vars, -}; -use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; +use crate::model_config::_helpers::config::config_errors::config_exception; +use crate::model_config::_helpers::config::config_templates::TemplateOutcome; +use crate::model_config::_helpers::config::config_templates::expand_config_templates; +use crate::model_config::_helpers::config::config_templates::lone_surrogate_site; +use crate::model_config::_helpers::config::config_templates::resolve_effective_vars; +use crate::model_config::_helpers::model_nodes::unicode_environment::unicode_environment; +use crate::model_config::models::TemplateSources; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; @@ -211,3 +212,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; Ok(()) } + +/// `record_template_reads` as a Python function. +pub(crate) fn record_template_reads_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(record_template_reads, py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs index 190cb7abbf..5d628ab405 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs @@ -1,11 +1,13 @@ //! Built model config trees retained natively, with Python-authored values converted on first read. +use crate::model_config::models::TypedConfigValues; +use crate::model_config::types::BuiltConfigValues; +use crate::model_config::types::ConfigValues; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::pyclass; use pyo3::types::{PyString, PyTuple}; use sqlbuild_model_config::config_values::errors::ConfigTypeError; use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; @@ -15,10 +17,13 @@ use sqlbuild_model_config::config_values::main::checked_text::checked_text; use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::{ConfigNode, CursorBound, HostChildren}; -use crate::bindings::_helpers::model_config::config_nodes::{ - HostHandle, PyConfigEntries, PyConfigNode, config_node, config_object, is_exact_container, -}; -use crate::bindings::_helpers::model_config::config_values::config_value_type_error; +use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::config_values::config_value_type_error; +use crate::model_config::models::HostHandle; +use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::compile_driver::constants::TYPED_CONFIG_READS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::cached_value::cached_value; @@ -34,16 +39,6 @@ const ISOFORMAT_METHOD: &str = "isoformat"; const TYPED_VALUES_SLOT: &str = "__typed_values__"; const POISONED: &str = "typed config values cache is poisoned"; -/// A built config's string-keyed tree; values shared across reused configs stay native. -pub(crate) type BuiltConfigValues = Arc>>; - -/// The built tree and any Python-authored values opened by the record's readers. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TypedConfigValues { - pub(crate) built: Option, - read: Mutex)>>, -} - /// Keep the builder's tree without interpreting custom Python mapping keys natively. pub(crate) fn built_values(entries: PyConfigEntries) -> Option { let mut values = HashMap::with_capacity(entries.len()); @@ -76,9 +71,6 @@ pub(crate) fn retain_values( Ok(()) } -/// A config record's values and the typed nodes read of them, each key converted on first read. -pub(crate) type ConfigValues<'py> = (Bound<'py, PyAny>, Bound<'py, TypedConfigValues>); - /// `values.get(key)` as a typed node, `None` when absent; each key is converted once. fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { let (mapping, typed) = values; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs new file mode 100644 index 0000000000..1811576f97 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs @@ -0,0 +1,107 @@ +//! Retained audit contracts; Python-authored factory results remain opaque values. + +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_discovery::model_files::main::config_value::config_value; +use sqlbuild_discovery::model_files::models::NativeAudit; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::types::HeaderValue; +use crate::model_config::types::PyConfigNode; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use sqlbuild_python_base::result_output::main::json_value::json_value; + +/// Native discovery facts keep their authored values rather than building audit records. +pub(crate) fn authored_audit( + py: Python<'_>, + spec: &NativeAudit, +) -> PyResult> { + let value = |value: &AuthoredValue| match config_value(value) { + Some(value) => Ok(value), + None => config_node(authored_value_to_python(py, value)?.bind(py)), + }; + Ok(ParsedAudit { + definition_name: ConfigNode::Str(spec.definition_name.clone()), + arguments: spec + .arguments + .iter() + .map(|(key, item)| Ok((ConfigNode::Str(key.clone()), value(item)?))) + .collect::>()?, + name: spec.name.clone().map(ConfigNode::Str), + description: spec.description.clone().map(ConfigNode::Str), + severity: spec.severity.clone().map(ConfigNode::Str), + run_scope: spec.run_scope.clone().map(ConfigNode::Str), + always_run: spec.always_run.map(ConfigNode::Bool), + thresholds: spec.thresholds, + minimum_samples: spec.minimum_samples.as_ref().map(value).transpose()?, + evidence_limit: spec.evidence_limit.as_ref().map(value).transpose()?, + meta: spec.meta.as_ref().map(value).transpose()?, + }) +} + +/// Preserve parsed leaves without rebuilding their public argument dictionaries. +pub(crate) fn retain_audit( + py: Python<'_>, + spec: &ParsedAudit, +) -> PyResult> { + let optional = |node: &Option| node.as_ref().map(|node| node.retained(py)).transpose(); + Ok(ParsedAudit { + definition_name: spec.definition_name.retained(py)?, + arguments: spec + .arguments + .iter() + .map(|(key, value)| Ok((key.retained(py)?, value.retained(py)?))) + .collect::>()?, + name: optional(&spec.name)?, + description: optional(&spec.description)?, + severity: optional(&spec.severity)?, + run_scope: optional(&spec.run_scope)?, + always_run: optional(&spec.always_run)?, + thresholds: spec.thresholds, + minimum_samples: optional(&spec.minimum_samples)?, + evidence_limit: optional(&spec.evidence_limit)?, + meta: optional(&spec.meta)?, + }) +} + +/// Read metadata without projecting retained containers; opaque user values keep Python semantics. +pub(crate) fn config_json_value( + py: Python<'_>, + value: &PyConfigNode, +) -> PyResult> { + Ok(Some(match value { + ConfigNode::Null => JsonValue::Null, + ConfigNode::Bool(value) => JsonValue::Bool(*value), + ConfigNode::Int(value) => JsonValue::Integer(JsonInteger::from(*value)), + ConfigNode::Float(value) => JsonValue::Float(*value), + ConfigNode::Str(value) => JsonValue::String(value.clone()), + ConfigNode::List(items) | ConfigNode::Tuple(items) => { + let mut result: Vec = Vec::with_capacity(items.len()); + for item in items { + let Some(value) = config_json_value(py, item)? else { + return Ok(None); + }; + result.push(value); + } + JsonValue::Array(result) + } + ConfigNode::Map(entries) => { + let mut result: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); + for (key, item) in entries { + let ConfigNode::Str(key) = key else { + return json_value(&config_object(py, value)?); + }; + let Some(value) = config_json_value(py, item)? else { + return Ok(None); + }; + result.push((key.clone(), value)); + } + JsonValue::Object(result) + } + ConfigNode::Host(_) => return json_value(&config_object(py, value)?), + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs index 87e9ba1940..86f53dccfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs @@ -9,11 +9,12 @@ use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_schema_columns as parse_columns; use sqlbuild_model_config::header_metadata::models::ColumnRequest; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::config_errors::config_exception; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::contract_classes::contract_classes; -use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, column_tuple}; +use crate::model_config::_helpers::config::config_errors::config_exception; +use crate::model_config::_helpers::config::config_errors::native_config_error; +use crate::model_config::_helpers::headers::header_metadata::column_tuple; +use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::PyNode; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs new file mode 100644 index 0000000000..144dc5045a --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs @@ -0,0 +1,24 @@ +//! Native dynamic-column contracts with a shared, lazy public projection. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +use sqlbuild_python_base::records::main::fields::field::field; + +/// A Python-authored family contract crosses into native ownership once. +pub(crate) fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { + families + .try_iter()? + .map(|family| { + let family = family?; + Ok(DynamicFamily { + name: field(&family, "name")?.extract()?, + pivot_column: field(&family, "pivot_column")?.extract()?, + value_column: field(&family, "value_column")?.extract()?, + aggregate: field(&family, "aggregate")?.extract()?, + data_type: field(&family, "type")?.extract()?, + name_pattern: field(&family, "name_pattern")?.extract()?, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs index 01f4e978fc..068e2e8701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs @@ -1,40 +1,36 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. +use crate::model_config::models::ContractClasses; +use crate::model_config::models::HeaderColumn; +use crate::model_config::models::NativeColumnSpec; +use crate::model_config::types::HeaderValue; use std::sync::OnceLock; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt}; +use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; use sqlbuild_discovery::models::LineColumnSpan; -use sqlbuild_model_config::header_metadata::constants::AUDITS_KEY; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; -use sqlbuild_model_config::types::AuthoredNode; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::config_nodes::{ - PyConfigNode, config_node, config_object, -}; -use crate::bindings::_helpers::model_config::contract_classes::contract_classes; -use crate::bindings::_helpers::model_config::header_audits::{ - AuditInstance, HeaderAudits, authored_audit, -}; -use crate::bindings::_helpers::model_config::typed_config::node_text; -use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::_helpers::config::typed_config::node_text; +use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; +use crate::model_config::models::HeaderAudits; +use crate::model_config::models::PyNode; +use crate::model_config::types::PyConfigNode; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::all_specs::all_specs; -use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; /// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. -const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ +pub(crate) const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ "name", "type", "nullable", @@ -71,19 +67,6 @@ const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), ]; -/// A parsed leaf projected only when a public contract needs its Python value. -pub(crate) trait HeaderValue: AuthoredNode { - fn object<'py>(&self, py: Python<'py>) -> PyResult>; - - fn retained(&self, py: Python<'_>) -> PyResult { - config_node(&self.object(py)?) - } - - fn exact_text(&self, py: Python<'_>) -> PyResult { - self.object(py)?.extract() - } -} - impl HeaderValue for PyNode<'_> { fn object<'py>(&self, py: Python<'py>) -> PyResult> { Ok(self.0.as_unbound().bind(py).clone()) @@ -118,16 +101,8 @@ impl HeaderValue for PyConfigNode { } } -/// Native column fields; a Python record is constructed only for public consumers. -pub(crate) struct HeaderColumn { - fields: ColumnFields, - class: Py, - audits: Option, - projection: PyOnceLock>, -} - /// What a column's Python record is built from. -enum ColumnFields { +pub(crate) enum ColumnFields { /// The record's field values, in declaration order. Python([Py; 8]), /// A header column read natively, with what its location and audits are built with. @@ -136,204 +111,6 @@ enum ColumnFields { Projected, } -/// A natively parsed header column and what its Python record is built with. -pub(crate) struct NativeColumnSpec { - pub(crate) column: NativeColumn, - /// The model file whose relative path the column's location names, and the location. - pub(crate) location: Option<(Py, LineColumnSpan)>, -} - -impl HeaderColumn { - /// Retype a builtin column without discarding its retained audit specs. - pub(crate) fn enum_type( - mut self, - py: Python<'_>, - scalar_type: Bound<'_, PyAny>, - generated: Option>, - ) -> PyResult { - if self.audits.is_none() - && let ColumnFields::Native(spec) = &self.fields - { - let audits = spec - .column - .audits - .iter() - .map(|audit| authored_audit(py, audit)) - .collect::>>()?; - let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; - self.audits = Some(HeaderAudits::new( - py, - &classes, - &audits, - &native_location(py, spec)?, - )?); - } - if let Some(audits) = &mut self.audits { - if let Some(generated) = generated { - audits.push_native(generated); - } - match &mut self.fields { - ColumnFields::Native(spec) => spec.column.column_type = scalar_type.extract()?, - ColumnFields::Python(values) => values[1] = scalar_type.unbind(), - ColumnFields::Projected => { - return Err(PyRuntimeError::new_err( - "enum column has no retained fields", - )); - } - } - self.projection = PyOnceLock::new(); - return Ok(self); - } - Err(PyRuntimeError::new_err( - "enum column has no retained audits", - )) - } - - pub(crate) fn retains_audits(&self) -> bool { - self.audits.is_some() || matches!(self.fields, ColumnFields::Native(_)) - } - - pub(crate) fn audit_instances<'py>( - &self, - py: Python<'py>, - ) -> PyResult>> { - if let Some(audits) = &self.audits { - return Ok(audits.instances(py)); - } - if let ColumnFields::Native(spec) = &self.fields { - let audits = spec - .column - .audits - .iter() - .map(|audit| authored_audit(py, audit)) - .collect::>>()?; - let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; - return Ok( - HeaderAudits::new(py, &classes, &audits, &native_location(py, spec)?)? - .instances(py), - ); - } - self.field(py, "audits")? - .try_iter()? - .map(|value| value.map(AuditInstance::Python)) - .collect() - } - - pub(crate) fn replaced<'py>( - class: &Bound<'py, PyAny>, - values: [Bound<'py, PyAny>; 8], - keyword_order: [usize; 8], - ) -> PyResult { - if class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { - return Self::new(class, values); - } - let fields = PyDict::new(class.py()); - for index in keyword_order { - fields.set_item(SCHEMA_COLUMN_FIELDS[index], &values[index])?; - } - Self::from_python(class.call((), Some(&fields))?) - } - - pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let index = || { - SCHEMA_COLUMN_FIELDS - .iter() - .position(|field| *field == name) - .ok_or_else(|| PyRuntimeError::new_err("unknown column field")) - }; - if let Some(projected) = self.projection.get(py) { - return field(projected.bind(py), name); - } - if name == AUDITS_KEY - && let Some(audits) = &self.audits - { - return Ok(audits.project(py)?.into_any()); - } - match &self.fields { - ColumnFields::Python(values) => Ok(values[index()?].bind(py).clone()), - ColumnFields::Native(spec) => native_value(py, spec, index()?), - ColumnFields::Projected => field(&self.project(py)?, name), - } - } - - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { - self.projection - .get_or_try_init(py, || { - let mut values: [Bound<'py, PyAny>; 8] = match &self.fields { - ColumnFields::Python(values) => { - values.each_ref().map(|value| value.bind(py).clone()) - } - ColumnFields::Native(spec) => native_values(py, spec)?, - ColumnFields::Projected => { - return Err(PyRuntimeError::new_err("missing column projection")); - } - }; - if let Some(audits) = &self.audits { - values[5] = audits.project(py)?.into_any(); - } - contract_object(self.class.bind(py), values).map(Bound::unbind) - }) - .map(|value| value.bind(py).clone()) - } - - pub(crate) fn from_python(value: Bound<'_, PyAny>) -> PyResult { - let py = value.py(); - let projection = PyOnceLock::new(); - let class = value.get_type().into_any().unbind(); - let _ = projection.set(py, value.unbind()); - Ok(Self { - fields: ColumnFields::Projected, - class, - audits: None, - projection, - }) - } - - pub(crate) fn new<'py>( - class: &Bound<'py, PyAny>, - values: [Bound<'py, PyAny>; 8], - ) -> PyResult { - if !class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { - return Self::from_python(contract_object(class, values)?); - } - checked_field_orders()?; - Ok(Self { - fields: ColumnFields::Python(values.map(Bound::unbind)), - class: class.clone().unbind(), - audits: None, - projection: PyOnceLock::new(), - }) - } - - /// A natively parsed column, whose Python record is built only when Python reads it. - pub(crate) fn native(class: &Bound<'_, PyAny>, spec: NativeColumnSpec) -> PyResult { - let py: Python<'_> = class.py(); - if !class.is(&record_class(py, RecordId::SchemaColumn)?) { - return Self::from_python(contract_object(class, native_values(py, &spec)?)?); - } - checked_field_orders()?; - Ok(Self { - fields: ColumnFields::Native(Box::new(spec)), - class: class.clone().unbind(), - audits: None, - projection: PyOnceLock::new(), - }) - } -} - -impl HeaderColumn { - /// The original native column, if no retained audit edits changed its public contract. - pub(crate) fn native_spec(&self) -> Option<&NativeColumnSpec> { - if self.audits.is_some() { - return None; - } - match &self.fields { - ColumnFields::Native(spec) => Some(spec), - ColumnFields::Python(_) | ColumnFields::Projected => None, - } - } -} - /// The `SchemaColumn` a native column projects to when its location names `location`'s path. pub(crate) fn native_column_record<'py>( py: Python<'py>, @@ -351,7 +128,7 @@ pub(crate) fn native_column_record<'py>( } /// A native column's record fields, in declaration order. -fn native_values<'py>( +pub(crate) fn native_values<'py>( py: Python<'py>, spec: &NativeColumnSpec, ) -> PyResult<[Bound<'py, PyAny>; 8]> { @@ -378,7 +155,7 @@ fn column_values<'py>( } /// The native column's record field at `index` of `SCHEMA_COLUMN_FIELDS`. -fn native_value<'py>( +pub(crate) fn native_value<'py>( py: Python<'py>, spec: &NativeColumnSpec, index: usize, @@ -397,7 +174,10 @@ fn native_value<'py>( } /// The column's `SourceLocation`, or `None`. -fn native_location<'py>(py: Python<'py>, spec: &NativeColumnSpec) -> PyResult> { +pub(crate) fn native_location<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, +) -> PyResult> { match &spec.location { Some((model_file, span)) => { location_record(py, model_file.get().path_objects(py)?.1, *span) @@ -478,69 +258,6 @@ fn native_audit_instance<'py>( ) } -/// Retained native columns and their shared, lazy public tuple. -pub(crate) struct HeaderColumns { - pub(crate) specs: Vec, - projection: PyOnceLock>, -} - -impl HeaderColumns { - pub(crate) fn new(specs: Vec) -> Self { - Self { - specs, - projection: PyOnceLock::new(), - } - } - - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { - self.projection - .get_or_try_init(py, || { - let columns = self - .specs - .iter() - .map(|column| column.project(py)) - .collect::>>()?; - PyTuple::new(py, columns).map(Bound::unbind) - }) - .map(|value| value.bind(py).clone()) - } -} - -/// The Python classes the parsed metadata becomes, read from a Python mapping. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ContractClasses<'py> { - pub(crate) schema_column: Bound<'py, PyAny>, - pub(crate) schema_audit_instance: Bound<'py, PyAny>, - pub(crate) schema_dynamic_column_family: Bound<'py, PyAny>, - pub(crate) source_location: Bound<'py, PyAny>, - measurement_thresholds: Bound<'py, PyAny>, - measurement_threshold_bound: Bound<'py, PyAny>, - pub(crate) severities: Bound<'py, PyDict>, - threshold_operators: Bound<'py, PyDict>, -} - -impl ContractClasses<'_> { - /// Whether these are the classes native header parsing builds by default. - pub(crate) fn uses_default_classes(&self) -> PyResult { - let py: Python<'_> = self.schema_column.py(); - let default: ContractClasses<'_> = contract_classes(py)?.extract()?; - Ok(self.schema_column.is(&default.schema_column) - && self - .schema_audit_instance - .is(&default.schema_audit_instance) - && self.source_location.is(&default.source_location) - && self - .measurement_thresholds - .is(&default.measurement_thresholds) - && self - .measurement_threshold_bound - .is(&default.measurement_threshold_bound) - && self.severities.is(&default.severities) - && self.threshold_operators.is(&default.threshold_operators)) - } -} - pub(crate) fn column_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, @@ -600,7 +317,7 @@ pub(crate) fn parsed_audit_identity( } /// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. -fn contract_object<'py, const N: usize>( +pub(crate) fn contract_object<'py, const N: usize>( class: &Bound<'py, PyAny>, values: [Bound<'py, PyAny>; N], ) -> PyResult> { @@ -609,7 +326,7 @@ fn contract_object<'py, const N: usize>( } /// Fail when a positionally built record no longer declares its fields in the expected order. -fn checked_field_orders() -> PyResult<()> { +pub(crate) fn checked_field_orders() -> PyResult<()> { static MISMATCH: OnceLock> = OnceLock::new(); match MISMATCH.get_or_init(mismatched_record) { Some(record) => Err(PyRuntimeError::new_err(format!( @@ -651,7 +368,7 @@ fn optional<'py, N: HeaderValue>( .unwrap_or_else(|| py.None().into_bound(py))) } -fn schema_column<'py>( +pub(crate) fn schema_column<'py>( py: Python<'py>, classes: &ContractClasses<'py>, column: &ParsedColumn>, diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs new file mode 100644 index 0000000000..cf1bbd1e20 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod header_audits; +pub(crate) mod header_columns; +pub(crate) mod header_families; +pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/mod.rs new file mode 100644 index 0000000000..d496976569 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod config; +pub(crate) mod headers; +pub(crate) mod model_nodes; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/authored_nodes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/authored_nodes.rs new file mode 100644 index 0000000000..1229cfc2f6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/authored_nodes.rs @@ -0,0 +1,10 @@ +//! Python objects read in place as authored header values. + +use crate::model_config::models::PyNode; +use pyo3::prelude::{Bound, PyAny}; + +pub(crate) fn entry_nodes<'py>( + (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> (PyNode<'py>, PyNode<'py>) { + (PyNode(key), PyNode(value)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/column_migrations.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/column_migrations.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs index c810a1a13a..4a1a1ef509 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs @@ -11,11 +11,14 @@ use sqlbuild_model_config::path_defaults::main::select_path_default::select_path use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; use sqlbuild_model_config::types::AuthoredNode; -use crate::bindings::_helpers::model_config::config_nodes::{ - PyConfigEntries, PyConfigNode, config_entries, config_node, config_object, is_text_node, -}; -use crate::bindings::_helpers::model_config::model_config_builder::raised_config_error; -use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; +use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; +use crate::model_config::_helpers::config::config_nodes::config_entries; +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::_helpers::config::config_nodes::is_text_node; +use crate::model_config::_helpers::model_nodes::model_config_builder::raised_config_error; +use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; +use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/mod.rs new file mode 100644 index 0000000000..6c3ecc0b48 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod authored_nodes; +pub(crate) mod column_migrations; +pub(crate) mod contract_classes; +pub(crate) mod described_models; +pub(crate) mod model_config_builder; +pub(crate) mod model_validator; +pub(crate) mod project_defaults; +pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs new file mode 100644 index 0000000000..bbf87cae21 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs @@ -0,0 +1,275 @@ +//! The native model config build handed Python inputs once, and its results projected for Python. + +use crate::model_config::models::BuiltConfig; +use crate::model_config::models::NativeModelConfigBuilder; +use std::collections::HashMap; + +use pyo3::PyErr; +use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use sqlbuild_discovery::model_files::main::config_value::config_value; +use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; +use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::model_config_build::models::{ + BuiltModelConfig, ConfigBuildHalt, HeaderEntryFacts, +}; +use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::authored_values::main::project_var_text::project_var_text; +use crate::model_config::_helpers::config::config_errors::config_exception; +use crate::model_config::_helpers::config::config_errors::native_config_error; +use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; +use crate::model_config::_helpers::config::config_nodes::carried_node; +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::_helpers::config::config_nodes::hook_entry; +use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::typed_config::built_values; +use crate::model_config::errors::NativeConfigError; +use crate::model_config::models::HostHandle; +use crate::model_config::types::PyConfigNode; + +/// The config keys a model's context values read. +pub(crate) const CONTEXT_KEYS: [&str; 3] = ["database", "schema", "alias"]; +const BUILD_STOPPED: &str = "a config value read as text without an error"; + +/// A header's public mapping, parsed facts and native authored values. +pub(crate) type HeaderBuildInputs<'a, 'py> = ( + &'a Bound<'py, PyDict>, + Option<&'a HeaderFacts>, + Option<&'a [(String, AuthoredValue)]>, +); + +/// `((target name, run id), (database, schema) or None)` of the active target. +pub(crate) type TargetInputs<'py> = ( + (Bound<'py, PyAny>, Bound<'py, PyAny>), + Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, +); + +/// The process environment and project-var rendering, read through Python. +pub(crate) struct PythonConfigHost<'py> { + pub(crate) environment: Bound<'py, PyAny>, +} + +impl ConfigBuildHost for PythonConfigHost<'_> { + type Handle = HostHandle; + type Error = PyErr; + + fn environment(&self, name: &str) -> PyResult> { + if !self.environment.contains(name)? { + return Ok(None); + } + Ok(Some(config_node(&self.environment.get_item(name)?)?)) + } + + fn render(&self, value: &PyConfigNode, label: &str) -> PyResult> { + let py: Python<'_> = self.environment.py(); + let object: Bound<'_, PyAny> = config_object(py, value)?; + if let Ok(text) = object.downcast_exact::() { + return Ok(Ok(text.to_str()?.to_owned())); + } + match project_var_text(&object, label) { + Ok(text) => Ok(Ok(text)), + Err(error) if error.is_instance_of::(py) => { + Ok(Err(error.value(py).to_string())) + } + Err(error) => Err(error), + } + } + + fn text_error(&self, value: &PyConfigNode) -> PyErr { + let py: Python<'_> = self.environment.py(); + let read = || -> PyResult<()> { + let object: Bound<'_, PyAny> = config_object(py, value)?; + let text: Bound<'_, PyString> = match object.downcast::() { + Ok(text) => text.clone(), + Err(_) => object.str()?, + }; + let _ = text.to_str()?; + Ok(()) + }; + match read() { + Ok(()) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error, + } + } + + fn python_str(&self, value: &PyConfigNode, exact: bool) -> PyResult { + let text: Bound<'_, PyString> = config_object(self.environment.py(), value)?.str()?; + if exact { + return Ok(text.to_str()?.to_owned()); + } + Ok(text.to_string_lossy().into_owned()) + } + + fn typed_hook_entry(&self, value: &PyConfigNode) -> PyResult { + hook_entry(&config_object(self.environment.py(), value)?) + } +} + +/// Header keys whose values the build reads item by item, which are always converted. +const INSPECTED_KEYS: [&str; 7] = [ + "tags", + "row_diff_exclude_columns", + "row_diff_tolerances", + "pre_hooks", + "post_hooks", + "time_travel_retention", + "table_type", +]; + +/// Whether `key`'s container may be carried whole: no template or macro in it, items never read. +fn carried_whole(key: &str, facts: Option<&[(String, HeaderEntryFacts)]>) -> bool { + !INSPECTED_KEYS.contains(&key) + && facts + .and_then(|facts| facts.iter().find(|(name, _)| name == key)) + .is_some_and(|(_, facts)| !facts.template && facts.macro_path == MacroPath::Absent) +} + +/// A header's entries as config nodes; containers the build never looks into stay opaque. +pub(crate) fn header_entries( + header: &Bound<'_, PyDict>, + facts: Option<&[(String, HeaderEntryFacts)]>, + authored: Option<&[(String, AuthoredValue)]>, +) -> PyResult { + let authored: HashMap<&str, &AuthoredValue> = authored + .into_iter() + .flatten() + .map(|(key, value)| (key.as_str(), value)) + .collect(); + let mut entries: PyConfigEntries = Vec::with_capacity(header.len()); + for (key, value) in header.iter() { + let native: Option = match key.downcast_exact::() { + Ok(key) => match key.to_str() { + Ok(key) => authored.get(key).and_then(|value| config_value(value)), + Err(_) => None, + }, + Err(_) => None, + }; + if let Some(native) = native { + entries.push((config_node(&key)?, native)); + continue; + } + let whole: bool = is_exact_container(&value) + && key + .downcast_exact::() + .is_ok_and(|key| key.to_str().is_ok_and(|key| carried_whole(key, facts))); + let node: PyConfigNode = if whole { + carried_node(&value)? + } else { + config_node(&value)? + }; + entries.push((config_node(&key)?, node)); + } + Ok(entries) +} + +/// One header entry's parsed facts as the build reads them. +pub(crate) fn entry_facts((key, entry): &(String, EntryFacts)) -> (String, HeaderEntryFacts) { + let facts: HeaderEntryFacts = HeaderEntryFacts { + template: entry.template, + macro_path: entry.macro_path.clone(), + }; + (key.clone(), facts) +} + +pub(crate) fn optional_node(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + config_node(&value).map(Some) +} + +/// The Python error a halt raises, or the config error it is rejected with. +pub(crate) fn halted<'py>( + py: Python<'py>, + halt: ConfigBuildHalt, +) -> PyResult> { + Ok(match halt { + ConfigBuildHalt::Config(error) => Ok(error), + ConfigBuildHalt::Host(error) => Err(error), + ConfigBuildHalt::MissingPathDefault(key) => Err(PyKeyError::new_err(key)), + ConfigBuildHalt::KeyNotText(key) => { + Err(match config_object(py, &key)?.downcast::() { + Ok(_) => PyRuntimeError::new_err(BUILD_STOPPED), + Err(error) => error.into(), + }) + } + }) +} + +/// The built values as a new `dict`; entries still the header's own keep the header's objects. +fn own_values_dict<'py>( + header: &Bound<'py, PyDict>, + built: &BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in &built.values { + let own: bool = own_key(key, &built.own_header_keys); + let key: Bound<'py, PyAny> = config_object(py, key)?; + let value: Bound<'py, PyAny> = match own.then(|| header.get_item(&key)).transpose()? { + Some(Some(value)) => value, + _ => config_object(py, value)?, + }; + mapping.set_item(key, value)?; + } + Ok(mapping) +} + +/// Whether `key` is a string key named among the header's own keys. +fn own_key(key: &PyConfigNode, own: &[String]) -> bool { + let text: Option<&str> = match key { + ConfigNode::Str(text) => Some(text), + ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.as_deref(), + _ => None, + }; + text.is_some_and(|text| own.iter().any(|name| name == text)) +} + +/// One model's build projected for Python. +pub(crate) fn projected_build<'py>( + header: &Bound<'py, PyDict>, + built: BuiltModelConfig, +) -> PyResult> { + let py: Python<'py> = header.py(); + Ok(BuiltConfig { + values: own_values_dict(header, &built)?, + nodes: built_values(built.values), + header_keys: built.header_keys, + logical_schema: projected(py, built.logical_schema)?, + layer_schema_configured: built.layer_schema_configured, + logical_database: projected(py, built.logical_database)?, + retention: built.retention, + table_type: built.table_type, + reads: built.reads, + }) +} + +fn projected<'py>( + py: Python<'py>, + value: Option, +) -> PyResult>> { + value.map(|value| config_object(py, &value)).transpose() +} + +/// The Python exception a native config error raises as. +pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { + let raised = || -> PyResult { + let error: Py = native_config_error(py, error)?; + Ok(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, + )?)) + }; + raised().unwrap_or_else(|error| error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs new file mode 100644 index 0000000000..d90bce6055 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs @@ -0,0 +1,107 @@ +//! The Python model validators, run natively up to the first error they raise. + +use crate::model_config::models::NativeModelValidator; +use sqlbuild_python_base::positional_record; +use std::collections::HashSet; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_model_config::model_validation::models::ValidationStop; + +use crate::discovery::main::project::column_name::column_name; +use crate::discovery::main::project::each::each; +use crate::discovery::main::project::stem::stem; +use crate::model_config::_helpers::config::config_errors::native_config_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; + +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const RETURNS_KEY: &str = "returns"; +/// The only key of a table function's `returns` mapping. +const TABLE_RETURNS_KEY: &str = "table"; + +positional_record! { + extract; + /// Model, seed, source, function and table function names. + pub(crate) struct ResourceNames { + pub(crate) models: HashSet, + pub(crate) seeds: HashSet, + pub(crate) sources: HashSet, + pub(crate) functions: HashSet, + pub(crate) table_functions: HashSet, + } +} + +positional_record! { + extract; + /// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. + pub(crate) struct ModelFacts { + pub(crate) references: Vec<(String, String, bool)>, + pub(crate) declared_columns: Option>, + pub(crate) retention_unmanaged: bool, + pub(crate) table_type_declared: bool, + } +} + +/// Model, seed, source, function and table function names discovered in `inputs`. +pub(crate) fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { + let model_stem = |file: &Bound<'_, PyAny>| stem(file, FILE_PATH_ATTRIBUTE); + let mut models: HashSet = each(inputs, "model_files", model_stem)? + .into_iter() + .collect(); + if let Some(injection) = + sqlbuild_python_base::compile_context::main::delta::injection::injection() + { + models.extend(injection.model_names.iter().cloned()); + } + let mut seeds: HashSet = HashSet::new(); + for schema in native_field(inputs, "schema_files")?.try_iter()? { + seeds.extend(each(&schema?, "seed_entries", column_name)?); + } + let mut sources: HashSet = HashSet::new(); + for file in native_field(inputs, "source_files")?.try_iter()? { + sources.extend(each(&file?, "source_entries", column_name)?); + } + let mut functions: HashSet = HashSet::new(); + let mut table_functions: HashSet = HashSet::new(); + for file in native_field(inputs, "sql_function_files")?.try_iter()? { + let file: Bound<'_, PyAny> = file?; + let name: String = model_stem(&file)?; + if returns_table(&file)? { + table_functions.insert(name.clone()); + } + functions.insert(name); + } + functions.extend(each(inputs, "python_function_files", model_stem)?); + Ok(ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + }) +} + +/// Whether a SQL function's header `returns` is a mapping whose only key is `table`. +fn returns_table(file: &Bound<'_, PyAny>) -> PyResult { + let headers: Bound<'_, PyAny> = native_field(file, "header_values")?; + let returns: Bound<'_, PyAny> = mapping_get(&headers, RETURNS_KEY)?; + let Ok(returns) = returns.downcast::() else { + return Ok(false); + }; + let keys: Vec> = returns.keys().into_iter().collect(); + Ok(keys.len() == 1 && keys[0].eq(TABLE_RETURNS_KEY)?) +} + +pub(crate) fn stop_object(py: Python<'_>, stop: ValidationStop) -> PyResult> { + match stop.into_error() { + Err(index) => Ok(index.into_pyobject(py)?.into_any().unbind()), + Ok(error) => Ok(native_config_error(py, error)?.into_any()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/project_defaults.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/project_defaults.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/project_defaults.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/project_defaults.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs rename to crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs diff --git a/crates/sqlbuild-python-front/src/model_config/constants.rs b/crates/sqlbuild-python-front/src/model_config/constants.rs new file mode 100644 index 0000000000..b171bb61ae --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of model config. + +use crate::model_config::_helpers::config::getters::read_native_config_error; +use crate::model_config::errors::NativeConfigError; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_native_config_error, +}]; diff --git a/crates/sqlbuild-python-front/src/model_config/errors.rs b/crates/sqlbuild-python-front/src/model_config/errors.rs new file mode 100644 index 0000000000..8f4c0257b4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/errors.rs @@ -0,0 +1,14 @@ +//! Errors model configuration reports to Python. + +use pyo3::pyclass; + +/// One exact error: the Python class to raise, its message, code, help and named config key. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct NativeConfigError { + pub class_name: &'static str, + pub message: String, + pub code: Option<&'static str>, + pub help: Option, + pub key: Option, +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs new file mode 100644 index 0000000000..4c32af982e --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs @@ -0,0 +1,13 @@ +//! Carry an uninspected value: exact containers keep their own objects without opening items. + +use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Carry an uninspected value: exact containers keep their own objects without opening items. +#[inline] +pub fn carried_node(value: &Bound<'_, PyAny>) -> PyResult { + config_nodes::carried_node(value) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs new file mode 100644 index 0000000000..d018ca76ee --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs @@ -0,0 +1,18 @@ +//! `key`'s cursor bound as text, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `key`'s cursor bound as text, or `None` when absent or `None`. +#[inline] +pub fn config_cursor_bound<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult>> { + typed_config::config_cursor_bound(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs new file mode 100644 index 0000000000..385beff58a --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs @@ -0,0 +1,18 @@ +//! Every `CompileModelConfig` field but `values`, recording the build's template reads first. + +use crate::model_config::_helpers::config::config_classes; +use crate::model_config::models::BuiltConfig; +use crate::model_config::models::ConfigClasses; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Every `CompileModelConfig` field but `values`, recording the build's template reads first. +#[inline] +pub fn config_fields<'py>( + classes: &ConfigClasses<'py>, + built: &BuiltConfig<'py>, + matched: Option<&str>, +) -> PyResult> { + config_classes::config_fields(classes, built, matched) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs new file mode 100644 index 0000000000..6ae5cde364 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs @@ -0,0 +1,12 @@ +//! `key`'s boolean value, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `key`'s boolean value, or `None` when absent or `None`. +#[inline] +pub fn config_flag(py: Python<'_>, values: &ConfigValues<'_>, key: &str) -> PyResult> { + typed_config::config_flag(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs new file mode 100644 index 0000000000..a6ad072d82 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs @@ -0,0 +1,18 @@ +//! `key`'s integer value as its Python object, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `key`'s integer value as its Python object, or `None` when absent or `None`. +#[inline] +pub fn config_integer_object<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult>> { + typed_config::config_integer_object(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs new file mode 100644 index 0000000000..cfa8c0e2fd --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs @@ -0,0 +1,13 @@ +//! One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. + +use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. +#[inline] +pub fn config_node(value: &Bound<'_, PyAny>) -> PyResult { + config_nodes::config_node(value) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs new file mode 100644 index 0000000000..8fa7b8d8f3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs @@ -0,0 +1,13 @@ +//! `values.get(key)` as its typed node, `None` (null) when absent. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::PyResult; +use std::sync::Arc; + +/// `values.get(key)` as its typed node, `None` (null) when absent. +#[inline] +pub fn config_node_of(values: &ConfigValues<'_>, key: &str) -> PyResult> { + typed_config::config_node_of(values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs new file mode 100644 index 0000000000..dbe0dcfb9c --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs @@ -0,0 +1,14 @@ +//! The Python object of one config value: new builtins, and opaque values as themselves. + +use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The Python object of one config value: new builtins, and opaque values as themselves. +#[inline] +pub fn config_object<'py>(py: Python<'py>, node: &PyConfigNode) -> PyResult> { + config_nodes::config_object(py, node) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs new file mode 100644 index 0000000000..2e2bf43ad3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs @@ -0,0 +1,15 @@ +//! An authored string, preserving a present empty string. + +use crate::model_config::_helpers::config::config_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An authored string, preserving a present empty string. +#[inline] +pub fn config_str<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { + config_values::config_str(values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs new file mode 100644 index 0000000000..50d70bbe63 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs @@ -0,0 +1,16 @@ +//! `key`'s string value, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `key`'s string value, or `None` when absent or `None`. +#[inline] +pub fn config_text( + py: Python<'_>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult> { + typed_config::config_text(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs new file mode 100644 index 0000000000..44ca9d53fe --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs @@ -0,0 +1,17 @@ +//! Whether `key` holds a string equal to `expected`, as Python's `==` compares them. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. +#[inline] +pub fn config_text_is( + py: Python<'_>, + values: &ConfigValues<'_>, + key: &str, + expected: &str, +) -> PyResult { + typed_config::config_text_is(py, values, key, expected) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs new file mode 100644 index 0000000000..21b9eb3c03 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs @@ -0,0 +1,18 @@ +//! `key`'s string value as its Python object, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `key`'s string value as its Python object, or `None` when absent or `None`. +#[inline] +pub fn config_text_object<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult>> { + typed_config::config_text_object(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs new file mode 100644 index 0000000000..97dfd82467 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs @@ -0,0 +1,18 @@ +//! `key`'s strings as a tuple, or `None` when absent or `None`. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// `key`'s strings as a tuple, or `None` when absent or `None`. +#[inline] +pub fn config_text_tuple<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult>> { + typed_config::config_text_tuple(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs new file mode 100644 index 0000000000..dac00dad84 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs @@ -0,0 +1,18 @@ +//! `values.get(key)` as its Python object, `None` when absent. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `values.get(key)` as its Python object, `None` when absent. +#[inline] +pub fn config_value<'py>( + py: Python<'py>, + values: &ConfigValues<'_>, + key: &str, +) -> PyResult> { + typed_config::config_value(py, values, key) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs new file mode 100644 index 0000000000..aa4ac4f1f3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs @@ -0,0 +1,16 @@ +//! Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. +#[inline] +pub fn early_model_templates<'py>( + values: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + template_data::early_model_templates(values, effective_vars, (target_name, run_id)) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs new file mode 100644 index 0000000000..77bd879839 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs @@ -0,0 +1,18 @@ +//! Expand every template in `value` and the containers inside it. + +use crate::model_config::_helpers::config::config_templates; +use crate::model_config::models::PythonHost; +use crate::model_config::models::Stop; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use sqlbuild_model_config::templates::models::TemplateOptions; + +/// Expand every template in `value` and the containers inside it. +#[inline] +pub fn expanded<'py>( + host: &PythonHost<'py>, + value: &Bound<'py, PyAny>, + options: TemplateOptions, +) -> Result, Stop> { + config_templates::expanded(host, value, options) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs new file mode 100644 index 0000000000..4ad1859ae5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs @@ -0,0 +1,15 @@ +//! Merged effective vars with recursive `${name}` expansion. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Merged effective vars with recursive `${name}` expansion. +#[inline] +pub fn expanded_effective_vars<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult> { + template_data::expanded_effective_vars(raw_values) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs new file mode 100644 index 0000000000..d4d335fff6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs @@ -0,0 +1,18 @@ +//! `value` with every `${...}` template expanded, recording the reads it made. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_model_config::templates::models::TemplateOptions; + +/// `value` with every `${...}` template expanded, recording the reads it made. +#[inline] +pub fn expanded_template_data<'py>( + value: &Bound<'py, PyAny>, + (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + (options, label): (TemplateOptions, &str), +) -> PyResult> { + template_data::expanded_template_data(value, (variables, context_values), (options, label)) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/mod.rs b/crates/sqlbuild-python-front/src/model_config/main/config/mod.rs new file mode 100644 index 0000000000..65f3e50443 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/mod.rs @@ -0,0 +1,31 @@ +pub mod carried_node; +pub mod config_cursor_bound; +pub mod config_fields; +pub mod config_flag; +pub mod config_integer_object; +pub mod config_node; +pub mod config_node_of; +pub mod config_object; +pub mod config_str; +pub mod config_text; +pub mod config_text_is; +pub mod config_text_object; +pub mod config_text_tuple; +pub mod config_value; +pub mod early_model_templates; +pub mod expanded; +pub mod expanded_effective_vars; +pub mod expanded_template_data; +pub mod model_values; +pub mod native_config_error; +pub mod raised_config_exception; +pub mod record_reads; +pub mod record_template_reads_function; +pub mod register_config_errors; +pub mod register_config_values; +pub mod register_template_data; +pub mod retain_values; +pub mod run_id; +pub mod template_error; +pub mod typed_values; +pub mod view; diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs new file mode 100644 index 0000000000..f8a5513f46 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs @@ -0,0 +1,13 @@ +//! A model's config values as typed nodes, read key by key. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A model's config values as typed nodes, read key by key. +#[inline] +pub fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + typed_config::model_values(model) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs new file mode 100644 index 0000000000..ecd8bf24c2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs @@ -0,0 +1,14 @@ +//! Wrap `error` for Python. + +use crate::model_config::_helpers::config::config_errors; +use crate::model_config::errors::NativeConfigError; +use pyo3::Py; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_model_config::errors::ConfigError; + +/// Wrap `error` for Python. +#[inline] +pub fn native_config_error(py: Python<'_>, error: ConfigError) -> PyResult> { + config_errors::native_config_error(py, error) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs new file mode 100644 index 0000000000..a9c52c7bc0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs @@ -0,0 +1,15 @@ +//! `config_exception` as the raised error, or the failure building it. + +use crate::model_config::_helpers::config::config_errors; +use pyo3::PyErr; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// `config_exception` as the raised error, or the failure building it. +#[inline] +pub fn raised_config_exception( + error: &Bound<'_, PyAny>, + values: Option<&Bound<'_, PyAny>>, +) -> PyErr { + config_errors::raised_config_exception(error, values) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs new file mode 100644 index 0000000000..b45e88ec17 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs @@ -0,0 +1,14 @@ +//! Record native template environment and context reads for project reuse, in order. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Record native template environment and context reads for project reuse, in order. +#[inline] +pub fn record_reads<'a>( + py: Python<'_>, + reads: impl IntoIterator, +) -> PyResult<()> { + template_data::record_reads(py, reads) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs new file mode 100644 index 0000000000..975be395fa --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs @@ -0,0 +1,11 @@ +//! `record_template_reads` as a Python function. + +use crate::model_config::_helpers::config::template_data; + +/// `record_template_reads` as a Python function. +#[inline] +pub fn record_template_reads_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + template_data::record_template_reads_function(py) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs new file mode 100644 index 0000000000..bfb2b4dff2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs @@ -0,0 +1,12 @@ +//! Register the config errors functions and classes. + +use crate::model_config::_helpers::config::config_errors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the config errors functions and classes. +#[inline] +pub fn register_config_errors(module: &Bound<'_, PyModule>) -> PyResult<()> { + config_errors::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs new file mode 100644 index 0000000000..209cb3c148 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs @@ -0,0 +1,12 @@ +//! Register the config values functions and classes. + +use crate::model_config::_helpers::config::config_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the config values functions and classes. +#[inline] +pub fn register_config_values(module: &Bound<'_, PyModule>) -> PyResult<()> { + config_values::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs new file mode 100644 index 0000000000..916f95a334 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs @@ -0,0 +1,12 @@ +//! Register the template data functions and classes. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the template data functions and classes. +#[inline] +pub fn register_template_data(module: &Bound<'_, PyModule>) -> PyResult<()> { + template_data::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs new file mode 100644 index 0000000000..22552f8b28 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs @@ -0,0 +1,13 @@ +//! Attach the built tree to its public record, rather than reconstructing it from that view. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::BuiltConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Attach the built tree to its public record, rather than reconstructing it from that view. +#[inline] +pub fn retain_values(config: &Bound<'_, PyAny>, built: Option<&BuiltConfigValues>) -> PyResult<()> { + typed_config::retain_values(config, built) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs new file mode 100644 index 0000000000..c10fda69f6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs @@ -0,0 +1,12 @@ +//! The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. + +use crate::model_config::_helpers::config::template_data; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. +#[inline] +pub fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { + template_data::run_id(selected_run_id) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs new file mode 100644 index 0000000000..edc8f6f2a5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs @@ -0,0 +1,11 @@ +//! The `CompileInputError` Python raises for a template error in the context `label`. + +use crate::model_config::_helpers::config::config_templates; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::templates::errors::TemplateError; + +/// The `CompileInputError` Python raises for a template error in the context `label`. +#[inline] +pub fn template_error(error: &TemplateError, label: &str) -> ConfigError { + config_templates::template_error(error, label) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs new file mode 100644 index 0000000000..ca4d969669 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs @@ -0,0 +1,13 @@ +//! A config record's values, their typed nodes kept on the record so each key converts once. + +use crate::model_config::_helpers::config::typed_config; +use crate::model_config::types::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A config record's values, their typed nodes kept on the record so each key converts once. +#[inline] +pub fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { + typed_config::typed_values(config) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/view.rs b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs new file mode 100644 index 0000000000..2f6ed7ebdd --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs @@ -0,0 +1,20 @@ +//! A `CompileModelConfig` of `values` and the other `fields`. + +use crate::model_config::_helpers::config::config_classes; +use crate::model_config::models::ConfigClasses; +use crate::model_config::types::BuiltConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// A `CompileModelConfig` of `values` and the other `fields`. +#[inline] +pub fn view<'py>( + classes: &ConfigClasses<'py>, + values: &Bound<'py, PyDict>, + fields: &Bound<'py, PyDict>, + nodes: Option<&BuiltConfigValues>, +) -> PyResult> { + config_classes::view(classes, values, fields, nodes) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs new file mode 100644 index 0000000000..b933a5c862 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs @@ -0,0 +1,20 @@ +//! Audit tuple. + +use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::PyNode; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; + +/// Audit tuple. +#[inline] +pub fn audit_tuple<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: &[ParsedAudit>], +) -> PyResult> { + header_metadata::audit_tuple(py, classes, parsed) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs new file mode 100644 index 0000000000..040da859c0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs @@ -0,0 +1,13 @@ +//! A Python-authored family contract crosses into native ownership once. + +use crate::model_config::_helpers::headers::header_families; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +/// A Python-authored family contract crosses into native ownership once. +#[inline] +pub fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { + header_families::family_specs(families) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/mod.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/mod.rs new file mode 100644 index 0000000000..9db9082d83 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/mod.rs @@ -0,0 +1,8 @@ +pub mod audit_tuple; +pub mod family_specs; +pub mod native_column_record; +pub mod native_schema_column; +pub mod parsed_audit_identity; +pub mod register_header_columns; +pub mod schema_columns; +pub mod thresholds; diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs new file mode 100644 index 0000000000..78e06cd911 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs @@ -0,0 +1,19 @@ +//! The `SchemaColumn` a native column projects to when its location names `location`'s path. + +use crate::model_config::_helpers::headers::header_metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_discovery::model_files::models::NativeColumn; +use sqlbuild_discovery::models::LineColumnSpan; + +/// The `SchemaColumn` a native column projects to when its location names `location`'s path. +#[inline] +pub fn native_column_record<'py>( + py: Python<'py>, + column: &NativeColumn, + location: Option<(Bound<'py, PyAny>, LineColumnSpan)>, +) -> PyResult> { + header_metadata::native_column_record(py, column, location) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs new file mode 100644 index 0000000000..5fe9654ff2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs @@ -0,0 +1,22 @@ +//! Native schema column. + +use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::HeaderColumn; +use crate::model_config::types::HeaderValue; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use sqlbuild_model_config::header_metadata::models::ParsedColumn; + +/// Native schema column. +#[inline] +pub fn native_schema_column<'py, N: HeaderValue>( + py: Python<'py>, + classes: &ContractClasses<'py>, + column: &ParsedColumn, + locations: &Bound<'py, PyDict>, +) -> PyResult { + header_metadata::native_schema_column(py, classes, column, locations) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs new file mode 100644 index 0000000000..aa2fc6838a --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs @@ -0,0 +1,16 @@ +//! `audit.name or audit.definition_name` of the instance `audit` builds. + +use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::types::HeaderValue; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; + +/// `audit.name or audit.definition_name` of the instance `audit` builds. +#[inline] +pub fn parsed_audit_identity( + py: Python<'_>, + audit: &ParsedAudit, +) -> PyResult { + header_metadata::parsed_audit_identity(py, audit) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs new file mode 100644 index 0000000000..398b17e024 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs @@ -0,0 +1,12 @@ +//! Register the header columns functions and classes. + +use crate::model_config::_helpers::headers::header_columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the header columns functions and classes. +#[inline] +pub fn register_header_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + header_columns::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs new file mode 100644 index 0000000000..6c5318feff --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs @@ -0,0 +1,21 @@ +//! The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. + +use crate::model_config::_helpers::headers::header_columns; +use crate::model_config::_helpers::headers::header_columns::ColumnOptions; +use crate::model_config::_helpers::headers::header_columns::ColumnSite; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. +#[inline] +pub fn schema_columns<'py>( + raw_columns: &Bound<'py, PyAny>, + site: ColumnSite<'_>, + options: ColumnOptions, + column_locations: &Bound<'py, PyDict>, +) -> PyResult> { + header_columns::schema_columns(raw_columns, site, options, column_locations) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs new file mode 100644 index 0000000000..74ba4ef14c --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs @@ -0,0 +1,19 @@ +//! `MeasurementThresholds` of `parsed`, or `None`. + +use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::models::ContractClasses; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_model_config::header_metadata::models::ParsedThresholds; + +/// `MeasurementThresholds` of `parsed`, or `None`. +#[inline] +pub fn thresholds<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: Option<&ParsedThresholds>, +) -> PyResult> { + header_metadata::thresholds(py, classes, parsed) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/mod.rs b/crates/sqlbuild-python-front/src/model_config/main/mod.rs new file mode 100644 index 0000000000..54261f6e8c --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/mod.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod headers; +pub mod model_nodes; diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs new file mode 100644 index 0000000000..583bffaebc --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs @@ -0,0 +1,18 @@ +//! The first column-rename problem for a model with these config values, if any. + +use crate::model_config::_helpers::model_nodes::column_migrations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::model_validation::models::ColumnMigration; + +/// The first column-rename problem for a model with these config values, if any. +#[inline] +pub fn column_migration_error( + model_name: &str, + values: &Bound<'_, PyDict>, + columns: &[ColumnMigration], +) -> PyResult> { + column_migrations::column_migration_error(model_name, values, columns) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs new file mode 100644 index 0000000000..47eaec549e --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs @@ -0,0 +1,13 @@ +//! The contract classes native header parsing builds columns and audits as. + +use crate::model_config::_helpers::model_nodes::contract_classes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// The contract classes native header parsing builds columns and audits as. +#[inline] +pub fn contract_classes(py: Python<'_>) -> PyResult> { + contract_classes::contract_classes(py) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs new file mode 100644 index 0000000000..699db0625e --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs @@ -0,0 +1,16 @@ +//! The file path of each model whose layered config or model schema gives it a description. + +use crate::model_config::_helpers::model_nodes::described_models; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The file path of each model whose layered config or model schema gives it a description. +#[inline] +pub fn externally_described_models<'py>( + inputs: Bound<'py, PyAny>, + model_schemas: Bound<'py, PyDict>, +) -> PyResult>> { + described_models::externally_described_models(inputs, model_schemas) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/mod.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/mod.rs new file mode 100644 index 0000000000..2a26dde974 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/mod.rs @@ -0,0 +1,9 @@ +pub mod column_migration_error; +pub mod contract_classes; +pub mod externally_described_models; +pub mod raised_config_error; +pub mod register_model_config_builder; +pub mod register_model_validator; +pub mod register_unicode_environment; +pub mod unicode_environment; +pub mod yaml_entry_classes; diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs new file mode 100644 index 0000000000..62a6e2593b --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs @@ -0,0 +1,12 @@ +//! The Python exception a native config error raises as. + +use crate::model_config::_helpers::model_nodes::model_config_builder; +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_model_config::errors::ConfigError; + +/// The Python exception a native config error raises as. +#[inline] +pub fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { + model_config_builder::raised_config_error(py, error) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs new file mode 100644 index 0000000000..c191e8b135 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs @@ -0,0 +1,12 @@ +//! Register the model config builder functions and classes. + +use crate::model_config::_helpers::model_nodes::model_config_builder; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model config builder functions and classes. +#[inline] +pub fn register_model_config_builder(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_config_builder::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs new file mode 100644 index 0000000000..bd5ccb345e --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs @@ -0,0 +1,12 @@ +//! Register the model validator functions and classes. + +use crate::model_config::_helpers::model_nodes::model_validator; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model validator functions and classes. +#[inline] +pub fn register_model_validator(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_validator::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs new file mode 100644 index 0000000000..6399b67927 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs @@ -0,0 +1,12 @@ +//! Register the unicode environment functions and classes. + +use crate::model_config::_helpers::model_nodes::unicode_environment; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the unicode environment functions and classes. +#[inline] +pub fn register_unicode_environment(module: &Bound<'_, PyModule>) -> PyResult<()> { + unicode_environment::register(module) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs new file mode 100644 index 0000000000..59c24504a6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs @@ -0,0 +1,13 @@ +//! A new view of the process environment as compile text reads it. + +use crate::model_config::_helpers::model_nodes::unicode_environment; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A new view of the process environment as compile text reads it. +#[inline] +pub fn unicode_environment(py: Python<'_>) -> PyResult> { + unicode_environment::unicode_environment(py) +} diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs new file mode 100644 index 0000000000..bc68a1655a --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs @@ -0,0 +1,13 @@ +//! The classes parsed seed and source declarations become. + +use crate::model_config::_helpers::model_nodes::contract_classes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// The classes parsed seed and source declarations become. +#[inline] +pub fn yaml_entry_classes(py: Python<'_>) -> PyResult> { + contract_classes::yaml_entry_classes(py) +} diff --git a/crates/sqlbuild-python-front/src/model_config/mod.rs b/crates/sqlbuild-python-front/src/model_config/mod.rs new file mode 100644 index 0000000000..ada7d471cf --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/mod.rs @@ -0,0 +1,8 @@ +//! Model configuration: authored nodes, config classes, templates, header metadata and validation. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-front/src/model_config/models.rs b/crates/sqlbuild-python-front/src/model_config/models.rs new file mode 100644 index 0000000000..2b2d80cc80 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/models.rs @@ -0,0 +1,1477 @@ +//! Public records of model config. + +use crate::attachments::main::argument_pairs::argument_pairs; +use crate::attachments::main::config_argument_value::config_argument_value; +use crate::authored_values::main::project_var_text::project_var_text; +use crate::discovery::main::project::each::each; +use crate::discovery::models::NativeModelFile; +use crate::model_config::_helpers::config::config_classes::built_config; +use crate::model_config::_helpers::config::config_errors::native_config_error; +use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; +use crate::model_config::_helpers::config::config_nodes::config_dict; +use crate::model_config::_helpers::config::config_nodes::config_entries; +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::_helpers::config::config_nodes::config_object; +use crate::model_config::_helpers::config::config_templates::CONTEXT_READ; +use crate::model_config::_helpers::config::config_templates::EFFECTIVE_VARS_LABEL; +use crate::model_config::_helpers::config::config_templates::ENVIRONMENT_READ; +use crate::model_config::_helpers::config::config_templates::EffectiveVars; +use crate::model_config::_helpers::headers::header_audits::authored_audit; +use crate::model_config::_helpers::headers::header_audits::config_json_value; +use crate::model_config::_helpers::headers::header_audits::retain_audit; +use crate::model_config::_helpers::headers::header_families::family_specs; +use crate::model_config::_helpers::headers::header_metadata::ColumnFields; +use crate::model_config::_helpers::headers::header_metadata::SCHEMA_COLUMN_FIELDS; +use crate::model_config::_helpers::headers::header_metadata::audit_instance; +use crate::model_config::_helpers::headers::header_metadata::checked_field_orders; +use crate::model_config::_helpers::headers::header_metadata::contract_object; +use crate::model_config::_helpers::headers::header_metadata::native_location; +use crate::model_config::_helpers::headers::header_metadata::native_value; +use crate::model_config::_helpers::headers::header_metadata::native_values; +use crate::model_config::_helpers::headers::header_metadata::thresholds; +use crate::model_config::_helpers::model_nodes::authored_nodes::entry_nodes; +use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; +use crate::model_config::_helpers::model_nodes::model_config_builder::CONTEXT_KEYS; +use crate::model_config::_helpers::model_nodes::model_config_builder::HeaderBuildInputs; +use crate::model_config::_helpers::model_nodes::model_config_builder::PythonConfigHost; +use crate::model_config::_helpers::model_nodes::model_config_builder::TargetInputs; +use crate::model_config::_helpers::model_nodes::model_config_builder::entry_facts; +use crate::model_config::_helpers::model_nodes::model_config_builder::halted; +use crate::model_config::_helpers::model_nodes::model_config_builder::header_entries; +use crate::model_config::_helpers::model_nodes::model_config_builder::optional_node; +use crate::model_config::_helpers::model_nodes::model_config_builder::projected_build; +use crate::model_config::_helpers::model_nodes::model_validator::ModelFacts; +use crate::model_config::_helpers::model_nodes::model_validator::ResourceNames; +use crate::model_config::_helpers::model_nodes::model_validator::discovered_names; +use crate::model_config::_helpers::model_nodes::model_validator::stop_object; +use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; +use crate::model_config::errors::NativeConfigError; +use crate::model_config::types::BuiltConfigValues; +use crate::model_config::types::HeaderValue; +use crate::model_config::types::PyConfigNode; +use pyo3::FromPyObject; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyBool; +use pyo3::types::PyBoolMethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyFloat; +use pyo3::types::PyInt; +use pyo3::types::PyList; +use pyo3::types::PyListMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use pyo3::types::PyTuple; +use pyo3::types::PyTupleMethods; +use sqlbuild_attachments::audits::models::ArgumentValue; +use sqlbuild_attachments::functions::models::KnownResources; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_discovery::model_files::models::NativeColumn; +use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::config_values::types::HostObject; +use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; +use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_AUDIT; +use sqlbuild_model_config::header_metadata::constants::AUDITS_KEY; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; +use sqlbuild_model_config::model_config_build::main::build_model_config::build_model_config; +use sqlbuild_model_config::model_config_build::main::inherited_config::inherited_config; +use sqlbuild_model_config::model_config_build::main::model_context_values::model_context_values; +use sqlbuild_model_config::model_config_build::models::ConfigBuildInputs; +use sqlbuild_model_config::model_config_build::models::HeaderEntryFacts; +use sqlbuild_model_config::model_config_build::models::ModelHeader; +use sqlbuild_model_config::model_validation::main::validate_model_config::validate_model_config; +use sqlbuild_model_config::model_validation::main::validate_model_references::validate_model_references; +use sqlbuild_model_config::model_validation::models::ModelReference; +use sqlbuild_model_config::model_validation::models::ModelValidationFacts; +use sqlbuild_model_config::model_validation::models::ProjectValidationFacts; +use sqlbuild_model_config::model_validation::models::RetentionOverride; +use sqlbuild_model_config::model_validation::models::TableTypeOverride; +use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; +use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_model_config::templates::errors::TemplateError; +use sqlbuild_model_config::templates::main::expand_template_text::expand_template_text; +use sqlbuild_model_config::templates::models::ContextValue; +use sqlbuild_model_config::templates::models::Scalar; +use sqlbuild_model_config::templates::models::TemplateFailure; +use sqlbuild_model_config::templates::models::TemplateOptions; +use sqlbuild_model_config::templates::types::TemplateHost; +use sqlbuild_model_config::types::AuthoredNode; +use sqlbuild_model_config::types::NodeKind; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::native_host::models::AuditProjection; +use sqlbuild_python_base::native_host::models::FamilyProjection; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::result_output::main::json_value::json_value; +use std::cell::RefCell; +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::Mutex; + +/// One Python object viewed as an authored value; parsed results keep the object itself. +#[derive(Clone, Debug)] +pub struct PyNode<'py>(pub Bound<'py, PyAny>); + +impl AuthoredNode for PyNode<'_> { + fn kind(&self) -> NodeKind { + let value = &self.0; + if value.is_none() { + NodeKind::Null + } else if let Ok(flag) = value.downcast::() { + NodeKind::Bool(flag.is_true()) + } else if value.is_instance_of::() { + value + .lt(0) + .map_or(NodeKind::Other, |negative| NodeKind::Int { negative }) + } else if value.is_instance_of::() { + NodeKind::Str + } else if value.is_instance_of::() { + NodeKind::List + } else if value.is_instance_of::() { + NodeKind::Tuple + } else if value.is_instance_of::() { + NodeKind::Map + } else { + NodeKind::Other + } + } + + fn integer(&self) -> Option { + if !self.0.is_instance_of::() + && self.0.is_instance_of::() + && let Ok(number) = self.0.extract::() + { + Some(number) + } else { + None + } + } + + fn number(&self) -> Option { + let value = &self.0; + let is_number: bool = value.is_instance_of::() + || (value.is_instance_of::() && !value.is_instance_of::()); + if !is_number { + return None; + } + if let Ok(number) = value.extract::() { + return Some(number); + } + let Ok(number) = self.python_str().parse::() else { + return None; + }; + Some(number) + } + + fn text(&self) -> Option { + self.with_text(str::to_owned) + } + + /// Read a string's text; lone surrogates, rejected where text enters a compile, read as U+FFFD. + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + match self.0.downcast::() { + Ok(value) => Some(read(&value.to_string_lossy())), + Err(_) => None, + } + } + + fn is_text(&self, text: &str) -> bool { + self.with_text(|value| value == text).unwrap_or(false) + } + + fn items(&self) -> Vec { + if let Ok(items) = self.0.downcast::() { + items.iter().map(PyNode).collect() + } else if let Ok(items) = self.0.downcast::() { + items.iter().map(PyNode).collect() + } else { + Vec::new() + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + if let Ok(entries) = self.0.downcast::() { + entries.iter().map(entry_nodes).collect() + } else { + Vec::new() + } + } + + fn python_str(&self) -> String { + self.0 + .str() + .map(|text| text.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + fn python_repr(&self) -> String { + self.0 + .repr() + .map(|text| text.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + fn is_date_like(&self) -> bool { + let py = self.0.py(); + py.import("datetime") + .and_then(|module| native_field(&module, "date")) + .and_then(|date| self.0.is_instance(&date)) + .unwrap_or(false) + } +} + +/// The builder, storage defaults and Python classes a model's `CompileModelConfig` is made of. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct ConfigClasses<'py> { + pub builder: Bound<'py, NativeModelConfigBuilder>, + /// Inherited `(retention, table type)` by table-backed materialization, or `None`. + pub inherited_storage: Bound<'py, PyDict>, + pub table_backed: Vec, + pub compile_model_config: Bound<'py, PyAny>, + pub resolved_retention: Bound<'py, PyAny>, + pub retention_source: Bound<'py, PyAny>, + pub resolved_table_type: Bound<'py, PyAny>, + pub table_type: Bound<'py, PyAny>, + pub table_type_source: Bound<'py, PyAny>, + /// `record_template_reads`, the compile-reuse sink of template reads. + pub record_template_reads: Bound<'py, PyAny>, +} + +/// The Python object an opaque config value stands for. +#[derive(Clone, Debug)] +pub struct HostHandle(pub Arc>); + +impl HostObject for HostHandle { + fn is_date_like(&self) -> bool { + Python::attach(|py| PyNode(self.0.bind(py).clone()).is_date_like()) + } + + /// Python's text of the projected value; a `str()` or `repr()` that raises reads as empty, as `PyNode`. + fn display(node: &ConfigNode, repr: bool) -> String { + Python::attach(|py| { + config_object(py, node).map_or_else( + |_| String::new(), + |value| { + let value: PyNode<'_> = PyNode(value); + if repr { + value.python_repr() + } else { + value.python_str() + } + }, + ) + }) + } +} + +/// Variables, the process environment and context values, all read through Python. +pub struct PythonHost<'py> { + pub(crate) py: Python<'py>, + pub(crate) sources: TemplateSources<'py>, + pub(crate) label: String, + pub(crate) effective: Option>, + pub(crate) reads: RefCell>, + pub(crate) raised: RefCell>, +} + +impl std::fmt::Debug for PythonHost<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("PythonHost").finish_non_exhaustive() + } +} + +impl<'py> PythonHost<'py> { + /// Read variables, the environment and context values from these Python objects. + pub fn new(py: Python<'py>, sources: TemplateSources<'py>, label: &str) -> Self { + Self { + py, + sources, + label: label.to_owned(), + effective: None, + reads: RefCell::new(Vec::new()), + raised: RefCell::new(None), + } + } + + /// Return the `(kind, name)` reads in order. + pub fn into_reads(self) -> Vec<(&'static str, String)> { + self.reads.into_inner() + } + + /// Hold a Python error until the expansion returns, and stop evaluation. + pub(crate) fn hold(&self, error: pyo3::PyErr) -> TemplateFailure { + self.raised.borrow_mut().get_or_insert(error); + TemplateFailure::Host + } + + /// The stop for `failure`, taking the held Python error of a host failure. + pub fn stop(&self, failure: TemplateFailure) -> Stop { + match failure { + TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { + Stop::Failure(error) + } + TemplateFailure::Host => { + Stop::Python(self.raised.borrow_mut().take().unwrap_or_else(|| { + PyRuntimeError::new_err("template expansion stopped without an error") + })) + } + } + } + + pub(crate) fn text_of(&self, value: &Bound<'py, PyAny>) -> Result { + value + .downcast::() + .map_err(|error| self.hold(error.into()))? + .to_str() + .map(str::to_owned) + .map_err(|error| self.hold(error)) + } + + /// Resolve effective var `name`, keeping Python's resolving stack on errors. + pub(crate) fn resolve( + &self, + effective: &EffectiveVars<'py>, + name: &str, + ) -> Result>, TemplateFailure> { + let Some(raw) = self + .sources + .0 + .get_item(name) + .map_err(|error| self.hold(error))? + else { + return Ok(None); + }; + if let Some(value) = effective + .resolved + .get_item(name) + .map_err(|error| self.hold(error))? + { + return Ok(Some(value)); + } + if effective.resolving.borrow().iter().any(|key| key == name) { + let mut cycle = effective.resolving.borrow().clone(); + cycle.push(name.to_owned()); + return Err(TemplateFailure::Invalid(TemplateError::Message(format!( + "{EFFECTIVE_VARS_LABEL} contain a cyclic reference: {}", + cycle.join(" -> ") + )))); + } + effective.resolving.borrow_mut().push(name.to_owned()); + let value = if raw.is_instance_of::() { + let text = self.text_of(&raw)?; + let options = TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }; + PyString::new(self.py, &expand_template_text(self, &text, options)?).into_any() + } else { + raw + }; + effective.resolving.borrow_mut().pop(); + effective + .resolved + .set_item(name, &value) + .map_err(|error| self.hold(error))?; + Ok(Some(value)) + } +} + +/// The variables, the `os.environ` mapping and the context values one expansion reads. +#[derive(Debug, FromPyObject)] +pub struct TemplateSources<'py>( + pub Bound<'py, PyDict>, + pub Bound<'py, PyAny>, + pub Bound<'py, PyDict>, +); + +/// A template error, or a Python error raised while reading values or building containers. +#[derive(Debug)] +pub enum Stop { + Failure(TemplateError), + Python(pyo3::PyErr), +} + +impl<'py> TemplateHost for PythonHost<'py> { + type Value = Bound<'py, PyAny>; + + fn variable(&self, name: &str) -> Result, TemplateFailure> { + if let Some(effective) = &self.effective { + return self.resolve(effective, name); + } + self.sources + .0 + .get_item(name) + .map_err(|error| self.hold(error)) + } + + fn environment(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((ENVIRONMENT_READ, name.to_owned())); + let environment: &Bound<'py, PyAny> = &self.sources.1; + if environment + .contains(name) + .map_err(|error| self.hold(error))? + { + environment + .get_item(name) + .map(Some) + .map_err(|error| self.hold(error)) + } else { + Ok(None) + } + } + + fn context(&self, name: &str) -> Result, TemplateFailure> { + self.reads + .borrow_mut() + .push((CONTEXT_READ, name.to_owned())); + match self + .sources + .2 + .get_item(name) + .map_err(|error| self.hold(error))? + { + None => Ok(ContextValue::Unknown), + Some(value) if value.is_none() => Ok(ContextValue::Unavailable), + Some(value) => Ok(ContextValue::Value(value)), + } + } + + fn text(&self, text: &str) -> Result { + Ok(PyString::new(self.py, text).into_any()) + } + + fn boolean(&self, flag: bool) -> Result { + Ok(PyBool::new(self.py, flag).to_owned().into_any()) + } + + fn null(&self) -> Self::Value { + self.py.None().into_bound(self.py) + } + + fn scalar(&self, value: &Self::Value) -> Result { + if value.is_none() { + return Ok(Scalar::Null); + } + if let Ok(flag) = value.downcast_exact::() { + return Ok(Scalar::Bool(flag.is_true())); + } + let text = value.str().map_err(|error| self.hold(error))?; + self.text_of(text.as_any()).map(Scalar::Text) + } + + fn render(&self, value: &Self::Value, label: &str) -> Result { + if value.is_exact_instance_of::() { + return self.text_of(value); + } + match project_var_text(value, label) { + Ok(text) => Ok(text), + Err(error) if error.is_instance_of::(self.py) => Err( + TemplateFailure::Invalid(TemplateError::Message(error.value(self.py).to_string())), + ), + Err(error) => Err(self.hold(error)), + } + } + + fn label(&self) -> &str { + &self.label + } +} + +/// Native-built specs never become Python audit records just to be read by attachment. +#[derive(Debug)] +pub enum AuditInstance<'py> { + Native(Arc>, Bound<'py, PyAny>), + Python(Bound<'py, PyAny>), +} + +impl<'py> AuditInstance<'py> { + /// Equality with the builtin generated enum audit, without constructing argument containers. + pub fn accepts_enum(&self, values: &[PyConfigNode]) -> PyResult { + let Self::Native(spec, location) = self else { + return Ok(false); + }; + let py = location.py(); + if spec.definition_name.exact_text(py)? != ACCEPTED_VALUES_AUDIT + || spec.thresholds.is_some() + { + return Ok(false); + } + for name in [ + "name", + "description", + "severity", + "run_scope", + "minimum_samples", + "evidence_limit", + "meta", + ] { + if !self.field(name)?.is_none() { + return Ok(false); + } + } + if !self.field("always_run")?.eq(false)? || spec.arguments.len() != 1 { + return Ok(false); + } + let (key, authored) = &spec.arguments[0]; + let matches = match key { + ConfigNode::Str(key) => key == ACCEPTED_VALUES_ARGUMENT, + ConfigNode::Host(_) => { + let key = key.object(py)?; + key.is_instance_of::() && key.eq(ACCEPTED_VALUES_ARGUMENT)? + } + _ => false, + }; + if !matches { + return Ok(false); + } + let ConfigNode::Tuple(authored) = authored else { + return Ok(false); + }; + if authored.len() != values.len() { + return Ok(false); + } + for (left, right) in authored.iter().zip(values) { + if !left.object(py)?.eq(right.object(py)?)? { + return Ok(false); + } + } + Ok(true) + } + + pub fn meta_present(&self) -> PyResult { + match self { + Self::Native(spec, _) => Ok(spec + .meta + .as_ref() + .is_some_and(|value| !matches!(value, ConfigNode::Null))), + Self::Python(value) => Ok(!field(value, "meta")?.is_none()), + } + } + + pub fn threshold_policy(&self) -> PyResult<(bool, bool)> { + match self { + Self::Native(spec, _) => Ok(( + spec.thresholds.is_some(), + spec.thresholds.is_some_and(|value| value.error.is_some()), + )), + Self::Python(value) => { + let thresholds = field(value, "thresholds")?; + Ok(( + !thresholds.is_none(), + !thresholds.is_none() && !field(&thresholds, "error")?.is_none(), + )) + } + } + } + + pub fn field(&self, name: &str) -> PyResult> { + let (spec, location) = match self { + Self::Native(spec, location) => (spec, location), + Self::Python(value) => return field(value, name), + }; + let py = location.py(); + let value = match name { + "definition_name" => return spec.definition_name.object(py), + "name" => &spec.name, + "description" => &spec.description, + "run_scope" => &spec.run_scope, + "minimum_samples" => &spec.minimum_samples, + "evidence_limit" => &spec.evidence_limit, + "meta" => &spec.meta, + "always_run" => { + return match &spec.always_run { + Some(value) => value.object(py), + None => false.into_bound_py_any(py), + }; + } + "severity" => { + let classes = contract_classes(py)?; + let classes: ContractClasses<'_> = classes.extract()?; + return match &spec.severity { + Some(value) => Ok(classes + .severities + .get_item(value.object(py)?)? + .unwrap_or_else(|| py.None().into_bound(py))), + None => Ok(py.None().into_bound(py)), + }; + } + "thresholds" => { + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + return thresholds(py, &classes, spec.thresholds.as_ref()); + } + "location" => return Ok(location.clone()), + _ => return Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), + }; + value + .as_ref() + .map(|value| value.object(py)) + .transpose() + .map(|value| value.unwrap_or_else(|| py.None().into_bound(py))) + } + + pub fn argument(&self, name: &str) -> PyResult>> { + match self { + Self::Native(spec, location) => { + for (key, value) in &spec.arguments { + let matches = match key { + ConfigNode::Str(key) => key == name, + ConfigNode::Host(_) => { + let key = key.object(location.py())?; + key.is_instance_of::() && key.eq(name)? + } + _ => false, + }; + if matches { + return value.object(location.py()).map(Some); + } + } + Ok(None) + } + Self::Python(value) => field(value, "arguments")?.cast::()?.get_item(name), + } + } + + pub fn arguments(&self) -> PyResult> { + match self { + Self::Native(spec, location) => { + let mut arguments = Vec::with_capacity(spec.arguments.len()); + for (key, value) in &spec.arguments { + let key = match key { + ConfigNode::Str(key) => key.clone(), + ConfigNode::Host(_) => { + let key = key.object(location.py())?; + let Ok(key) = key.cast::() else { + continue; + }; + key.to_string_lossy().into_owned() + } + _ => continue, + }; + arguments.push((key, config_argument_value(location.py(), value)?)); + } + Ok(arguments) + } + Self::Python(value) => argument_pairs(field(value, "arguments")?.cast()?), + } + } + + pub fn meta(&self) -> PyResult> { + match self { + Self::Native(spec, location) => spec + .meta + .as_ref() + .map(|value| config_json_value(location.py(), value)) + .transpose() + .map(Option::flatten), + Self::Python(value) => { + let meta = field(value, "meta")?; + if meta.is_none() { + Ok(None) + } else { + json_value(&meta) + } + } + } + } +} + +/// The parsed specs and genuine Python audit objects of one model or column. +#[derive(Debug)] +pub struct HeaderAudits { + pub(crate) specs: Vec>, + pub(crate) native_generated: Vec>, + pub(crate) public: AuditProjection, +} + +impl HeaderAudits { + pub fn new( + py: Python<'_>, + classes: &ContractClasses<'_>, + specs: &[ParsedAudit], + location: &Bound<'_, PyAny>, + ) -> PyResult { + let custom = if classes.uses_default_classes()? { + None + } else { + Some( + specs + .iter() + .map(|spec| audit_instance(py, classes, spec, location).map(Bound::unbind)) + .collect::>()?, + ) + }; + Ok(Self { + specs: specs + .iter() + .map(|spec| retain_audit(py, spec)) + .collect::>()?, + native_generated: Vec::new(), + public: AuditProjection { + custom, + generated: Vec::new(), + location: location.clone().unbind(), + records: PyOnceLock::new(), + }, + }) + } + + pub fn extend(&mut self, generated: Vec>) { + self.public + .generated + .extend(generated.into_iter().map(Bound::unbind)); + } + + pub fn push_native(&mut self, spec: ParsedAudit) { + self.native_generated.push(spec); + } + + /// The number of audits the header declares, generated ones included. + pub fn count(&self) -> usize { + self.specs.len() + self.public.generated.len() + self.native_generated.len() + } + + pub fn native_specs(&self) -> &[ParsedAudit] { + if self.public.custom.is_some() { + &[] + } else { + &self.specs + } + } + + pub fn python_specs<'py>(&self, py: Python<'py>) -> Vec> { + self.public + .custom + .iter() + .flatten() + .chain(&self.public.generated) + .map(|spec| spec.bind(py).clone()) + .collect() + } + + pub fn instances<'py>(&self, py: Python<'py>) -> Vec> { + self.native_specs() + .iter() + .map(|spec| { + AuditInstance::Native( + Arc::new(spec.clone()), + self.public.location.bind(py).clone(), + ) + }) + .chain(self.python_specs(py).into_iter().map(AuditInstance::Python)) + .chain(self.native_generated.iter().map(|spec| { + AuditInstance::Native(Arc::new(spec.clone()), py.None().into_bound(py)) + })) + .collect() + } + + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.public + .records + .get_or_try_init(py, || { + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + let mut audits = self + .native_specs() + .iter() + .map(|spec| audit_instance(py, &classes, spec, self.public.location.bind(py))) + .collect::>>()?; + audits.extend(self.python_specs(py)); + for spec in &self.native_generated { + audits.push(audit_instance( + py, + &classes, + spec, + &py.None().into_bound(py), + )?); + } + PyTuple::new(py, audits).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } + + pub fn key_rows(&self, py: Python<'_>) -> PyResult)>> { + let mut rows = Vec::with_capacity(self.count()); + for spec in self.instances(py) { + let column = spec.argument("column")?; + rows.push(( + spec.field("definition_name")?.extract()?, + column + .filter(|value| value.is_instance_of::()) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) + } +} + +/// Parsed family specs stay native until a public schema attribute needs records. +#[derive(Debug)] +pub struct HeaderFamilies { + pub specs: Vec, + pub(crate) public: FamilyProjection, +} + +impl HeaderFamilies { + pub fn new(class: &Bound<'_, PyAny>, specs: Vec) -> PyResult { + let result = Self { + specs, + public: FamilyProjection::new( + class, + !class.is(&record_class( + class.py(), + RecordId::SchemaDynamicColumnFamily, + )?), + ), + }; + if result.public.custom { + let _ = result.project(class.py())?; + } + Ok(result) + } + + pub fn read(&self, py: Python<'_>) -> PyResult> { + if self.public.custom { + family_specs(self.project(py)?.as_any()) + } else { + Ok(self.specs.clone()) + } + } + + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.public.project(py, &self.specs) + } +} + +/// Native column fields; a Python record is constructed only for public consumers. +pub struct HeaderColumn { + pub(crate) fields: ColumnFields, + pub(crate) class: Py, + pub(crate) audits: Option, + pub(crate) projection: PyOnceLock>, +} + +impl std::fmt::Debug for HeaderColumn { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HeaderColumn") + .finish_non_exhaustive() + } +} + +/// A natively parsed header column and what its Python record is built with. +#[derive(Debug)] +pub struct NativeColumnSpec { + pub column: NativeColumn, + /// The model file whose relative path the column's location names, and the location. + pub location: Option<(Py, LineColumnSpan)>, +} + +impl HeaderColumn { + /// Retype a builtin column without discarding its retained audit specs. + pub fn enum_type( + mut self, + py: Python<'_>, + scalar_type: Bound<'_, PyAny>, + generated: Option>, + ) -> PyResult { + if self.audits.is_none() + && let ColumnFields::Native(spec) = &self.fields + { + let audits = spec + .column + .audits + .iter() + .map(|audit| authored_audit(py, audit)) + .collect::>>()?; + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + self.audits = Some(HeaderAudits::new( + py, + &classes, + &audits, + &native_location(py, spec)?, + )?); + } + if let Some(audits) = &mut self.audits { + if let Some(generated) = generated { + audits.push_native(generated); + } + match &mut self.fields { + ColumnFields::Native(spec) => spec.column.column_type = scalar_type.extract()?, + ColumnFields::Python(values) => values[1] = scalar_type.unbind(), + ColumnFields::Projected => { + return Err(PyRuntimeError::new_err( + "enum column has no retained fields", + )); + } + } + self.projection = PyOnceLock::new(); + return Ok(self); + } + Err(PyRuntimeError::new_err( + "enum column has no retained audits", + )) + } + + pub fn retains_audits(&self) -> bool { + self.audits.is_some() || matches!(self.fields, ColumnFields::Native(_)) + } + + pub fn audit_instances<'py>(&self, py: Python<'py>) -> PyResult>> { + if let Some(audits) = &self.audits { + return Ok(audits.instances(py)); + } + if let ColumnFields::Native(spec) = &self.fields { + let audits = spec + .column + .audits + .iter() + .map(|audit| authored_audit(py, audit)) + .collect::>>()?; + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + return Ok( + HeaderAudits::new(py, &classes, &audits, &native_location(py, spec)?)? + .instances(py), + ); + } + self.field(py, "audits")? + .try_iter()? + .map(|value| value.map(AuditInstance::Python)) + .collect() + } + + pub fn replaced<'py>( + class: &Bound<'py, PyAny>, + values: [Bound<'py, PyAny>; 8], + keyword_order: [usize; 8], + ) -> PyResult { + if class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::new(class, values); + } + let fields = PyDict::new(class.py()); + for index in keyword_order { + fields.set_item(SCHEMA_COLUMN_FIELDS[index], &values[index])?; + } + Self::from_python(class.call((), Some(&fields))?) + } + + pub fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let index = || { + SCHEMA_COLUMN_FIELDS + .iter() + .position(|field| *field == name) + .ok_or_else(|| PyRuntimeError::new_err("unknown column field")) + }; + if let Some(projected) = self.projection.get(py) { + return field(projected.bind(py), name); + } + if name == AUDITS_KEY + && let Some(audits) = &self.audits + { + return Ok(audits.project(py)?.into_any()); + } + match &self.fields { + ColumnFields::Python(values) => Ok(values[index()?].bind(py).clone()), + ColumnFields::Native(spec) => native_value(py, spec, index()?), + ColumnFields::Projected => field(&self.project(py)?, name), + } + } + + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let mut values: [Bound<'py, PyAny>; 8] = match &self.fields { + ColumnFields::Python(values) => { + values.each_ref().map(|value| value.bind(py).clone()) + } + ColumnFields::Native(spec) => native_values(py, spec)?, + ColumnFields::Projected => { + return Err(PyRuntimeError::new_err("missing column projection")); + } + }; + if let Some(audits) = &self.audits { + values[5] = audits.project(py)?.into_any(); + } + contract_object(self.class.bind(py), values).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } + + pub fn from_python(value: Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + let projection = PyOnceLock::new(); + let class = value.get_type().into_any().unbind(); + let _ = projection.set(py, value.unbind()); + Ok(Self { + fields: ColumnFields::Projected, + class, + audits: None, + projection, + }) + } + + pub fn new<'py>(class: &Bound<'py, PyAny>, values: [Bound<'py, PyAny>; 8]) -> PyResult { + if !class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, values)?); + } + checked_field_orders()?; + Ok(Self { + fields: ColumnFields::Python(values.map(Bound::unbind)), + class: class.clone().unbind(), + audits: None, + projection: PyOnceLock::new(), + }) + } + + /// A natively parsed column, whose Python record is built only when Python reads it. + pub fn native(class: &Bound<'_, PyAny>, spec: NativeColumnSpec) -> PyResult { + let py: Python<'_> = class.py(); + if !class.is(&record_class(py, RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, native_values(py, &spec)?)?); + } + checked_field_orders()?; + Ok(Self { + fields: ColumnFields::Native(Box::new(spec)), + class: class.clone().unbind(), + audits: None, + projection: PyOnceLock::new(), + }) + } +} + +impl HeaderColumn { + /// The original native column, if no retained audit edits changed its public contract. + pub fn native_spec(&self) -> Option<&NativeColumnSpec> { + if self.audits.is_some() { + return None; + } + match &self.fields { + ColumnFields::Native(spec) => Some(spec), + ColumnFields::Python(_) | ColumnFields::Projected => None, + } + } +} + +/// Retained native columns and their shared, lazy public tuple. +pub struct HeaderColumns { + pub specs: Vec, + pub(crate) projection: PyOnceLock>, +} + +impl std::fmt::Debug for HeaderColumns { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HeaderColumns") + .finish_non_exhaustive() + } +} + +impl HeaderColumns { + pub fn new(specs: Vec) -> Self { + Self { + specs, + projection: PyOnceLock::new(), + } + } + + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let columns = self + .specs + .iter() + .map(|column| column.project(py)) + .collect::>>()?; + PyTuple::new(py, columns).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } +} + +/// The Python classes the parsed metadata becomes, read from a Python mapping. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct ContractClasses<'py> { + pub schema_column: Bound<'py, PyAny>, + pub schema_audit_instance: Bound<'py, PyAny>, + pub schema_dynamic_column_family: Bound<'py, PyAny>, + pub source_location: Bound<'py, PyAny>, + pub(crate) measurement_thresholds: Bound<'py, PyAny>, + pub(crate) measurement_threshold_bound: Bound<'py, PyAny>, + pub severities: Bound<'py, PyDict>, + pub(crate) threshold_operators: Bound<'py, PyDict>, +} + +impl ContractClasses<'_> { + /// Whether these are the classes native header parsing builds by default. + pub fn uses_default_classes(&self) -> PyResult { + let py: Python<'_> = self.schema_column.py(); + let default: ContractClasses<'_> = contract_classes(py)?.extract()?; + Ok(self.schema_column.is(&default.schema_column) + && self + .schema_audit_instance + .is(&default.schema_audit_instance) + && self.source_location.is(&default.source_location) + && self + .measurement_thresholds + .is(&default.measurement_thresholds) + && self + .measurement_threshold_bound + .is(&default.measurement_threshold_bound) + && self.severities.is(&default.severities) + && self.threshold_operators.is(&default.threshold_operators)) + } +} + +/// The project-wide inputs every model's config build reads, converted once per compile. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeModelConfigBuilder { + pub(crate) inputs: ConfigBuildInputs, + pub(crate) path_keys: Vec, + pub(crate) environment: Py, +} + +/// One model's built config before Python wraps it in `CompileModelConfig`. +#[derive(Debug)] +pub struct BuiltConfig<'py> { + pub values: Bound<'py, PyDict>, + pub nodes: Option, + pub header_keys: Vec, + pub logical_schema: Option>, + pub layer_schema_configured: bool, + pub logical_database: Option>, + pub retention: RetentionOverride, + pub table_type: TableTypeOverride, + pub reads: Vec<(&'static str, String)>, +} + +/// A native build's outcome: the built config, or the config error it raises. +#[derive(Debug)] +pub enum ConfigBuild<'py> { + Built(BuiltConfig<'py>), + Rejected(Py), +} + +#[pymethods] +impl NativeModelConfigBuilder { + #[new] + pub fn new( + layers: (Bound<'_, PyAny>, Bound<'_, PyDict>, Bound<'_, PyTuple>), + sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), + target: TargetInputs<'_>, + ) -> PyResult { + let (run, target_namespace) = target; + let (defaults, path_defaults, _hook_entry_types) = layers; + let mut path_keys: Vec = Vec::with_capacity(path_defaults.len()); + let mut path_values: Vec<(String, PyConfigEntries)> = + Vec::with_capacity(path_defaults.len()); + for (key, values) in path_defaults.iter() { + let key: String = key.extract()?; + path_values.push((key.clone(), config_entries(values.downcast::()?)?)); + path_keys.push(key); + } + let (target_name, run_id) = run; + Ok(Self { + inputs: ConfigBuildInputs { + defaults: config_entries(&project_defaults_mapping(&defaults)?)?, + path_defaults: path_values, + variables: config_entries(&sources.0)?, + run_id: config_node(&run_id)?, + target_name: optional_node(target_name)?.unwrap_or(ConfigNode::Null), + target_namespace: target_namespace + .map(|(database, schema)| -> PyResult<_> { + Ok((config_node(&database)?, config_node(&schema)?)) + }) + .transpose()?, + }, + path_keys, + environment: sources.1.unbind(), + }) + } + + /// Return the model's `CompileModelConfig` or the first config error. + pub(crate) fn model_config<'py>( + &self, + py: Python<'py>, + header: Bound<'py, PyDict>, + model: (Option, String), + classes: ConfigClasses<'py>, + ) -> PyResult> { + let (matched_path_default, model_name) = model; + let matched_path_default: Option<&str> = matched_path_default.as_deref(); + compiler_guard(|| { + match self.built(py, (&header, None, None), matched_path_default, &model_name)? { + ConfigBuild::Built(built) => { + Ok(built_config(&classes, &built, matched_path_default)?.unbind()) + } + ConfigBuild::Rejected(error) => Ok(error.into_any()), + } + }) + } +} + +impl NativeModelConfigBuilder { + pub(crate) fn host<'py>(&self, py: Python<'py>) -> PythonConfigHost<'py> { + PythonConfigHost { + environment: self.environment.bind(py).clone(), + } + } + + /// The selected path-default key, or the conflict error Python raises. + pub fn selected_path_default( + &self, + py: Python<'_>, + model_path: &str, + ) -> PyResult, Py>> { + Ok(match select_path_default(model_path, &self.path_keys) { + PathDefaultChoice::Selected(key) => Ok(key), + PathDefaultChoice::Conflict(error) => Err(native_config_error(py, error)?), + }) + } + + /// Build one model's config natively. + pub fn built<'py>( + &self, + py: Python<'py>, + (header, facts, authored): HeaderBuildInputs<'_, 'py>, + matched_path_default: Option<&str>, + model_name: &str, + ) -> PyResult> { + let facts: Option> = + facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); + let entries: PyConfigEntries = header_entries(header, facts.as_deref(), authored)?; + let model_header = ModelHeader { + entries: &entries, + facts: facts.as_deref(), + }; + let host: PythonConfigHost<'py> = self.host(py); + match build_model_config( + (&host, &self.inputs), + model_header, + matched_path_default, + model_name, + ) { + Ok(built) => Ok(ConfigBuild::Built(projected_build(header, built)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), + Err(error) => Err(error), + }, + } + } + + /// `build_layered_model_values` with an empty header: what a path default's models inherit. + pub fn inherited_values<'py>( + &self, + py: Python<'py>, + matched_path_default: Option<&str>, + ) -> PyResult, Py>> { + let host: PythonConfigHost<'py> = self.host(py); + match inherited_config((&host, &self.inputs), matched_path_default) { + Ok(values) => Ok(Ok(config_dict(py, &values)?)), + Err(halt) => match halted(py, halt)? { + Ok(error) => Ok(Err(native_config_error(py, error)?)), + Err(error) => Err(error), + }, + } + } + + /// `build_model_context_values` for a model's config values. + pub fn model_context_values<'py>( + &self, + values: &Bound<'py, PyDict>, + model_name: &str, + include_target_values: bool, + ) -> PyResult> { + let mut read: PyConfigEntries = Vec::with_capacity(CONTEXT_KEYS.len()); + for key in CONTEXT_KEYS { + if let Some(value) = values.get_item(key)? { + read.push((ConfigNode::Str(key.to_owned()), config_node(&value)?)); + } + } + let context: PyConfigEntries = + model_context_values(&self.inputs, &read, model_name, include_target_values); + config_dict(values.py(), &context) + } + + /// The active target's `(database, schema)`, or `(None, None)` without a target. + pub fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.inputs.target_namespace { + Some((database, schema)) => PyTuple::new( + py, + [config_object(py, database)?, config_object(py, schema)?], + ), + None => PyTuple::new(py, [py.None(), py.None()]), + } + } +} + +/// The project facts every model's validation reads, captured once per compile. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeModelValidator { + pub(crate) project: ProjectValidationFacts, +} + +impl NativeModelValidator { + /// The project's discovered resource names, which every reference resolves against. + pub fn known_resources(&self) -> KnownResources<'_> { + KnownResources { + models: &self.project.models, + seeds: &self.project.seeds, + sources: &self.project.sources, + functions: &self.project.functions, + table_functions: &self.project.table_functions, + } + } +} + +#[pymethods] +impl NativeModelValidator { + /// The validator for the resources and materializations `inputs` discovered. + #[staticmethod] + pub fn from_inputs(inputs: Bound<'_, PyAny>, microbatch_concurrency: bool) -> PyResult { + let names: ResourceNames = discovered_names(&inputs)?; + let custom_materializations: HashSet = + each(&inputs, "materialization_files", |file| { + native_field(file, "name")?.extract() + })? + .into_iter() + .collect(); + Self::new(names, custom_materializations, microbatch_concurrency) + } + + /// The discovered model names references resolve against. + pub(crate) fn model_names(&self) -> HashSet { + self.project.models.clone() + } + + #[new] + pub(crate) fn new( + names: ResourceNames, + custom_materializations: HashSet, + microbatch_concurrency: bool, + ) -> PyResult { + let ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + } = names; + Ok(Self { + project: ProjectValidationFacts { + custom_materializations, + microbatch_concurrency, + models, + seeds, + sources, + functions, + table_functions, + }, + }) + } + + /// `None` when accepted, the first error, or the first externally rejected reference. + pub(crate) fn validate( + &self, + py: Python<'_>, + values: Bound<'_, PyDict>, + model: (String, String, String), + facts: ModelFacts, + ) -> PyResult> { + compiler_guard(|| { + let (model_name, query_sql, relative_path) = model; + let ModelFacts { + references, + declared_columns, + retention_unmanaged, + table_type_declared, + } = facts; + let references: Vec = references + .into_iter() + .map(|(kind, name, externally_rejected)| ModelReference { + kind, + name, + externally_rejected, + }) + .collect(); + let entries: Vec<(PyNode<'_>, PyNode<'_>)> = values + .iter() + .map(|(key, value)| (PyNode(key), PyNode(value))) + .collect(); + let facts = ModelValidationFacts { + model_name: &model_name, + relative_path: &relative_path, + references: &references, + declared_columns: declared_columns.as_deref(), + query_sql: &query_sql, + retention_unmanaged, + table_type_declared, + }; + match validate_model_config(entries, &self.project, &facts) { + Ok(()) => Ok(py.None()), + Err(stop) => stop_object(py, stop), + } + }) + } + + /// Check references alone: `None`, the first error, or a rejected dbt reference's index. + pub(crate) fn references( + &self, + py: Python<'_>, + model: (String, String), + references: Vec<(String, String, bool)>, + ) -> PyResult> { + compiler_guard(|| { + let (model_name, relative_path) = model; + let references: Vec = references + .into_iter() + .map(|(kind, name, externally_rejected)| ModelReference { + kind, + name, + externally_rejected, + }) + .collect(); + let facts = ModelValidationFacts { + model_name: &model_name, + relative_path: &relative_path, + references: &references, + declared_columns: None, + query_sql: "", + retention_unmanaged: false, + table_type_declared: false, + }; + match validate_model_references(&self.project, &facts) { + Ok(()) => Ok(py.None()), + Err(stop) => stop_object(py, stop), + } + }) + } +} + +/// The built tree and any Python-authored values opened by the record's readers. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct TypedConfigValues { + pub built: Option, + pub(crate) read: Mutex)>>, +} diff --git a/crates/sqlbuild-python-front/src/model_config/types.rs b/crates/sqlbuild-python-front/src/model_config/types.rs new file mode 100644 index 0000000000..bf2205a325 --- /dev/null +++ b/crates/sqlbuild-python-front/src/model_config/types.rs @@ -0,0 +1,36 @@ +//! Public traits and type aliases of model config. + +use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::models::HostHandle; +use crate::model_config::models::TypedConfigValues; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::types::AuthoredNode; +use std::collections::HashMap; +use std::sync::Arc; + +/// A config value converted from Python. +pub type PyConfigNode = ConfigNode; + +/// A parsed leaf projected only when a public contract needs its Python value. +pub trait HeaderValue: AuthoredNode { + fn object<'py>(&self, py: Python<'py>) -> PyResult>; + + fn retained(&self, py: Python<'_>) -> PyResult { + config_node(&self.object(py)?) + } + + fn exact_text(&self, py: Python<'_>) -> PyResult { + self.object(py)?.extract() + } +} + +/// A built config's string-keyed tree; values shared across reused configs stay native. +pub type BuiltConfigValues = Arc>>; + +/// A config record's values and the typed nodes read of them, each key converted on first read. +pub type ConfigValues<'py> = (Bound<'py, PyAny>, Bound<'py, TypedConfigValues>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs rename to crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs index dbb1234c40..2a2ecb2147 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs @@ -5,7 +5,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::AUDIT_RESULTS_TABLE_NAME; use sqlbuild_state::audit_results::main::prune_audit_results_sql::prune_audit_results_sql; -use crate::bindings::_helpers::native_host::consumption_state::state_table_name; +use crate::native_host::_helpers::consumption_state::state_table_name; /// SQL deleting audit results beyond the newest `retain_versions` per audit binding. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs similarity index 73% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs rename to crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 363921da4b..59e36ee016 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -1,10 +1,11 @@ //! The consumption table through the adapter, and the connection and callbacks a recording build holds. +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::models::Location; use chrono::{DateTime, NaiveDateTime, Utc}; -use std::sync::Mutex; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; @@ -13,7 +14,6 @@ use sqlbuild_state::consumption::constants::{ }; use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; -use sqlbuild_state::consumption::main::recent_executions::recent_executions; use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, @@ -21,14 +21,11 @@ use sqlbuild_state::consumption::models::{ use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; +use crate::state_rows::main::tables::state_table_create_sql::state_table_create_sql; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -/// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. -pub(crate) const ROWS_PER_STATEMENT: usize = 500; /// Execution IDs per `IN` list when reading executions back by ID. const EXECUTION_ID_BATCH: usize = 500; /// Warehouse timestamp texts with a UTC offset, as drivers render aware values. @@ -36,35 +33,7 @@ const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H: /// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. const NAIVE_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; -/// One destination schema of the consumption table. -#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) struct Location { - pub(crate) database: Option, - pub(crate) schema: String, -} - -impl Location { - pub(crate) fn of(relation: &RelationKey) -> Self { - Self { - database: relation.database.clone(), - schema: relation.schema.clone(), - } - } - - pub(crate) fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { - vec![ - ( - "database", - self.database - .as_deref() - .map_or_else(|| py.None().into_bound(py), |database| text(py, database)), - ), - ("schema", text(py, &self.schema)), - ] - } -} - -fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } @@ -188,7 +157,7 @@ pub(crate) fn execute<'py>( } /// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. -fn ensure_table( +pub(crate) fn ensure_table( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -233,7 +202,7 @@ fn table_exists( } /// Rows of the newest executions per relation in `location`; none when the table is missing. -fn read_recent( +pub(crate) fn read_recent( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -256,7 +225,7 @@ fn read_recent( } /// Rows of the named executions in `location`, read in batches; none when the table is missing. -fn read_executions( +pub(crate) fn read_executions( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -400,7 +369,7 @@ fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyRe } /// Append `rows` to `location`'s table, in statements no longer than every warehouse accepts. -fn append( +pub(crate) fn append( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -430,114 +399,6 @@ fn append( Ok(()) } -/// One build's consumption I/O, connected only per phase so loaders can lock file warehouses. -pub(crate) struct ConsumptionConnection { - adapter: Py, - connection_config: Py, - connection: Mutex>>, -} - -impl ConsumptionConnection { - pub(crate) fn new(adapter: &Bound<'_, PyAny>, connection_config: &Bound<'_, PyAny>) -> Self { - Self { - adapter: adapter.clone().unbind(), - connection_config: connection_config.clone().unbind(), - connection: Mutex::new(None), - } - } - - pub(crate) fn connected<'py>(&self, py: Python<'py>) -> PyResult> { - let mut open = self - .connection - .lock() - .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))?; - if let Some(connection) = open.as_ref() { - return Ok(connection.bind(py).clone()); - } - let connection: Bound<'py, PyAny> = native_field(self.adapter.bind(py), "connect")? - .call1((self.connection_config.bind(py),))?; - *open = Some(connection.clone().unbind()); - Ok(connection) - } - - pub(crate) fn ensure_table(&self, py: Python<'_>, location: &Location) -> PyResult<()> { - ensure_table(self.adapter.bind(py), &self.connected(py)?, location) - } - - pub(crate) fn read_recent( - &self, - py: Python<'_>, - location: &Location, - ) -> PyResult> { - if let Ok(copy) = self.adapter.bind(py).downcast::() { - return Ok(recent_executions( - © - .get() - .consumption((location.database.as_deref(), &location.schema)), - RECENT_EXECUTIONS, - )); - } - read_recent(self.adapter.bind(py), &self.connected(py)?, location) - } - - pub(crate) fn read_executions( - &self, - py: Python<'_>, - location: &Location, - execution_ids: &[&str], - ) -> PyResult> { - if let Ok(copy) = self.adapter.bind(py).downcast::() { - let mut rows: Vec = copy - .get() - .consumption((location.database.as_deref(), &location.schema)); - rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); - return Ok(rows); - } - read_executions( - self.adapter.bind(py), - &self.connected(py)?, - location, - execution_ids, - ) - } - - pub(crate) fn append( - &self, - py: Python<'_>, - location: &Location, - rows: &[ConsumptionAppend], - ) -> PyResult<()> { - append(self.adapter.bind(py), &self.connected(py)?, location, rows) - } - - /// Close the connection if one is open; the next operation connects again. - pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { - let open: Option> = self - .connection - .lock() - .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))? - .take(); - if let Some(connection) = open { - native_field(self.adapter.bind(py), "close")?.call1((connection.bind(py),))?; - } - Ok(()) - } -} - -/// The build's own node-completion callback, which the recorder forwards every result to. -pub(crate) struct NodeCompleteHook(Py); - -impl NodeCompleteHook { - pub(crate) fn new(callback: &Bound<'_, PyAny>) -> Self { - Self(callback.clone().unbind()) - } - - pub(crate) fn forward(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { - self.0.bind(result.py()).call1((result,))?; - Ok(()) - } -} - /// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. pub(crate) fn recorder_callbacks<'py>( recorder: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs new file mode 100644 index 0000000000..378e185cd4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod audit_results_state; +pub(crate) mod consumption_state; +pub(crate) mod observations_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs rename to crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs index c1c06cbf59..bb8276aadc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs @@ -1,5 +1,6 @@ //! DDL and pruning of the display-only observations table through the adapter. +use crate::native_host::models::ObservationAppend; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; @@ -10,9 +11,12 @@ use sqlbuild_state::observations::main::prune_observations_sql::prune_observatio use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use uuid::Uuid; -use crate::bindings::_helpers::native_host::consumption_state::{ - Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, literal, state_table_name, -}; +use crate::native_host::_helpers::consumption_state::execute; +use crate::native_host::_helpers::consumption_state::fixed_state_table_sql; +use crate::native_host::_helpers::consumption_state::literal; +use crate::native_host::_helpers::consumption_state::state_table_name; +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::models::Location; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -54,15 +58,6 @@ fn build_observations_prune_sql( .unwrap_or_default()) } -/// One observation to append; the warehouse clock fills `observed_at`. -pub(crate) struct ObservationAppend { - pub(crate) kind: &'static str, - pub(crate) node: String, - pub(crate) cursor_type: Option, - pub(crate) value: String, - pub(crate) run_id: String, -} - /// Create the schema and observations table in `location` if missing, then append `rows`. pub(crate) fn append_observations( adapter: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/native_host/constants.rs b/crates/sqlbuild-python-front/src/native_host/constants.rs new file mode 100644 index 0000000000..7c8c29b580 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of native host. + +/// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. +pub const ROWS_PER_STATEMENT: usize = 500; diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs new file mode 100644 index 0000000000..5d63443aa6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs @@ -0,0 +1,19 @@ +//! Create the schema and observations table in `location` if missing, then append `rows`. + +use crate::native_host::_helpers::observations_state; +use crate::native_host::models::Location; +use crate::native_host::models::ObservationAppend; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Create the schema and observations table in `location` if missing, then append `rows`. +#[inline] +pub fn append_observations( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ObservationAppend], +) -> PyResult<()> { + observations_state::append_observations(adapter, connection, location, rows) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/execute.rs b/crates/sqlbuild-python-front/src/native_host/main/execute.rs new file mode 100644 index 0000000000..1c9e3b6ff0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/execute.rs @@ -0,0 +1,16 @@ +//! Execute. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Execute. +#[inline] +pub fn execute<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + sql: &str, +) -> PyResult> { + consumption_state::execute(adapter, connection, sql) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs new file mode 100644 index 0000000000..91cfd5cb21 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs @@ -0,0 +1,20 @@ +//! Create-if-missing DDL for a state table with fixed columns, typed by the adapter. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. +#[inline] +pub fn fixed_state_table_sql( + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), +) -> PyResult { + consumption_state::fixed_state_table_sql( + (database, schema), + (render_qualified_name, render_framework_type), + (table_name, column_types, nullable), + ) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs new file mode 100644 index 0000000000..290ad1dd2a --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs @@ -0,0 +1,11 @@ +//! Integer literal. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Integer literal. +#[inline] +pub fn integer_literal(py: Python<'_>, value: Option) -> PyResult { + consumption_state::integer_literal(py, value) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/literal.rs b/crates/sqlbuild-python-front/src/native_host/main/literal.rs new file mode 100644 index 0000000000..5a274ff9fb --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/literal.rs @@ -0,0 +1,11 @@ +//! Literal. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Literal. +#[inline] +pub fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { + consumption_state::literal(py, value) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs new file mode 100644 index 0000000000..effd6e652f --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -0,0 +1,11 @@ +pub mod append_observations; +pub mod execute; +pub mod fixed_state_table_sql; +pub mod integer_literal; +pub mod literal; +pub mod optional_text; +pub mod recorder_callbacks; +pub mod register_audit_results_state; +pub mod register_consumption_state; +pub mod register_observations_state; +pub mod state_table_name; diff --git a/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs new file mode 100644 index 0000000000..1babb19549 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs @@ -0,0 +1,12 @@ +//! Optional text. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Optional text. +#[inline] +pub fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + consumption_state::optional_text(value) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs new file mode 100644 index 0000000000..3a3544c506 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs @@ -0,0 +1,14 @@ +//! The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. +#[inline] +pub fn recorder_callbacks<'py>( + recorder: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + consumption_state::recorder_callbacks(recorder) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs new file mode 100644 index 0000000000..d7cdba3673 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs @@ -0,0 +1,12 @@ +//! Register the audit results state functions and classes. + +use crate::native_host::_helpers::audit_results_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the audit results state functions and classes. +#[inline] +pub fn register_audit_results_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + audit_results_state::register(module) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs new file mode 100644 index 0000000000..84da4f5782 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs @@ -0,0 +1,12 @@ +//! Register the consumption state functions and classes. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the consumption state functions and classes. +#[inline] +pub fn register_consumption_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + consumption_state::register(module) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs new file mode 100644 index 0000000000..97ab0d3395 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs @@ -0,0 +1,12 @@ +//! Register the observations state functions and classes. + +use crate::native_host::_helpers::observations_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the observations state functions and classes. +#[inline] +pub fn register_observations_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + observations_state::register(module) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs new file mode 100644 index 0000000000..5d1c8fd59c --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs @@ -0,0 +1,16 @@ +//! The adapter's qualified name of a state table in `(database, schema)`, if it has one. + +use crate::native_host::_helpers::consumption_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The adapter's qualified name of a state table in `(database, schema)`, if it has one. +#[inline] +pub fn state_table_name( + render_qualified_name: &Bound<'_, PyAny>, + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + table: &str, +) -> PyResult> { + consumption_state::state_table_name(render_qualified_name, (database, schema), table) +} diff --git a/crates/sqlbuild-python-front/src/native_host/mod.rs b/crates/sqlbuild-python-front/src/native_host/mod.rs new file mode 100644 index 0000000000..d3f58713b5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/mod.rs @@ -0,0 +1,6 @@ +//! Warehouse state tables for audit results, consumption and observations. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/native_host/models.rs b/crates/sqlbuild-python-front/src/native_host/models.rs new file mode 100644 index 0000000000..d4fb5a61ea --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/models.rs @@ -0,0 +1,170 @@ +//! Public records of native host. + +use crate::native_host::_helpers::consumption_state::append; +use crate::native_host::_helpers::consumption_state::ensure_table; +use crate::native_host::_helpers::consumption_state::read_executions; +use crate::native_host::_helpers::consumption_state::read_recent; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; +use sqlbuild_state::consumption::main::recent_executions::recent_executions; +use sqlbuild_state::consumption::models::ConsumptionAppend; +use sqlbuild_state::consumption::models::ConsumptionRow; +use sqlbuild_state::consumption::models::RelationKey; +use std::sync::Mutex; + +/// One destination schema of the consumption table. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Location { + pub database: Option, + pub schema: String, +} + +impl Location { + pub fn of(relation: &RelationKey) -> Self { + Self { + database: relation.database.clone(), + schema: relation.schema.clone(), + } + } + + pub fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { + vec![ + ( + "database", + self.database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)), + ), + ("schema", text(py, &self.schema)), + ] + } +} + +/// One build's consumption I/O, connected only per phase so loaders can lock file warehouses. +#[derive(Debug)] +pub struct ConsumptionConnection { + pub(crate) adapter: Py, + pub(crate) connection_config: Py, + pub(crate) connection: Mutex>>, +} + +impl ConsumptionConnection { + pub fn new(adapter: &Bound<'_, PyAny>, connection_config: &Bound<'_, PyAny>) -> Self { + Self { + adapter: adapter.clone().unbind(), + connection_config: connection_config.clone().unbind(), + connection: Mutex::new(None), + } + } + + pub fn connected<'py>(&self, py: Python<'py>) -> PyResult> { + let mut open = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))?; + if let Some(connection) = open.as_ref() { + return Ok(connection.bind(py).clone()); + } + let connection: Bound<'py, PyAny> = native_field(self.adapter.bind(py), "connect")? + .call1((self.connection_config.bind(py),))?; + *open = Some(connection.clone().unbind()); + Ok(connection) + } + + pub fn ensure_table(&self, py: Python<'_>, location: &Location) -> PyResult<()> { + ensure_table(self.adapter.bind(py), &self.connected(py)?, location) + } + + pub fn read_recent( + &self, + py: Python<'_>, + location: &Location, + ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + return Ok(recent_executions( + © + .get() + .consumption((location.database.as_deref(), &location.schema)), + RECENT_EXECUTIONS, + )); + } + read_recent(self.adapter.bind(py), &self.connected(py)?, location) + } + + pub fn read_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + let mut rows: Vec = copy + .get() + .consumption((location.database.as_deref(), &location.schema)); + rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); + return Ok(rows); + } + read_executions( + self.adapter.bind(py), + &self.connected(py)?, + location, + execution_ids, + ) + } + + pub fn append( + &self, + py: Python<'_>, + location: &Location, + rows: &[ConsumptionAppend], + ) -> PyResult<()> { + append(self.adapter.bind(py), &self.connected(py)?, location, rows) + } + + /// Close the connection if one is open; the next operation connects again. + pub fn close(&self, py: Python<'_>) -> PyResult<()> { + let open: Option> = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))? + .take(); + if let Some(connection) = open { + native_field(self.adapter.bind(py), "close")?.call1((connection.bind(py),))?; + } + Ok(()) + } +} + +/// The build's own node-completion callback, which the recorder forwards every result to. +#[derive(Debug)] +pub struct NodeCompleteHook(Py); + +impl NodeCompleteHook { + pub fn new(callback: &Bound<'_, PyAny>) -> Self { + Self(callback.clone().unbind()) + } + + pub fn forward(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + self.0.bind(result.py()).call1((result,))?; + Ok(()) + } +} + +/// One observation to append; the warehouse clock fills `observed_at`. +#[derive(Debug)] +pub struct ObservationAppend { + pub kind: &'static str, + pub node: String, + pub cursor_type: Option, + pub value: String, + pub run_id: String, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs rename to crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs index 248b4cdcd0..374c433043 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs @@ -7,7 +7,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; +use crate::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs rename to crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/mod.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/mod.rs new file mode 100644 index 0000000000..eaa35e18a1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod contract_compare; +pub(crate) mod contract_edits; +pub(crate) mod python_strings; +pub(crate) mod refactor_facts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs rename to crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs rename to crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs index 3bfea8498c..bdee9fe23e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs @@ -12,11 +12,12 @@ use sqlbuild_refactor::refactoring::models::{ }; use sqlbuild_scopes::scope_index::models::{ResourceIdentity, ResourceKind}; -use crate::bindings::_helpers::model_config::config_values::config_str; -use crate::bindings::_helpers::refactor::python_strings::find_python_string_locations; -use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; -use crate::bindings::_helpers::scopes::scope_reports::{scope_move_preview, scope_rebound_usages}; -use crate::bindings::_helpers::scopes::scope_validation::relocated_declarations; +use crate::model_config::main::config::config_str::config_str; +use crate::refactor::_helpers::python_strings::find_python_string_locations; +use crate::scopes::main::checks::relocated_declarations::relocated_declarations; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use crate::scopes::main::reports::scope_move_preview::scope_move_preview; +use crate::scopes::main::reports::scope_rebound_usages::scope_rebound_usages; use sqlbuild_python_base::records::main::fields::field::field as native_field; const SQL_SUFFIX: &str = ".sql"; diff --git a/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs new file mode 100644 index 0000000000..ef10768f9f --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs @@ -0,0 +1,15 @@ +//! Selected declarations compared with one bulk read of physical columns. + +use crate::refactor::_helpers::contract_compare; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Selected declarations compared with one bulk read of physical columns. +#[inline] +pub fn compare_contracts<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + contract_compare::compare_contracts(request) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs new file mode 100644 index 0000000000..14ff083598 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs @@ -0,0 +1,18 @@ +//! Where scope placement puts declarations once every `(model, destination)` moves together. + +use crate::refactor::_helpers::refactor_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_refactor::refactoring::models::DeclarationPlacement; +use std::collections::BTreeSet; + +/// Where scope placement puts declarations once every `(model, destination)` moves together. +#[inline] +pub fn declaration_placement( + project: &Bound<'_, PyAny>, + moves: &[(String, String)], + vacated: &BTreeSet, +) -> PyResult { + refactor_facts::declaration_placement(project, moves, vacated) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs new file mode 100644 index 0000000000..c924eea433 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs @@ -0,0 +1,18 @@ +//! The model's MODEL header with physical columns adopted, or the ownership conflict. + +use crate::refactor::_helpers::contract_edits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The model's MODEL header with physical columns adopted, or the ownership conflict. +#[inline] +pub fn edit_contract_model( + py: Python<'_>, + model: &Bound<'_, PyAny>, + physical_columns: &Bound<'_, PyAny>, + overwrite: bool, +) -> PyResult<(Option, Option)> { + contract_edits::edit_contract_model(py, model, physical_columns, overwrite) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs new file mode 100644 index 0000000000..ca1b85b4d6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs @@ -0,0 +1,17 @@ +//! The source's YAML declaration with physical columns adopted, or the ownership conflict. + +use crate::refactor::_helpers::contract_edits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The source's YAML declaration with physical columns adopted, or the ownership conflict. +#[inline] +pub fn edit_contract_source( + source: &Bound<'_, PyAny>, + physical_columns: &Bound<'_, PyAny>, + overwrite: bool, + contents: Option, +) -> PyResult<(Option, Option)> { + contract_edits::edit_contract_source(source, physical_columns, overwrite, contents) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/mod.rs b/crates/sqlbuild-python-front/src/refactor/main/mod.rs new file mode 100644 index 0000000000..7a22599a8d --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/mod.rs @@ -0,0 +1,11 @@ +pub mod compare_contracts; +pub mod declaration_placement; +pub mod edit_contract_model; +pub mod edit_contract_source; +pub mod model_facts; +pub mod refactor_facts; +pub mod register_contract_compare; +pub mod register_contract_edits; +pub mod register_python_strings; +pub mod remaining_contract_findings; +pub mod unit_test_rebindings; diff --git a/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs new file mode 100644 index 0000000000..315bfc6065 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs @@ -0,0 +1,13 @@ +//! Every compiled model's refactoring facts, or none without a project. + +use crate::refactor::_helpers::refactor_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_refactor::refactoring::models::ModelFacts; + +/// Every compiled model's refactoring facts, or none without a project. +#[inline] +pub fn model_facts(refactor_project: &Bound<'_, PyAny>) -> PyResult>> { + refactor_facts::model_facts(refactor_project) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs new file mode 100644 index 0000000000..dc984f9324 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs @@ -0,0 +1,17 @@ +//! The facts one refactoring plan of `requests` reads. + +use crate::refactor::_helpers::refactor_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_refactor::refactoring::models::RefactorFacts; +use sqlbuild_refactor::refactoring::models::RefactorRequest; + +/// The facts one refactoring plan of `requests` reads. +#[inline] +pub fn refactor_facts( + project: &Bound<'_, PyAny>, + requests: &[RefactorRequest], +) -> PyResult { + refactor_facts::refactor_facts(project, requests) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs new file mode 100644 index 0000000000..bdf7573d6a --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs @@ -0,0 +1,12 @@ +//! Register the contract compare functions and classes. + +use crate::refactor::_helpers::contract_compare; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the contract compare functions and classes. +#[inline] +pub fn register_contract_compare(module: &Bound<'_, PyModule>) -> PyResult<()> { + contract_compare::register(module) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs new file mode 100644 index 0000000000..0b3046740f --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs @@ -0,0 +1,12 @@ +//! Register the contract edits functions and classes. + +use crate::refactor::_helpers::contract_edits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the contract edits functions and classes. +#[inline] +pub fn register_contract_edits(module: &Bound<'_, PyModule>) -> PyResult<()> { + contract_edits::register(module) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs new file mode 100644 index 0000000000..e5f93791ba --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs @@ -0,0 +1,12 @@ +//! Register the python strings functions and classes. + +use crate::refactor::_helpers::python_strings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python strings functions and classes. +#[inline] +pub fn register_python_strings(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_strings::register(module) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs new file mode 100644 index 0000000000..f8ef7ff7d7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs @@ -0,0 +1,19 @@ +//! The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. + +use crate::refactor::_helpers::contract_edits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. +#[inline] +pub fn remaining_contract_findings<'py>( + py: Python<'py>, + findings: &Bound<'py, PyAny>, + overwrite: bool, + conflict: Option, +) -> PyResult> { + contract_edits::remaining_contract_findings(py, findings, overwrite, conflict) +} diff --git a/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs new file mode 100644 index 0000000000..9b31ee349d --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs @@ -0,0 +1,15 @@ +//! Scoped declarations the test file at `path` uses that resolve differently from `destination`. + +use crate::refactor::_helpers::refactor_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Scoped declarations the test file at `path` uses that resolve differently from `destination`. +#[inline] +pub fn unit_test_rebindings( + project: &Bound<'_, PyAny>, + paths: (&str, &str), +) -> PyResult>> { + refactor_facts::unit_test_rebindings(project, paths) +} diff --git a/crates/sqlbuild-python-front/src/refactor/mod.rs b/crates/sqlbuild-python-front/src/refactor/mod.rs new file mode 100644 index 0000000000..5911569de6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/refactor/mod.rs @@ -0,0 +1,4 @@ +//! Contract comparison and edits, Python string literals and refactor facts. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs rename to crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs rename to crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs index fa114160c3..c46dd48fb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs @@ -7,8 +7,9 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_render::macro_calls::main::call_class_store_text::call_class_store_text; -use crate::bindings::_helpers::macro_bridge::store_keys::{context_store_token, macro_store_token}; -use crate::bindings::_helpers::render::macro_expansion_facts::ResourceRef; +use crate::macro_bridge::main::context_store_token::context_store_token; +use crate::macro_bridge::main::macro_store_token::macro_store_token; +use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_memo.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs rename to crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_memo.rs index 8ad46bdaf7..f95c22d737 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_memo.rs @@ -10,7 +10,7 @@ use std::sync::Mutex; use sqlbuild_python_base::cache::main::open_store::open_store; use sqlbuild_python_base::cache::main::save_store::save_store; -use crate::bindings::_helpers::render::macro_call_classes::CallClassState; +use crate::render::_helpers::calls::macro_call_classes::CallClassState; /// The native store kind holding recorded macro calls. const MACRO_CALL_STORE_KIND: &str = "macro-calls"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs rename to crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs index a693d6aee9..6d5b316bd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs @@ -1,9 +1,10 @@ //! Python rows of macro call sites and splice spans. +use crate::render::models::SpanTuple; +use crate::render::types::SpanRow; use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_python_base::positional_record; use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; -use sqlbuild_sqltext::compiler::models::CharSpan; positional_record! { into; @@ -17,19 +18,6 @@ positional_record! { typed_reference_text: bool, } } -/// One substitution span: source start and end, output start and end. -pub(crate) type SpanRow = CharSpan; - -positional_record! { - into; - /// A span as the tuple Python reads: source start and end, output start and end. - pub(crate) struct SpanTuple { - source_start: usize, - source_end: usize, - output_start: usize, - output_end: usize, - } -} /// A span as the tuple Python reads. pub(crate) fn span_tuple(span: &SpanRow) -> SpanTuple { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs rename to crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs index 5c8d4f0f10..3a574a0d63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs @@ -9,11 +9,13 @@ use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_s use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroScanFailure}; -use crate::bindings::_helpers::render::macro_call_memo::MacroCallMemo; -use crate::bindings::_helpers::render::macro_call_rows::{ - SiteRow, SpanTuple, site_row, span_row, span_tuple, -}; -use crate::bindings::_helpers::render::macro_expansion::raised; +use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; +use crate::render::_helpers::calls::macro_call_rows::SiteRow; +use crate::render::_helpers::calls::macro_call_rows::site_row; +use crate::render::_helpers::calls::macro_call_rows::span_row; +use crate::render::_helpers::calls::macro_call_rows::span_tuple; +use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::models::SpanTuple; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/mod.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/mod.rs new file mode 100644 index 0000000000..9897c241f9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod macro_arguments; +pub(crate) mod macro_call_classes; +pub(crate) mod macro_call_memo; +pub(crate) mod macro_call_rows; +pub(crate) mod macro_calls; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs rename to crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs index 58280d96e9..386a946494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs @@ -1,6 +1,7 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. -use pyo3::prelude::Py; +use crate::render::models::AuthoredExpansion; +use crate::render::models::SqlExpansionInputs; use pyo3::prelude::{ Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -8,23 +9,27 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::model_config::template_data::record_reads; -use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, - expand_declarations, span_rows, -}; -use crate::bindings::_helpers::render::declaration_values::ConstantRendering; -use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; -use crate::bindings::_helpers::render::macro_call_rows::{span_row, span_tuple}; -use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; -use crate::bindings::_helpers::render::sql_macro_host::SqlMacroSource; -use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; -use crate::bindings::_helpers::sqltext::static_sql::{ - InterpolationRow, InterpolationSources, interpolation_batch, -}; +use crate::model_config::main::config::record_reads::record_reads; +use crate::model_config::main::model_nodes::unicode_environment::unicode_environment; +use crate::render::_helpers::calls::macro_call_rows::span_row; +use crate::render::_helpers::calls::macro_call_rows::span_tuple; +use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; +use crate::render::_helpers::declarations::declaration_expansion::span_rows; +use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; +use crate::render::_helpers::expansion::macro_expansion_result::macro_output; +use crate::render::models::ConstantRendering; +use crate::render::models::DeclarationExpansion; +use crate::render::models::DeclarationExpansionClasses; +use crate::render::models::ExpandedDeclarations; +use crate::render::models::ExpansionConsumer; +use crate::render::models::ExpansionRequest; +use crate::render::models::SqlMacroSource; +use crate::render::types::SpanRow; +use crate::sqltext::main::declaration_scan::declaration_scan; +use crate::sqltext::main::interpolation_batch::interpolation_batch; +use crate::sqltext::main::optional_record_syntax::optional_record_syntax; +use crate::sqltext::models::InterpolationRow; +use crate::sqltext::models::InterpolationSources; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -151,7 +156,7 @@ fn declarations_expanded<'py>( value_renderer.clone().unbind(), collection_rendering.clone().unbind(), ); - let classes = ExpansionClasses { + let classes = DeclarationExpansionClasses { usage_record: record_class(py, RecordId::UsageRecord)?, compile_error: error_class(py, ErrorId::CompileInput)?, }; @@ -179,13 +184,6 @@ fn declarations_expanded<'py>( Ok((expanded.spliced.sql, spans)) } -/// Authored SQL expanded through vars, declarations and macros, with each pass's spans. -pub(crate) struct AuthoredExpansion { - pub(crate) sql: String, - /// Interpolation, declaration and macro spans, in pass order. - pub(crate) passes: [Vec; 3], -} - /// What one authored SQL expansion reads. struct AuthoredSources<'a, 'py> { sql: &'a Bound<'py, PyAny>, @@ -400,66 +398,3 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; Ok(()) } - -/// The inputs authored SQL is expanded with outside a compile; the handles carry user values. -pub(crate) struct SqlExpansionInputs { - pub(crate) effective_vars: Py, - pub(crate) loaded_macros: Py, - pub(crate) macro_context: Py, - pub(crate) enums: Py, - pub(crate) constants: Py, - pub(crate) value_renderer: Py, - pub(crate) collection_rendering: Py, - /// Each model file's own declarations, by file path. - pub(crate) local_declarations: Py, - pub(crate) declaration_resolver: Py, -} - -impl SqlExpansionInputs { - /// The inputs a `SqlExpansionContext` record holds. - pub(crate) fn from_record(record: &Bound<'_, PyAny>) -> PyResult { - let field = - |name: &str| -> PyResult> { Ok(native_field(record, name)?.unbind()) }; - Ok(Self { - effective_vars: field("effective_vars")?, - loaded_macros: field("loaded_macros")?, - macro_context: field("macro_context")?, - enums: field("enums")?, - constants: field("constants")?, - value_renderer: field("value_renderer")?, - collection_rendering: field("collection_rendering")?, - local_declarations: native_field(record, "local_declarations")? - .downcast_into::()? - .unbind(), - declaration_resolver: field("declaration_resolver")?, - }) - } - - /// These inputs as a `SqlExpansionContext` record. - pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { - new_record_with( - py, - RecordId::SqlExpansionContext, - &[ - ("effective_vars", self.effective_vars.bind(py).clone()), - ("loaded_macros", self.loaded_macros.bind(py).clone()), - ("macro_context", self.macro_context.bind(py).clone()), - ("enums", self.enums.bind(py).clone()), - ("constants", self.constants.bind(py).clone()), - ("value_renderer", self.value_renderer.bind(py).clone()), - ( - "collection_rendering", - self.collection_rendering.bind(py).clone(), - ), - ( - "local_declarations", - self.local_declarations.bind(py).clone().into_any(), - ), - ( - "declaration_resolver", - self.declaration_resolver.bind(py).clone(), - ), - ], - ) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs rename to crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs index 5428788f91..a641972038 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs @@ -1,7 +1,9 @@ //! Enum and constant references expanded natively, for model queries and every other caller. -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; -use pyo3::FromPyObject; +use crate::render::models::DeclarationExpansion; +use crate::render::models::DeclarationExpansionClasses; +use crate::render::models::ExpandedDeclarations; +use crate::render::types::SpanRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; use sqlbuild_analysis::compiled_project::errors::DeclarationError; @@ -11,13 +13,12 @@ use sqlbuild_analysis::compiled_project::main::splice_declarations::splice_decla use sqlbuild_analysis::compiled_project::models::{ DeclarationSplice, InaccessibleDeclaration, SplicedSql, }; -use sqlbuild_analysis::sql_macros::models::{ExpandedText, SqlMacroCheck, TypedCall}; +use sqlbuild_analysis::sql_macros::models::{ExpandedText, TypedCall}; -use crate::bindings::_helpers::render::declaration_values::ConstantRendering; -use crate::bindings::_helpers::render::sql_macro_host::{ - SqlMacroSource, expanded_sql_macro_calls, is_sql_macro_reference, reject_bare_macro_calls, -}; -use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; +use crate::render::_helpers::sql_macros::sql_macro_host::expanded_sql_macro_calls; +use crate::render::_helpers::sql_macros::sql_macro_host::is_sql_macro_reference; +use crate::render::_helpers::sql_macros::sql_macro_host::reject_bare_macro_calls; +use crate::sqltext::models::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; @@ -45,47 +46,6 @@ const THROUGH: &str = "through"; const CONSUMER: &str = "consumer"; const ENUM_MEMBER: &str = "enum_member"; -/// The Python classes expansion builds usages and errors from. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ExpansionClasses<'py> { - pub(crate) usage_record: Bound<'py, PyAny>, - pub(crate) compile_error: Bound<'py, PyAny>, -} - -/// Who reads the declarations, whether it prefers relationship visibility, and its error path. -#[derive(Clone)] -pub(crate) struct ExpansionConsumer<'a, 'py> { - pub(crate) identity: Option>, - pub(crate) relationship_visibility: bool, - pub(crate) path: &'a str, -} - -/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant rendering. -pub(crate) struct DeclarationExpansion<'a, 'py> { - pub(crate) sql: &'a str, - pub(crate) references: &'a [ScannedReference], - pub(crate) stop: Option, - pub(crate) context: &'a Bound<'py, PyAny>, - pub(crate) consumer: ExpansionConsumer<'a, 'py>, - pub(crate) rendering: &'a ConstantRendering, - pub(crate) classes: &'a ExpansionClasses<'py>, - /// What SQL macro calls read beyond the context; `None` where SQL macros are not expanded. - pub(crate) sql_macros: Option>, -} - -/// The expanded SQL with its spans, and the usage records it adds in reference order. -pub(crate) struct ExpandedDeclarations<'py> { - pub(crate) spliced: SplicedSql, - pub(crate) usages: Vec>, - /// The SQL macros the SQL called, top-level and nested, once each. - pub(crate) sql_macros: Vec, - /// What the analysed SQL must show for each top-level call's arguments to fit. - pub(crate) sql_macro_checks: Vec, - /// Each expression macro call in the spliced SQL, by byte range. - pub(crate) typed_calls: Vec, -} - /// Splice each reference in order, raising the attachment's error for the first bad one. pub(crate) fn expand_declarations<'py>( request: &DeclarationExpansion<'_, 'py>, @@ -314,7 +274,7 @@ fn inaccessible_record(record: &Bound<'_, PyAny>) -> PyResult( +pub(crate) fn visibility<'py>( request: &DeclarationExpansion<'_, 'py>, attribute: &str, name: &str, @@ -345,8 +305,8 @@ pub(crate) fn usage_visibility<'py>( Ok(if related.is_empty() { records } else { related }) } -fn usage<'py>( - classes: &ExpansionClasses<'py>, +pub(crate) fn usage<'py>( + classes: &DeclarationExpansionClasses<'py>, consumer: &Bound<'py, PyAny>, record: &Bound<'py, PyAny>, member: Option<&str>, @@ -359,7 +319,10 @@ fn usage<'py>( classes.usage_record.call((), Some(&fields)) } -fn compile_error(request: &DeclarationExpansion<'_, '_>, error: &DeclarationError) -> PyErr { +pub(crate) fn compile_error( + request: &DeclarationExpansion<'_, '_>, + error: &DeclarationError, +) -> PyErr { let message: String = declaration_error_message(error, request.consumer.path); match request.classes.compile_error.call1((message,)) { Ok(instance) => PyErr::from_value(instance), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs rename to crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs rename to crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs index 886cf48f92..73f68d7474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs @@ -1,14 +1,14 @@ //! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; -use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use crate::scopes::models::NativeDeclarationContexts; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -432,49 +432,6 @@ pub(crate) fn constant_declaration_sql<'py>( Err(PyErr::from_value(compile_error)) } -/// The adapter's constant rendering a compile's declaration expansions share. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ConstantRendering { - value_renderer: Py, - collection_rendering: Py, -} - -impl ConstantRendering { - /// Rendering with the adapter's value renderer and the project's collection rendering. - pub(crate) fn new(value_renderer: Py, collection_rendering: Py) -> Self { - Self { - value_renderer, - collection_rendering, - } - } - - /// The adapter name the value renderer renders for, empty when it names none. - pub(crate) fn adapter_name(&self, py: Python<'_>) -> String { - native_field(self.value_renderer.bind(py), "adapter_name") - .and_then(|name| name.extract::()) - .unwrap_or_default() - } - - /// The SQL analysis dialect of the adapter the value renderer renders for, if it has one. - pub(crate) fn analysis_dialect(&self, py: Python<'_>) -> PyResult> { - native_field(self.value_renderer.bind(py), "sql_analysis_dialect_name")?.extract() - } - - /// One visible constant as SQL, its failures located at `file_path`. - pub(crate) fn render( - &self, - declaration: &Bound<'_, PyAny>, - file_path: &str, - ) -> PyResult { - let py: Python<'_> = declaration.py(); - constant_declaration_sql( - declaration, - self.value_renderer.bind(py), - (self.collection_rendering.bind(py), Some(file_path)), - ) - } -} - /// One enum member value as a SQL scalar literal. #[pyfunction] #[pyo3(signature = (*, value))] diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/mod.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/mod.rs new file mode 100644 index 0000000000..ab58ada601 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod authored_sql; +pub(crate) mod declaration_expansion; +pub(crate) mod declaration_indexes; +pub(crate) mod declaration_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs rename to crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs index b657179e1a..ce9fa04516 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs @@ -14,11 +14,11 @@ use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as sc use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; -use crate::bindings::_helpers::render::macro_arguments::parsed_arguments; -use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionClasses, NativeMacroExpansion, raised, -}; -use crate::bindings::_helpers::render::macro_expansion_facts::ResourceRef; +use crate::render::_helpers::calls::macro_arguments::parsed_arguments; +use crate::render::_helpers::expansion::macro_expansion::ExpansionClasses; +use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; +use crate::render::models::NativeMacroExpansion; use sqlbuild_python_base::records::main::fields::field::field as native_field; const CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs new file mode 100644 index 0000000000..9c23011592 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs @@ -0,0 +1,282 @@ +//! One authored string's macro expansion: scanned, memoised, evaluated and spliced natively. + +use crate::render::models::ExpansionOutput; +use crate::render::models::NativeMacroExpansion; +use sqlbuild_python_base::positional_record; + +use pyo3::FromPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_render::macro_calls::errors::ScanError; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; +use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; +use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; + +use crate::render::_helpers::calls::macro_call_classes::CallClass; +use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; +use crate::render::_helpers::calls::macro_call_memo::poisoned; +use crate::render::_helpers::expansion::macro_evaluation::CallText; +use crate::render::_helpers::expansion::macro_evaluation::Evaluation; +use crate::render::_helpers::expansion::macro_evaluation::evaluate_call; +use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; +use crate::render::_helpers::expansion::macro_generated_references::GeneratedReferenceClasses; +use crate::render::models::SpanTuple; +use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; + +pub(crate) const RELATION_PREFIX: &str = "__sqlbuild_relation_"; +const MEMO_ATTRIBUTE: &str = "memo"; +pub(crate) const LEXICAL_SYNTAX_ATTRIBUTE: &str = "_lexical_syntax"; +const OBSERVE_MODULES_METHOD: &str = "observe_modules"; +pub(crate) const RELATION_SUFFIX: &str = "__"; +pub(crate) const CONSTANT_KIND: &str = "constant"; +const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; +pub(crate) const SQL_LANGUAGE: &str = "sql"; +pub(crate) const DECLARATION_FORMS: [&str; 3] = ["const", "enum", "param"]; +pub(crate) const UNREAD_CALL_HELP: &str = "Close every quote, block comment and parenthesis in the macro \ + call, and follow its name with its opening parenthesis"; + +/// Why the scan could not read the call `@{called}`, if the scan stopped inside a call. +pub(crate) fn unread_call_detail(error: ScanError, called: &str) -> Option { + let reason: String = match error { + ScanError::UnclosedQuote => format!("a quoted string in {called} is not closed"), + ScanError::UnclosedComment => format!("a block comment in {called} is not closed"), + ScanError::UnclosedParenthesis => format!("{called} is not closed with ')'"), + ScanError::MissingParenthesis => { + format!("{called} is not followed by its opening parenthesis") + } + ScanError::BareCall | ScanError::SqlCall => return None, + }; + Some(format!("could not be read: {reason}")) +} + +/// How a scanned call is written: `@py.name`, or `@const` and the other declaration forms. +pub(crate) fn written_call(name: &str) -> String { + if DECLARATION_FORMS.contains(&name) { + format!("@{name}") + } else { + format!("@py.{name}") + } +} + +positional_record! { + into; + /// `(sql, spans, argument references, dependencies, usages)` of one expansion. + pub(crate) struct ExpansionRow { + pub(crate) sql: String, + pub(crate) spans: Vec, + pub(crate) argument_references: Vec, + pub(crate) dependencies: Vec>, + pub(crate) usages: Vec>, + } +} + +/// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ExpansionClasses<'py> { + pub(crate) compile_error: Bound<'py, PyAny>, + pub(crate) macro_argument_error: Bound<'py, PyAny>, + pub(crate) mismatch_error: Bound<'py, PyAny>, + pub(crate) declaration_identity: Bound<'py, PyAny>, + pub(crate) declaration_kind: Bound<'py, PyAny>, + pub(crate) macro_kind: Bound<'py, PyAny>, + pub(crate) usage_record: Bound<'py, PyAny>, + pub(crate) runtime_usage: Bound<'py, PyAny>, + pub(crate) resource_identity: Bound<'py, PyAny>, + pub(crate) model_kind: Bound<'py, PyAny>, + pub(crate) resource_ref: Bound<'py, PyAny>, + /// `SqlResourceRefKind` member by typed reference function, such as `__ref`. + pub(crate) ref_kinds: Bound<'py, PyDict>, + /// Typed reference function by `SqlResourceRefKind` value. + pub(crate) ref_functions: Bound<'py, PyDict>, + pub(crate) relation_renderer: Bound<'py, PyAny>, + /// Builds the `MacroContext` a macro is called with from the visible declarations. + pub(crate) invocation_context: Bound<'py, PyAny>, + /// What reference calls a macro returned are reported with. + pub(crate) generated: GeneratedReferenceClasses<'py>, + pub(crate) int_type: Bound<'py, PyAny>, + pub(crate) float_type: Bound<'py, PyAny>, +} + +/// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. +pub(crate) fn expanded<'py>( + expansion: &Bound<'py, NativeMacroExpansion>, + sql: &str, + bridge: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = expansion.py(); + let this: PyRef<'py, NativeMacroExpansion> = expansion.borrow(); + let classes: ExpansionClasses<'py> = this.classes.bind(py).extract()?; + let handle: Bound<'py, PyAny> = expansion.as_any().clone(); + let run: Evaluation<'_, 'py> = Evaluation { + expansion: &this, + handle: &handle, + classes: &classes, + }; + let text: CallText<'_> = CallText::new(sql); + let scan: MacroCallScan = scan_sites(text.text, this.syntax.as_ref()); + let memo: Option> = if bridge.is_none() { + None + } else { + Some(bridge.getattr(MEMO_ATTRIBUTE)?.downcast_into()?) + }; + let call_classes: Option> = match &memo { + Some(memo) => this.call_classes(&classes, memo.get(), &scan)?, + None => None, + }; + let outputs: Vec = match (call_classes, &memo) { + (Some(call_classes), Some(memo)) => { + let mut outputs: Vec = Vec::with_capacity(scan.sites.len()); + for (site, call_class) in scan.sites.iter().zip(call_classes) { + outputs.push(this.bridged_output( + &run, + (bridge, memo.get()), + (&text, site), + &call_class, + )?); + } + outputs + } + _ => this.unmemoised_outputs(&run, &text, &scan)?, + }; + let bounds: Vec<(usize, usize)> = scan + .sites + .iter() + .map(|site| (site.start, site.end)) + .collect(); + let (rendered, spans) = splice_calls(text.text, &bounds, &outputs) + .ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; + let facts = this.facts.borrow(); + Ok(ExpansionOutput { + sql: rendered, + spans, + argument_references: facts.argument_references.clone(), + dependencies: facts + .dependencies + .iter() + .map(|item| item.clone_ref(py)) + .collect(), + usages: facts.usages.iter().map(|item| item.clone_ref(py)).collect(), + }) +} + +/// One top-level call's SQL. +pub(crate) fn top_level_output( + run: &Evaluation<'_, '_>, + call: (&CallText<'_>, &MacroCallSite), +) -> PyResult { + evaluate_call(run, call, true)?.extract() +} + +/// A `SqlResourceRef`'s kind value and name. +pub(crate) fn resource_key(reference: &Bound<'_, PyAny>) -> PyResult { + Ok(( + native_field(&native_field(reference, "kind")?, "value")?.extract()?, + native_field(reference, "name")?.extract()?, + )) +} + +/// `reference_call_text`: the reference call naming `reference`, such as `__ref("orders")`. +pub(crate) fn reference_call_text( + classes: &ExpansionClasses<'_>, + reference: &ResourceRef, +) -> PyResult { + let (kind, name) = reference; + let function: String = classes + .ref_functions + .get_item(kind)? + .ok_or_else(|| PyErr::new::(kind.clone()))? + .extract()?; + Ok(format!("{function}(\"{name}\")")) +} + +/// Whether the consumer is a test or scenario, which records relationship visibility. +pub(crate) fn relationship_consumer( + classes: &ExpansionClasses<'_>, + consumer: &Bound<'_, PyAny>, +) -> PyResult { + if !consumer.is_instance(&classes.resource_identity)? { + return Ok(false); + } + let kind: String = native_field(&native_field(consumer, "kind")?, "value")?.extract()?; + Ok(RELATIONSHIP_CONSUMER_KINDS.contains(&kind.as_str())) +} + +/// A runtime `UsageRecord` of `declaration` by `consumer`, through a relationship if any. +pub(crate) fn usage<'py>( + classes: &ExpansionClasses<'py>, + usage: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + through: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let (consumer, declaration) = usage; + let keywords: Bound<'py, PyDict> = PyDict::new(consumer.py()); + if let Some(through) = through { + keywords.set_item("through", through)?; + } + classes.usage_record.call( + (consumer, declaration, &classes.runtime_usage), + Some(&keywords), + ) +} + +/// `", ".join(sorted(names)) or "none"`. +pub(crate) fn sorted_names(names: &Bound<'_, PyAny>) -> PyResult { + let mut sorted: Vec = + PyList::new(names.py(), names.try_iter()?.collect::>>()?)?.extract()?; + sorted.sort_unstable(); + Ok(if sorted.is_empty() { + "none".to_owned() + } else { + sorted.join(", ") + }) +} + +/// The `CompileInputError(message, code=code, help=help)` the classes' compile error builds. +pub(crate) fn coded(class: &Bound<'_, PyAny>, message: &str, (code, help): (&str, &str)) -> PyErr { + new_helped_error( + class.py(), + ErrorId::CompileInput, + (message.to_owned(), code), + Some(help), + ) +} + +/// The exception `class(message)` builds. +pub(crate) fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { + match class.call1((message,)) { + Ok(instance) => PyErr::from_value(instance), + Err(error) => error, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} + +/// Whether a macro's result calls any reference function. +pub(crate) fn result_has_markers(classes: &ExpansionClasses<'_>, result: &str) -> bool { + classes + .generated + .markers() + .any(|marker| result.contains(marker)) +} + +/// Observe modules a recorded call imported, so the store is not saved over changed code. +pub(crate) fn observe_new_modules(bridge: &Bound<'_, PyAny>, memo: &MacroCallMemo) -> PyResult<()> { + let Some(observed) = *memo.observed_modules.lock().map_err(poisoned)? else { + return Ok(()); + }; + let loaded: usize = PyModule::import(bridge.py(), "sys")? + .getattr("modules")? + .len()?; + if loaded != observed { + let _ = bridge.call_method0(OBSERVE_MODULES_METHOD)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_facts.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs rename to crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_facts.rs index 0748b6edf3..2e753aaf5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_facts.rs @@ -9,7 +9,7 @@ use sqlbuild_render::macro_calls::models::MacroCallEvent; pub(crate) type ResourceRef = (String, String); /// `MacroExpansionFacts`: dependencies, usages, relations, call-site references and the event log. -#[derive(Default)] +#[derive(Debug, Default)] pub(crate) struct ExpansionFacts { pub(crate) dependencies: Vec>, pub(crate) usages: Vec>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs rename to crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs index 5b9ee096d4..83221cb5ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs @@ -1,5 +1,6 @@ //! Expand one authored SQL string's macros into a `MacroExpansionResult`. +use crate::render::models::ExpansionRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -7,13 +8,15 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTu use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::discovery::guidance::explicit_references_help; -use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; -use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, StateInput, expanded, -}; -use crate::bindings::_helpers::sqltext::sql_references::extract_references; +use crate::discovery::main::failures::explicit_references_help::explicit_references_help; +use crate::render::_helpers::calls::macro_call_rows::span_row; +use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; +use crate::render::_helpers::expansion::macro_expansion::expanded; +use crate::render::models::ExpansionOutput; +use crate::render::models::NativeMacroExpansion; +use crate::render::models::StateInput; +use crate::render::types::SpanRow; +use crate::sqltext::main::extract_references::extract_references; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -47,18 +50,6 @@ const GENERATED_ALLOWED: &str = "macro-generated references"; static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); -/// One expansion's inputs, as Python's `expand_sql_macros_result` takes them. -pub(crate) struct ExpansionRequest<'py> { - pub(crate) sql: Bound<'py, PyString>, - pub(crate) file_path: Bound<'py, PyAny>, - pub(crate) loaded_macros: Bound<'py, PyDict>, - pub(crate) macro_overrides: Option>, - pub(crate) macro_context: Bound<'py, PyAny>, - pub(crate) declaration_resolver: Option>, - pub(crate) declarations: Option>, - pub(crate) consumer: Option>, -} - fn generated_classes(py: Python<'_>) -> PyResult> { let help: Bound<'_, PyAny> = PyString::new(py, &explicit_references_help(GENERATED_ALLOWED)).into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs rename to crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs index 720656612e..f8f58cfc9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs @@ -11,7 +11,7 @@ use sqlbuild_sqltext::sql_references::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs rename to crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/mod.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/mod.rs new file mode 100644 index 0000000000..f51bfd0ca3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod macro_evaluation; +pub(crate) mod macro_expansion; +pub(crate) mod macro_expansion_facts; +pub(crate) mod macro_expansion_result; +pub(crate) mod macro_generated_references; +pub(crate) mod macro_modules; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/mod.rs b/crates/sqlbuild-python-front/src/render/_helpers/mod.rs new file mode 100644 index 0000000000..39cd0dcc0f --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod calls; +pub(crate) mod declarations; +pub(crate) mod expansion; +pub(crate) mod sql_macros; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/mod.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/mod.rs new file mode 100644 index 0000000000..aeb781ae2e --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod sql_macro_host; +pub(crate) mod sql_macro_inventory; +pub(crate) mod sql_macro_mocks; +pub(crate) mod sql_macro_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs rename to crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs index afb0f41812..1d2445ca42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs @@ -1,5 +1,7 @@ //! SQL macro calls expanded in the declaration pass, reading the caller's visible declarations. +use crate::render::models::SqlCallOrigin; +use crate::render::models::SqlMacroSource; use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; @@ -26,10 +28,11 @@ use sqlbuild_analysis::sql_macros::models::{ use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, expand_declarations, usage_visibility, -}; -use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; +use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; +use crate::render::models::DeclarationExpansion; +use crate::render::models::ExpandedDeclarations; +use crate::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -51,35 +54,6 @@ const MODEL_KIND: &str = "model"; /// A macro the caller sees, and one it cannot see, by name. type LoadedMacro<'py> = (Option>, Option>); -/// Where the caller's text came from, to locate calls in its authored file. -#[derive(Clone, Copy)] -pub(crate) struct SqlCallOrigin<'a> { - /// The authored file's project-relative path. - pub(crate) relative_path: &'a str, - /// The authored file's contents. - pub(crate) contents: &'a str, - /// The authored query the expanded text came from, within `contents`. - pub(crate) query: &'a str, -} - -/// What SQL macro calls in one declaration pass read beyond the declaration context. -#[derive(Clone, Copy)] -pub(crate) struct SqlMacroSource<'a, 'py> { - /// Every loaded macro by name, for unscoped compiles whose context holds none. - pub(crate) loaded_macros: Option<&'a Bound<'py, PyAny>>, - pub(crate) origin: Option>, - /// Where macro bodies resolve their declarations; `None` outside the input loops. - pub(crate) scopes: Option<&'a dyn MacroScopes<'py>>, - /// The lexical rules the authored SQL is written in; `None` reads it under the generic rules. - pub(crate) syntax: Option<&'a LexicalSyntax>, -} - -/// The declaration contexts of macro files in a scoped compile. -pub(crate) trait MacroScopes<'py> { - /// The context the file at `path` sees, or `None` when the compile is unscoped. - fn file_context(&self, path: &str) -> PyResult>>; -} - /// The facts a declaration pass's SQL macro calls recorded. #[derive(Default)] pub(crate) struct SqlMacroFacts<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_inventory.rs rename to crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_mocks.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_mocks.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_mocks.rs rename to crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_mocks.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_tests.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_tests.rs rename to crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs diff --git a/crates/sqlbuild-python-front/src/render/main/calls/mod.rs b/crates/sqlbuild-python-front/src/render/main/calls/mod.rs new file mode 100644 index 0000000000..2447c45aa4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/mod.rs @@ -0,0 +1,6 @@ +pub mod register_macro_arguments; +pub mod register_macro_calls; +pub mod span_expansion; +pub mod span_row; +pub mod span_tuple; +pub mod top_level_macro_call_names; diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs new file mode 100644 index 0000000000..1f7be17726 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs @@ -0,0 +1,12 @@ +//! Register the macro arguments functions and classes. + +use crate::render::_helpers::calls::macro_arguments; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the macro arguments functions and classes. +#[inline] +pub fn register_macro_arguments(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_arguments::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs new file mode 100644 index 0000000000..d44a654e62 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs @@ -0,0 +1,12 @@ +//! Register the macro calls functions and classes. + +use crate::render::_helpers::calls::macro_calls; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the macro calls functions and classes. +#[inline] +pub fn register_macro_calls(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_calls::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs new file mode 100644 index 0000000000..e30ecf2a63 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs @@ -0,0 +1,11 @@ +//! A span as the expansion semantic validation maps offsets through. + +use crate::render::_helpers::calls::macro_call_rows; +use crate::render::types::SpanRow; +use sqlbuild_analysis::semantic_validation::models::Expansion; + +/// A span as the expansion semantic validation maps offsets through. +#[inline] +pub fn span_expansion(span: SpanRow) -> Expansion { + macro_call_rows::span_expansion(span) +} diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs new file mode 100644 index 0000000000..5a2e19132f --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs @@ -0,0 +1,11 @@ +//! Span row. + +use crate::render::_helpers::calls::macro_call_rows; +use crate::render::types::SpanRow; +use sqlbuild_render::macro_calls::models::MacroSpliceSpan; + +/// Span row. +#[inline] +pub fn span_row(span: MacroSpliceSpan) -> SpanRow { + macro_call_rows::span_row(span) +} diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs new file mode 100644 index 0000000000..d1c2173566 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs @@ -0,0 +1,11 @@ +//! A span as the tuple Python reads. + +use crate::render::_helpers::calls::macro_call_rows; +use crate::render::models::SpanTuple; +use crate::render::types::SpanRow; + +/// A span as the tuple Python reads. +#[inline] +pub fn span_tuple(span: &SpanRow) -> SpanTuple { + macro_call_rows::span_tuple(span) +} diff --git a/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs new file mode 100644 index 0000000000..f740688342 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs @@ -0,0 +1,18 @@ +//! Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. + +use crate::render::_helpers::calls::macro_calls; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. +#[inline] +pub fn top_level_macro_call_names<'py>( + py: Python<'py>, + sql: &str, + syntax: Option<&LexicalSyntax>, +) -> PyResult> { + macro_calls::top_level_macro_call_names(py, sql, syntax) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs new file mode 100644 index 0000000000..98c8ffe2cd --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs @@ -0,0 +1,20 @@ +//! Render one validated constant with the active adapter's typed-value contract. + +use crate::render::_helpers::declarations::declaration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Render one validated constant with the active adapter's typed-value contract. +#[inline] +pub fn constant_declaration_sql<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), +) -> PyResult { + declaration_values::constant_declaration_sql( + declaration, + value_renderer, + (collection_rendering, file_path), + ) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs new file mode 100644 index 0000000000..0f8d8e6cbd --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs @@ -0,0 +1,16 @@ +//! Pair the serializable static index with original process-local declaration values. + +use crate::render::_helpers::declarations::declaration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Pair the serializable static index with original process-local declaration values. +#[inline] +pub fn declaration_scope_resolver<'py>( + discovered: &Bound<'py, PyAny>, + scope_index: &Bound<'py, PyAny>, + (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + declaration_values::declaration_scope_resolver(discovered, scope_index, (loaded_macros, lookup)) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/expand_declarations.rs b/crates/sqlbuild-python-front/src/render/main/declarations/expand_declarations.rs new file mode 100644 index 0000000000..f0bfbf7df2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/expand_declarations.rs @@ -0,0 +1,14 @@ +//! Splice each reference in order, raising the attachment's error for the first bad one. + +use crate::render::_helpers::declarations::declaration_expansion; +use crate::render::models::DeclarationExpansion; +use crate::render::models::ExpandedDeclarations; +use pyo3::prelude::PyResult; + +/// Splice each reference in order, raising the attachment's error for the first bad one. +#[inline] +pub fn expand_declarations<'py>( + request: &DeclarationExpansion<'_, 'py>, +) -> PyResult> { + declaration_expansion::expand_declarations(request) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs new file mode 100644 index 0000000000..4b1bac8d3d --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs @@ -0,0 +1,18 @@ +//! `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. + +use crate::render::_helpers::declarations::authored_sql; +use crate::render::models::AuthoredExpansion; +use crate::render::models::SqlExpansionInputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. +#[inline] +pub fn expand_with_inputs( + inputs: &SqlExpansionInputs, + sql: &str, + file_path: &Bound<'_, PyAny>, +) -> PyResult { + authored_sql::expand_with_inputs(inputs, sql, file_path) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/mod.rs b/crates/sqlbuild-python-front/src/render/main/declarations/mod.rs new file mode 100644 index 0000000000..709e78359a --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/mod.rs @@ -0,0 +1,11 @@ +pub mod constant_declaration_sql; +pub mod declaration_scope_resolver; +pub mod expand_declarations; +pub mod expand_with_inputs; +pub mod public_declaration_indexes; +pub mod public_model_schema_index; +pub mod register_authored_sql; +pub mod register_declaration_indexes; +pub mod register_declaration_values; +pub mod render_enum_member_value; +pub mod span_rows; diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs new file mode 100644 index 0000000000..aada555fa0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs @@ -0,0 +1,15 @@ +//! Every public enum and constant by name, collision-checked. + +use crate::render::_helpers::declarations::declaration_indexes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Every public enum and constant by name, collision-checked. +#[inline] +pub fn public_declaration_indexes<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + declaration_indexes::public_declaration_indexes(inputs) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs new file mode 100644 index 0000000000..1c1d35991e --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs @@ -0,0 +1,13 @@ +//! Every public model schema by name, with its inherited columns first, collision-checked. + +use crate::render::_helpers::declarations::declaration_indexes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Every public model schema by name, with its inherited columns first, collision-checked. +#[inline] +pub fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { + declaration_indexes::public_model_schema_index(inputs) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs new file mode 100644 index 0000000000..1a77c6da6c --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs @@ -0,0 +1,12 @@ +//! Register the authored sql functions and classes. + +use crate::render::_helpers::declarations::authored_sql; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the authored sql functions and classes. +#[inline] +pub fn register_authored_sql(module: &Bound<'_, PyModule>) -> PyResult<()> { + authored_sql::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs new file mode 100644 index 0000000000..dab0eed265 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs @@ -0,0 +1,12 @@ +//! Register the declaration indexes functions and classes. + +use crate::render::_helpers::declarations::declaration_indexes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the declaration indexes functions and classes. +#[inline] +pub fn register_declaration_indexes(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_indexes::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs new file mode 100644 index 0000000000..92d91956ef --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs @@ -0,0 +1,12 @@ +//! Register the declaration values functions and classes. + +use crate::render::_helpers::declarations::declaration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the declaration values functions and classes. +#[inline] +pub fn register_declaration_values(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_values::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs new file mode 100644 index 0000000000..18b3148a1e --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs @@ -0,0 +1,12 @@ +//! One enum member value as a SQL scalar literal. + +use crate::render::_helpers::declarations::declaration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One enum member value as a SQL scalar literal. +#[inline] +pub fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { + declaration_values::render_enum_member_value(value) +} diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs new file mode 100644 index 0000000000..69f717b456 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs @@ -0,0 +1,11 @@ +//! Span rows. + +use crate::render::_helpers::declarations::declaration_expansion; +use crate::render::types::SpanRow; +use sqlbuild_analysis::compiled_project::models::SplicedSql; + +/// Span rows. +#[inline] +pub fn span_rows(spliced: &SplicedSql) -> Vec { + declaration_expansion::span_rows(spliced) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs new file mode 100644 index 0000000000..f3493f3740 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs @@ -0,0 +1,18 @@ +//! Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. + +use crate::render::_helpers::expansion::macro_expansion; +use crate::render::models::ExpansionOutput; +use crate::render::models::NativeMacroExpansion; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. +#[inline] +pub fn expanded<'py>( + expansion: &Bound<'py, NativeMacroExpansion>, + sql: &str, + bridge: &Bound<'py, PyAny>, +) -> PyResult { + macro_expansion::expanded(expansion, sql, bridge) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs new file mode 100644 index 0000000000..3cc313db31 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs @@ -0,0 +1,13 @@ +//! Expansion result. + +use crate::render::_helpers::expansion::macro_expansion_result; +use crate::render::models::ExpansionRequest; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Expansion result. +#[inline] +pub fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { + macro_expansion_result::expansion_result(request) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs new file mode 100644 index 0000000000..ceebea1865 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs @@ -0,0 +1,13 @@ +//! The classes, enums and helpers the native expansion builds facts and errors from, built once. + +use crate::render::_helpers::expansion::macro_expansion_result; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// The classes, enums and helpers the native expansion builds facts and errors from, built once. +#[inline] +pub fn macro_expansion_classes(py: Python<'_>) -> PyResult> { + macro_expansion_result::macro_expansion_classes(py) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/mod.rs b/crates/sqlbuild-python-front/src/render/main/expansion/mod.rs new file mode 100644 index 0000000000..e4c870bc17 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/mod.rs @@ -0,0 +1,6 @@ +pub mod expanded; +pub mod expansion_result; +pub mod macro_expansion_classes; +pub mod register_macro_expansion; +pub mod register_macro_expansion_result; +pub mod register_macro_modules; diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs new file mode 100644 index 0000000000..14a84995b5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs @@ -0,0 +1,12 @@ +//! Register the macro expansion functions and classes. + +use crate::render::_helpers::expansion::macro_expansion; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the macro expansion functions and classes. +#[inline] +pub fn register_macro_expansion(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_expansion::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs new file mode 100644 index 0000000000..bd34acc97d --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs @@ -0,0 +1,12 @@ +//! Register the macro expansion result functions and classes. + +use crate::render::_helpers::expansion::macro_expansion_result; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the macro expansion result functions and classes. +#[inline] +pub fn register_macro_expansion_result(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_expansion_result::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs new file mode 100644 index 0000000000..abe97ff144 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs @@ -0,0 +1,12 @@ +//! Register the macro modules functions and classes. + +use crate::render::_helpers::expansion::macro_modules; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the macro modules functions and classes. +#[inline] +pub fn register_macro_modules(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_modules::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/mod.rs b/crates/sqlbuild-python-front/src/render/main/mod.rs new file mode 100644 index 0000000000..0a859610d2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/mod.rs @@ -0,0 +1,4 @@ +pub mod calls; +pub mod declarations; +pub mod expansion; +pub mod sql_macros; diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/mod.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/mod.rs new file mode 100644 index 0000000000..93d5f7aeb5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/mod.rs @@ -0,0 +1,3 @@ +pub mod register_sql_macro_inventory; +pub mod with_sql_macro_mocks; +pub mod without_with_list_calls; diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs new file mode 100644 index 0000000000..2242424725 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs @@ -0,0 +1,12 @@ +//! Register the sql macro inventory functions and classes. + +use crate::render::_helpers::sql_macros::sql_macro_inventory; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sql macro inventory functions and classes. +#[inline] +pub fn register_sql_macro_inventory(module: &Bound<'_, PyModule>) -> PyResult<()> { + sql_macro_inventory::register(module) +} diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs new file mode 100644 index 0000000000..951d255eb0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs @@ -0,0 +1,17 @@ +//! `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. + +use crate::render::_helpers::sql_macros::sql_macro_mocks; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. +#[inline] +pub fn with_sql_macro_mocks( + model_input: &Bound<'_, PyAny>, + macro_source_sql: String, + (mocks, loaded_macros): (&Bound<'_, PyDict>, &Bound<'_, PyAny>), +) -> PyResult { + sql_macro_mocks::with_sql_macro_mocks(model_input, macro_source_sql, (mocks, loaded_macros)) +} diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs new file mode 100644 index 0000000000..4f4d902195 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs @@ -0,0 +1,9 @@ +//! The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. + +use crate::render::_helpers::sql_macros::sql_macro_tests; + +/// The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. +#[inline] +pub fn without_with_list_calls(sql: &str) -> (String, Vec) { + sql_macro_tests::without_with_list_calls(sql) +} diff --git a/crates/sqlbuild-python-front/src/render/mod.rs b/crates/sqlbuild-python-front/src/render/mod.rs new file mode 100644 index 0000000000..04e542ab75 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/mod.rs @@ -0,0 +1,6 @@ +//! SQL rendering: authored SQL, declaration expansion, macro calls and expansion, and SQL macros. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python-front/src/render/models.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs rename to crates/sqlbuild-python-front/src/render/models.rs index 2a38d11899..a350fc00b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/models.rs @@ -1,151 +1,312 @@ -//! One authored string's macro expansion: scanned, memoised, evaluated and spliced natively. - -use sqlbuild_python_base::positional_record; -use std::cell::RefCell; - -use pyo3::exceptions::{PyIndexError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; -use pyo3::{FromPyObject, pyclass, pymethods}; -use sqlbuild_analysis::sql_macros::constants::{BARE_MACRO_CALL_CODE, MIXED_KIND_CODE}; +//! Public records of render. + +use crate::render::_helpers::calls::macro_call_classes::CallClass; +use crate::render::_helpers::calls::macro_call_classes::ClassInputs; +use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; +use crate::render::_helpers::calls::macro_call_memo::poisoned; +use crate::render::_helpers::calls::macro_call_rows::span_row; +use crate::render::_helpers::calls::macro_call_rows::span_tuple; +use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; +use crate::render::_helpers::declarations::declaration_values::constant_declaration_sql; +use crate::render::_helpers::expansion::macro_evaluation::CallText; +use crate::render::_helpers::expansion::macro_evaluation::Evaluation; +use crate::render::_helpers::expansion::macro_evaluation::macro_argument_error; +use crate::render::_helpers::expansion::macro_expansion::CONSTANT_KIND; +use crate::render::_helpers::expansion::macro_expansion::DECLARATION_FORMS; +use crate::render::_helpers::expansion::macro_expansion::ExpansionClasses; +use crate::render::_helpers::expansion::macro_expansion::ExpansionRow; +use crate::render::_helpers::expansion::macro_expansion::LEXICAL_SYNTAX_ATTRIBUTE; +use crate::render::_helpers::expansion::macro_expansion::RELATION_PREFIX; +use crate::render::_helpers::expansion::macro_expansion::RELATION_SUFFIX; +use crate::render::_helpers::expansion::macro_expansion::SQL_LANGUAGE; +use crate::render::_helpers::expansion::macro_expansion::UNREAD_CALL_HELP; +use crate::render::_helpers::expansion::macro_expansion::coded; +use crate::render::_helpers::expansion::macro_expansion::expanded; +use crate::render::_helpers::expansion::macro_expansion::observe_new_modules; +use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::expansion::macro_expansion::reference_call_text; +use crate::render::_helpers::expansion::macro_expansion::relationship_consumer; +use crate::render::_helpers::expansion::macro_expansion::resource_key; +use crate::render::_helpers::expansion::macro_expansion::result_has_markers; +use crate::render::_helpers::expansion::macro_expansion::sorted_names; +use crate::render::_helpers::expansion::macro_expansion::top_level_output; +use crate::render::_helpers::expansion::macro_expansion::unread_call_detail; +use crate::render::_helpers::expansion::macro_expansion::written_call; +use crate::render::_helpers::expansion::macro_expansion_facts::ExpansionFacts; +use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; +use crate::render::_helpers::expansion::macro_generated_references::GeneratedOrigin; +use crate::render::_helpers::expansion::macro_generated_references::report_generated_references; +use crate::render::types::MacroScopes; +use crate::render::types::SpanRow; +use crate::render::types::StateTuple; +use crate::sqltext::main::optional_record_syntax::optional_record_syntax; +use crate::sqltext::models::ScannedReference; +use pyo3::FromPyObject; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use pyo3::types::PyTuple; +use sqlbuild_analysis::compiled_project::models::SplicedSql; +use sqlbuild_analysis::sql_macros::constants::BARE_MACRO_CALL_CODE; +use sqlbuild_analysis::sql_macros::constants::MIXED_KIND_CODE; +use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::field_text::field_text; +use sqlbuild_python_base::records::types::RecordId; use sqlbuild_render::macro_calls::errors::ScanError; -use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; -use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; -use sqlbuild_render::macro_calls::models::{ - MacroCallEntry, MacroCallEvent, MacroCallScan, MacroCallSite, MacroScanFailure, MacroSpliceSpan, -}; +use sqlbuild_render::macro_calls::models::MacroCallEntry; +use sqlbuild_render::macro_calls::models::MacroCallEvent; +use sqlbuild_render::macro_calls::models::MacroCallScan; +use sqlbuild_render::macro_calls::models::MacroCallSite; +use sqlbuild_render::macro_calls::models::MacroScanFailure; +use sqlbuild_render::macro_calls::models::MacroSpliceSpan; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::cell::RefCell; -use crate::bindings::_helpers::render::declaration_expansion::usage_visibility; -use crate::bindings::_helpers::render::macro_call_classes::{CallClass, ClassInputs}; -use crate::bindings::_helpers::render::macro_call_memo::{MacroCallMemo, poisoned}; -use crate::bindings::_helpers::render::macro_call_rows::{SpanTuple, span_row, span_tuple}; -use crate::bindings::_helpers::render::macro_evaluation::{ - CallText, Evaluation, evaluate_call, macro_argument_error, -}; -use crate::bindings::_helpers::render::macro_expansion_facts::{ExpansionFacts, ResourceRef}; -use crate::bindings::_helpers::render::macro_generated_references::{ - GeneratedOrigin, GeneratedReferenceClasses, report_generated_references, -}; -use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; -use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::field_text::field_text; -use sqlbuild_python_base::records::types::ErrorId; - -const RELATION_PREFIX: &str = "__sqlbuild_relation_"; -const MEMO_ATTRIBUTE: &str = "memo"; -const LEXICAL_SYNTAX_ATTRIBUTE: &str = "_lexical_syntax"; -const OBSERVE_MODULES_METHOD: &str = "observe_modules"; -const RELATION_SUFFIX: &str = "__"; -const CONSTANT_KIND: &str = "constant"; -const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; -const SQL_LANGUAGE: &str = "sql"; -const DECLARATION_FORMS: [&str; 3] = ["const", "enum", "param"]; -const UNREAD_CALL_HELP: &str = "Close every quote, block comment and parenthesis in the macro \ - call, and follow its name with its opening parenthesis"; - -/// Why the scan could not read the call `@{called}`, if the scan stopped inside a call. -fn unread_call_detail(error: ScanError, called: &str) -> Option { - let reason: String = match error { - ScanError::UnclosedQuote => format!("a quoted string in {called} is not closed"), - ScanError::UnclosedComment => format!("a block comment in {called} is not closed"), - ScanError::UnclosedParenthesis => format!("{called} is not closed with ')'"), - ScanError::MissingParenthesis => { - format!("{called} is not followed by its opening parenthesis") - } - ScanError::BareCall | ScanError::SqlCall => return None, - }; - Some(format!("could not be read: {reason}")) +/// Authored SQL expanded through vars, declarations and macros, with each pass's spans. +#[derive(Debug)] +pub struct AuthoredExpansion { + pub sql: String, + /// Interpolation, declaration and macro spans, in pass order. + pub passes: [Vec; 3], } -/// How a scanned call is written: `@py.name`, or `@const` and the other declaration forms. -fn written_call(name: &str) -> String { - if DECLARATION_FORMS.contains(&name) { - format!("@{name}") - } else { - format!("@py.{name}") - } +/// The inputs authored SQL is expanded with outside a compile; the handles carry user values. +#[derive(Debug)] +pub struct SqlExpansionInputs { + pub effective_vars: Py, + pub loaded_macros: Py, + pub macro_context: Py, + pub enums: Py, + pub constants: Py, + pub value_renderer: Py, + pub collection_rendering: Py, + /// Each model file's own declarations, by file path. + pub local_declarations: Py, + pub declaration_resolver: Py, } -positional_record! { - into; - /// `(sql, spans, argument references, dependencies, usages)` of one expansion. - struct ExpansionRow { - sql: String, - spans: Vec, - argument_references: Vec, - dependencies: Vec>, - usages: Vec>, +impl SqlExpansionInputs { + /// The inputs a `SqlExpansionContext` record holds. + pub fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + let field = + |name: &str| -> PyResult> { Ok(native_field(record, name)?.unbind()) }; + Ok(Self { + effective_vars: field("effective_vars")?, + loaded_macros: field("loaded_macros")?, + macro_context: field("macro_context")?, + enums: field("enums")?, + constants: field("constants")?, + value_renderer: field("value_renderer")?, + collection_rendering: field("collection_rendering")?, + local_declarations: native_field(record, "local_declarations")? + .downcast_into::()? + .unbind(), + declaration_resolver: field("declaration_resolver")?, + }) + } + + /// These inputs as a `SqlExpansionContext` record. + pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { + new_record_with( + py, + RecordId::SqlExpansionContext, + &[ + ("effective_vars", self.effective_vars.bind(py).clone()), + ("loaded_macros", self.loaded_macros.bind(py).clone()), + ("macro_context", self.macro_context.bind(py).clone()), + ("enums", self.enums.bind(py).clone()), + ("constants", self.constants.bind(py).clone()), + ("value_renderer", self.value_renderer.bind(py).clone()), + ( + "collection_rendering", + self.collection_rendering.bind(py).clone(), + ), + ( + "local_declarations", + self.local_declarations.bind(py).clone().into_any(), + ), + ( + "declaration_resolver", + self.declaration_resolver.bind(py).clone(), + ), + ], + ) } } -/// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. -#[derive(FromPyObject)] +/// The Python classes expansion builds usages and errors from. +#[derive(Debug, FromPyObject)] #[pyo3(from_item_all)] -pub(crate) struct ExpansionClasses<'py> { - pub(crate) compile_error: Bound<'py, PyAny>, - pub(crate) macro_argument_error: Bound<'py, PyAny>, - mismatch_error: Bound<'py, PyAny>, - declaration_identity: Bound<'py, PyAny>, - declaration_kind: Bound<'py, PyAny>, - macro_kind: Bound<'py, PyAny>, - usage_record: Bound<'py, PyAny>, - runtime_usage: Bound<'py, PyAny>, - resource_identity: Bound<'py, PyAny>, - model_kind: Bound<'py, PyAny>, - pub(crate) resource_ref: Bound<'py, PyAny>, - /// `SqlResourceRefKind` member by typed reference function, such as `__ref`. - pub(crate) ref_kinds: Bound<'py, PyDict>, - /// Typed reference function by `SqlResourceRefKind` value. - ref_functions: Bound<'py, PyDict>, - pub(crate) relation_renderer: Bound<'py, PyAny>, - /// Builds the `MacroContext` a macro is called with from the visible declarations. - pub(crate) invocation_context: Bound<'py, PyAny>, - /// What reference calls a macro returned are reported with. - generated: GeneratedReferenceClasses<'py>, - pub(crate) int_type: Bound<'py, PyAny>, - pub(crate) float_type: Bound<'py, PyAny>, +pub struct DeclarationExpansionClasses<'py> { + pub usage_record: Bound<'py, PyAny>, + pub compile_error: Bound<'py, PyAny>, +} + +/// Who reads the declarations, whether it prefers relationship visibility, and its error path. +#[derive(Debug, Clone)] +pub struct ExpansionConsumer<'a, 'py> { + pub identity: Option>, + pub relationship_visibility: bool, + pub path: &'a str, +} + +/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant rendering. +#[derive(Debug)] +pub struct DeclarationExpansion<'a, 'py> { + pub sql: &'a str, + pub references: &'a [ScannedReference], + pub stop: Option, + pub context: &'a Bound<'py, PyAny>, + pub consumer: ExpansionConsumer<'a, 'py>, + pub rendering: &'a ConstantRendering, + pub classes: &'a DeclarationExpansionClasses<'py>, + /// What SQL macro calls read beyond the context; `None` where SQL macros are not expanded. + pub sql_macros: Option>, +} + +/// The expanded SQL with its spans, and the crate::render::_helpers::declarations::declaration_expansion::usage records it adds in reference order. +#[derive(Debug)] +pub struct ExpandedDeclarations<'py> { + pub spliced: SplicedSql, + pub usages: Vec>, + /// The SQL macros the SQL called, top-level and nested, once each. + pub sql_macros: Vec, + /// What the analysed SQL must show for each top-level call's arguments to fit. + pub sql_macro_checks: Vec, + /// Each expression macro call in the spliced SQL, by byte range. + pub typed_calls: Vec, +} + +/// The adapter's constant rendering a compile's declaration expansions share. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct ConstantRendering { + pub(crate) value_renderer: Py, + pub(crate) collection_rendering: Py, +} + +impl ConstantRendering { + /// Rendering with the adapter's value renderer and the project's collection rendering. + pub fn new(value_renderer: Py, collection_rendering: Py) -> Self { + Self { + value_renderer, + collection_rendering, + } + } + + /// The adapter name the value renderer renders for, empty when it names none. + pub fn adapter_name(&self, py: Python<'_>) -> String { + native_field(self.value_renderer.bind(py), "adapter_name") + .and_then(|name| name.extract::()) + .unwrap_or_default() + } + + /// The SQL analysis dialect of the adapter the value renderer renders for, if it has one. + pub fn analysis_dialect(&self, py: Python<'_>) -> PyResult> { + native_field(self.value_renderer.bind(py), "sql_analysis_dialect_name")?.extract() + } + + /// One visible constant as SQL, its failures located at `file_path`. + pub fn render(&self, declaration: &Bound<'_, PyAny>, file_path: &str) -> PyResult { + let py: Python<'_> = declaration.py(); + constant_declaration_sql( + declaration, + self.value_renderer.bind(py), + (self.collection_rendering.bind(py), Some(file_path)), + ) + } +} + +/// A span as the tuple Python reads: source start and end, output start and end. +#[derive(Debug)] +pub struct SpanTuple { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + +impl<'py> pyo3::IntoPyObject<'py> for SpanTuple { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.source_start, + self.source_end, + self.output_start, + self.output_end, + ), + py, + ) + } } /// One authored string's expansion inputs and the facts it records. #[pyclass(module = "sqlbuild._native", unsendable)] -pub(crate) struct NativeMacroExpansion { - pub(crate) file_path: Py, - pub(crate) file_label: String, - pub(crate) loaded_macros: Py, - pub(crate) overrides: Py, - pub(crate) macro_context: Py, +#[derive(Debug)] +pub struct NativeMacroExpansion { + pub file_path: Py, + pub file_label: String, + pub loaded_macros: Py, + pub overrides: Py, + pub macro_context: Py, /// The visible `DeclarationResolutionContext`, or `None`. - pub(crate) declarations: Py, + pub declarations: Py, /// The consumer usages are recorded for, or `None`. - pub(crate) consumer: Py, - enforce_explicit: bool, - classes: Py, + pub consumer: Py, + pub(crate) enforce_explicit: bool, + pub(crate) classes: Py, pub(crate) facts: RefCell, /// The lexical rules the authored SQL is written in, from the macro context, if it has them. - pub(crate) syntax: Option, + pub syntax: Option, } -positional_record! { - extract; - /// The `(macros, overrides, context, declarations, consumer)` one expansion reads. - pub(crate) struct StateInput { - pub(crate) loaded_macros: Py, - pub(crate) overrides: Py, - pub(crate) macro_context: Py, - pub(crate) declarations: Py, - pub(crate) consumer: Py, +/// The `(macros, overrides, context, declarations, consumer)` one expansion reads. +#[derive(Debug)] +pub struct StateInput { + pub loaded_macros: Py, + pub overrides: Py, + pub macro_context: Py, + pub declarations: Py, + pub consumer: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for StateInput { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (loaded_macros, overrides, macro_context, declarations, consumer): StateTuple = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + loaded_macros, + overrides, + macro_context, + declarations, + consumer, + }) } } #[pymethods] impl NativeMacroExpansion { #[new] - pub(crate) fn new( + pub fn new( file_path: &Bound<'_, PyAny>, state: StateInput, enforce_explicit: bool, @@ -180,7 +341,7 @@ impl NativeMacroExpansion { } /// Expand `sql`, through the bridge's memo when one is active. - fn expand<'py>( + pub(crate) fn expand<'py>( slf: &Bound<'py, Self>, sql: &Bound<'py, PyString>, bridge: &Bound<'py, PyAny>, @@ -200,7 +361,7 @@ impl NativeMacroExpansion { } /// `render_relation`: an approved call-site reference's placeholder, else its reference call. - fn render_relation(&self, reference: &Bound<'_, PyAny>) -> PyResult { + pub(crate) fn render_relation(&self, reference: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = reference.py(); let classes: ExpansionClasses<'_> = self.classes.bind(py).extract()?; if !reference.is_instance(&classes.resource_ref)? { @@ -218,20 +379,20 @@ impl NativeMacroExpansion { } /// `_record_macro_declaration_usage`: a macro read a constant or enum through its context. - fn declaration_read(&self, kind: &str, name: &str, py: Python<'_>) -> PyResult<()> { + pub(crate) fn declaration_read(&self, kind: &str, name: &str, py: Python<'_>) -> PyResult<()> { let classes: ExpansionClasses<'_> = self.classes.bind(py).extract()?; self.record_declaration_read(&classes, kind, name) } /// Mark the executing call as dependent on its consumer, so it is never replayed. - fn block_memo(&self) { + pub(crate) fn block_memo(&self) { self.facts.borrow_mut().block_memo(); } } impl NativeMacroExpansion { /// Each site's call class, or `None` when the scan failed or a tree name is unresolvable. - fn call_classes<'py>( + pub(crate) fn call_classes<'py>( &self, classes: &ExpansionClasses<'py>, memo: &MacroCallMemo, @@ -261,7 +422,7 @@ impl NativeMacroExpansion { } /// `_bridged_call_output`: a recorded call replayed, or the call evaluated and recorded. - fn bridged_output<'py>( + pub(crate) fn bridged_output<'py>( &self, run: &Evaluation<'_, 'py>, bridge: (&Bound<'py, PyAny>, &MacroCallMemo), @@ -324,7 +485,7 @@ impl NativeMacroExpansion { } /// `_unmemoised_sql_macros`: every call evaluated in order, then the scan's failure raised. - fn unmemoised_outputs( + pub(crate) fn unmemoised_outputs( &self, run: &Evaluation<'_, '_>, text: &CallText<'_>, @@ -340,7 +501,7 @@ impl NativeMacroExpansion { Ok(outputs) } - fn evaluated_output( + pub(crate) fn evaluated_output( &self, run: &Evaluation<'_, '_>, call: (&CallText<'_>, &MacroCallSite), @@ -350,7 +511,7 @@ impl NativeMacroExpansion { } /// Whether a test overrides any macro in the call's tree. - fn mocked(&self, site: &MacroCallSite, py: Python<'_>) -> PyResult { + pub(crate) fn mocked(&self, site: &MacroCallSite, py: Python<'_>) -> PyResult { let overrides: &Bound<'_, PyDict> = self.overrides.bind(py); for name in site.tree_names.iter().flatten() { if overrides.contains(name)? { @@ -478,7 +639,7 @@ impl NativeMacroExpansion { Err(raised(&classes.compile_error, &message)) } - /// `_record_macro_use`: the macro's dependency and its runtime usage by the consumer. + /// `_record_macro_use`: the macro's dependency and its runtime crate::render::_helpers::expansion::macro_expansion::usage by the consumer. pub(crate) fn record_macro_use<'py>( &self, classes: &ExpansionClasses<'py>, @@ -508,7 +669,7 @@ impl NativeMacroExpansion { let mut usages: Vec> = Vec::new(); for visible in usage_visibility(&visibility, relationship_consumer(classes, consumer)?)? { - usages.push(usage( + usages.push(crate::render::_helpers::expansion::macro_expansion::usage( classes, (consumer, identity), Some(&native_field(&visible, "through")?), @@ -516,7 +677,11 @@ impl NativeMacroExpansion { } usages } else { - vec![usage(classes, (consumer, identity), None)?] + vec![crate::render::_helpers::expansion::macro_expansion::usage( + classes, + (consumer, identity), + None, + )?] }; self.facts .borrow_mut() @@ -526,7 +691,7 @@ impl NativeMacroExpansion { } /// `_record_macro_declaration_usage` for a constant or enum the macro read. - fn record_declaration_read( + pub(crate) fn record_declaration_read( &self, classes: &ExpansionClasses<'_>, kind: &str, @@ -562,7 +727,7 @@ impl NativeMacroExpansion { for visible in usage_visibility(&visibility, relationship_consumer(classes, &consumer)?)? { - usages.push(usage( + usages.push(crate::render::_helpers::expansion::macro_expansion::usage( classes, (&consumer, &native_field(&visible, "declaration")?), Some(&native_field(&visible, "through")?), @@ -587,7 +752,11 @@ impl NativeMacroExpansion { fields.set_item("owner", owner)?; let identity: Bound<'_, PyAny> = classes.declaration_identity.call((), Some(&fields))?; - usages.push(usage(classes, (&consumer, &identity), None)?); + usages.push(crate::render::_helpers::expansion::macro_expansion::usage( + classes, + (&consumer, &identity), + None, + )?); } self.facts .borrow_mut() @@ -622,7 +791,11 @@ impl NativeMacroExpansion { } /// `_replay_macro_call_event`: one recorded fact, recorded again for this consumer. - fn replay(&self, classes: &ExpansionClasses<'_>, event: MacroCallEvent) -> PyResult<()> { + pub(crate) fn replay( + &self, + classes: &ExpansionClasses<'_>, + event: MacroCallEvent, + ) -> PyResult<()> { let py: Python<'_> = classes.compile_error.py(); match event { MacroCallEvent::MacroUse { name } => { @@ -681,7 +854,11 @@ impl NativeMacroExpansion { } /// `render_relation_placeholders`: each placeholder as the reference call it stands for. - fn render_placeholders(&self, classes: &ExpansionClasses<'_>, sql: &str) -> PyResult { + pub(crate) fn render_placeholders( + &self, + classes: &ExpansionClasses<'_>, + sql: &str, + ) -> PyResult { let relations: Vec = self.facts.borrow().relations.clone(); if relations.is_empty() || !sql.contains(RELATION_PREFIX) { return Ok(sql.to_owned()); @@ -718,188 +895,55 @@ impl NativeMacroExpansion { } /// One expansion's SQL, substitution spans, call-site references, dependencies and usages. -pub(crate) struct ExpansionOutput { - pub(crate) sql: String, - pub(crate) spans: Vec, - pub(crate) argument_references: Vec, - pub(crate) dependencies: Vec>, - pub(crate) usages: Vec>, +#[derive(Debug)] +pub struct ExpansionOutput { + pub sql: String, + pub spans: Vec, + pub argument_references: Vec, + pub dependencies: Vec>, + pub usages: Vec>, } -/// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. -pub(crate) fn expanded<'py>( - expansion: &Bound<'py, NativeMacroExpansion>, - sql: &str, - bridge: &Bound<'py, PyAny>, -) -> PyResult { - let py: Python<'py> = expansion.py(); - let this: PyRef<'py, NativeMacroExpansion> = expansion.borrow(); - let classes: ExpansionClasses<'py> = this.classes.bind(py).extract()?; - let handle: Bound<'py, PyAny> = expansion.as_any().clone(); - let run: Evaluation<'_, 'py> = Evaluation { - expansion: &this, - handle: &handle, - classes: &classes, - }; - let text: CallText<'_> = CallText::new(sql); - let scan: MacroCallScan = scan_sites(text.text, this.syntax.as_ref()); - let memo: Option> = if bridge.is_none() { - None - } else { - Some(bridge.getattr(MEMO_ATTRIBUTE)?.downcast_into()?) - }; - let call_classes: Option> = match &memo { - Some(memo) => this.call_classes(&classes, memo.get(), &scan)?, - None => None, - }; - let outputs: Vec = match (call_classes, &memo) { - (Some(call_classes), Some(memo)) => { - let mut outputs: Vec = Vec::with_capacity(scan.sites.len()); - for (site, call_class) in scan.sites.iter().zip(call_classes) { - outputs.push(this.bridged_output( - &run, - (bridge, memo.get()), - (&text, site), - &call_class, - )?); - } - outputs - } - _ => this.unmemoised_outputs(&run, &text, &scan)?, - }; - let bounds: Vec<(usize, usize)> = scan - .sites - .iter() - .map(|site| (site.start, site.end)) - .collect(); - let (rendered, spans) = splice_calls(text.text, &bounds, &outputs) - .ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; - let facts = this.facts.borrow(); - Ok(ExpansionOutput { - sql: rendered, - spans, - argument_references: facts.argument_references.clone(), - dependencies: facts - .dependencies - .iter() - .map(|item| item.clone_ref(py)) - .collect(), - usages: facts.usages.iter().map(|item| item.clone_ref(py)).collect(), - }) -} - -/// One top-level call's SQL. -fn top_level_output( - run: &Evaluation<'_, '_>, - call: (&CallText<'_>, &MacroCallSite), -) -> PyResult { - evaluate_call(run, call, true)?.extract() -} - -/// A `SqlResourceRef`'s kind value and name. -pub(crate) fn resource_key(reference: &Bound<'_, PyAny>) -> PyResult { - Ok(( - native_field(&native_field(reference, "kind")?, "value")?.extract()?, - native_field(reference, "name")?.extract()?, - )) -} - -/// `reference_call_text`: the reference call naming `reference`, such as `__ref("orders")`. -fn reference_call_text( - classes: &ExpansionClasses<'_>, - reference: &ResourceRef, -) -> PyResult { - let (kind, name) = reference; - let function: String = classes - .ref_functions - .get_item(kind)? - .ok_or_else(|| PyErr::new::(kind.clone()))? - .extract()?; - Ok(format!("{function}(\"{name}\")")) -} - -/// Whether the consumer is a test or scenario, which records relationship visibility. -fn relationship_consumer( - classes: &ExpansionClasses<'_>, - consumer: &Bound<'_, PyAny>, -) -> PyResult { - if !consumer.is_instance(&classes.resource_identity)? { - return Ok(false); - } - let kind: String = native_field(&native_field(consumer, "kind")?, "value")?.extract()?; - Ok(RELATIONSHIP_CONSUMER_KINDS.contains(&kind.as_str())) -} - -/// A runtime `UsageRecord` of `declaration` by `consumer`, through a relationship if any. -fn usage<'py>( - classes: &ExpansionClasses<'py>, - usage: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - through: Option<&Bound<'py, PyAny>>, -) -> PyResult> { - let (consumer, declaration) = usage; - let keywords: Bound<'py, PyDict> = PyDict::new(consumer.py()); - if let Some(through) = through { - keywords.set_item("through", through)?; - } - classes.usage_record.call( - (consumer, declaration, &classes.runtime_usage), - Some(&keywords), - ) -} - -/// `", ".join(sorted(names)) or "none"`. -fn sorted_names(names: &Bound<'_, PyAny>) -> PyResult { - let mut sorted: Vec = - PyList::new(names.py(), names.try_iter()?.collect::>>()?)?.extract()?; - sorted.sort_unstable(); - Ok(if sorted.is_empty() { - "none".to_owned() - } else { - sorted.join(", ") - }) -} - -/// The `CompileInputError(message, code=code, help=help)` the classes' compile error builds. -fn coded(class: &Bound<'_, PyAny>, message: &str, (code, help): (&str, &str)) -> PyErr { - new_helped_error( - class.py(), - ErrorId::CompileInput, - (message.to_owned(), code), - Some(help), - ) -} - -/// The exception `class(message)` builds. -pub(crate) fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { - match class.call1((message,)) { - Ok(instance) => PyErr::from_value(instance), - Err(error) => error, - } +/// One expansion's inputs, as Python's `expand_sql_macros_result` takes them. +#[derive(Debug)] +pub struct ExpansionRequest<'py> { + pub sql: Bound<'py, PyString>, + pub file_path: Bound<'py, PyAny>, + pub loaded_macros: Bound<'py, PyDict>, + pub macro_overrides: Option>, + pub macro_context: Bound<'py, PyAny>, + pub declaration_resolver: Option>, + pub declarations: Option>, + pub consumer: Option>, } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) +/// Where the caller's text came from, to locate calls in its authored file. +#[derive(Debug, Clone, Copy)] +pub struct SqlCallOrigin<'a> { + /// The authored file's project-relative path. + pub relative_path: &'a str, + /// The authored file's contents. + pub contents: &'a str, + /// The authored query the expanded text came from, within `contents`. + pub query: &'a str, } -/// Whether a macro's result calls any reference function. -fn result_has_markers(classes: &ExpansionClasses<'_>, result: &str) -> bool { - classes - .generated - .markers() - .any(|marker| result.contains(marker)) +/// What SQL macro calls in one declaration pass read beyond the declaration context. +#[derive(Clone, Copy)] +pub struct SqlMacroSource<'a, 'py> { + /// Every loaded macro by name, for unscoped compiles whose context holds none. + pub loaded_macros: Option<&'a Bound<'py, PyAny>>, + pub origin: Option>, + /// Where macro bodies resolve their declarations; `None` outside the input loops. + pub scopes: Option<&'a dyn MacroScopes<'py>>, + /// The lexical rules the authored SQL is written in; `None` reads it under the generic rules. + pub syntax: Option<&'a LexicalSyntax>, } -/// Observe modules a recorded call imported, so the store is not saved over changed code. -fn observe_new_modules(bridge: &Bound<'_, PyAny>, memo: &MacroCallMemo) -> PyResult<()> { - let Some(observed) = *memo.observed_modules.lock().map_err(poisoned)? else { - return Ok(()); - }; - let loaded: usize = PyModule::import(bridge.py(), "sys")? - .getattr("modules")? - .len()?; - if loaded != observed { - let _ = bridge.call_method0(OBSERVE_MODULES_METHOD)?; +impl std::fmt::Debug for SqlMacroSource<'_, '_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SqlMacroSource") + .finish_non_exhaustive() } - Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/types.rs b/crates/sqlbuild-python-front/src/render/types.rs new file mode 100644 index 0000000000..d56caa2878 --- /dev/null +++ b/crates/sqlbuild-python-front/src/render/types.rs @@ -0,0 +1,20 @@ +//! Public traits and type aliases of render. + +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_sqltext::compiler::models::CharSpan; + +/// One substitution span: source start and end, output start and end. +pub type SpanRow = CharSpan; + +/// The `(macros, overrides, context, declarations, consumer)` tuple Python passes one expansion. +pub type StateTuple = (Py, Py, Py, Py, Py); + +/// The declaration contexts of macro files in a scoped compile. +pub trait MacroScopes<'py> { + /// The context the file at `path` sees, or `None` when the compile is unscoped. + fn file_context(&self, path: &str) -> PyResult>>; +} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/mod.rs new file mode 100644 index 0000000000..62be446495 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod scope_tolerant; +pub(crate) mod scope_validation; +pub(crate) mod scope_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs index 97f2dde9f2..6f05103216 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs @@ -8,10 +8,14 @@ use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, ResourceIdentity, ResourceKind, ResourceRecord, }; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, enum_text, field, optional_item, read_resource, root_object, tuple_attribute, -}; -use crate::bindings::_helpers::scopes::scope_validation::replace; +use crate::scopes::_helpers::checks::scope_validation::replace; +use crate::scopes::_helpers::index::scope_objects::enum_text; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::index::scope_objects::optional_item; +use crate::scopes::_helpers::index::scope_objects::read_resource; +use crate::scopes::_helpers::index::scope_objects::root_object; +use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs index dbbc0fd927..ab84f15f4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs @@ -14,14 +14,23 @@ use sqlbuild_scopes::placement::models::{ }; use sqlbuild_scopes::scope_index::models::ResourceIdentity; -use crate::bindings::_helpers::scopes::compile_usages::{ - MergeRows, SourceRow, merged_usage_rows, model_schema_scope_usages, -}; -use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, enum_text, field, identity_row, optional_item, qualified_row, read_completeness, - read_placement_index, root_object, tuple_attribute, value_of, -}; +use crate::scopes::_helpers::declarations::placement::QualifiedRow; +use crate::scopes::_helpers::declarations::placement::resource_identity; +use crate::scopes::_helpers::index::scope_objects::enum_text; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::index::scope_objects::identity_row; +use crate::scopes::_helpers::index::scope_objects::optional_item; +use crate::scopes::_helpers::index::scope_objects::qualified_row; +use crate::scopes::_helpers::index::scope_objects::read_completeness; +use crate::scopes::_helpers::index::scope_objects::read_placement_index; +use crate::scopes::_helpers::index::scope_objects::root_object; +use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::_helpers::index::scope_objects::value_of; +use crate::scopes::_helpers::relationships::compile_usages::MergeRows; +use crate::scopes::_helpers::relationships::compile_usages::SourceRow; +use crate::scopes::_helpers::relationships::compile_usages::merged_usage_rows; +use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs index 417516cccb..ba33e6ec00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs @@ -11,12 +11,14 @@ use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, }; -use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, ResolvedRows, TargetRows, scope_path_error, -}; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, qualified_row, read_resource, tuple_attribute, -}; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_lookup::ResolvedRows; +use crate::scopes::_helpers::index::scope_lookup::TargetRows; +use crate::scopes::_helpers::index::scope_lookup::scope_path_error; +use crate::scopes::_helpers::index::scope_objects::qualified_row; +use crate::scopes::_helpers::index::scope_objects::read_resource; +use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs new file mode 100644 index 0000000000..fafbf520b4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs @@ -0,0 +1,66 @@ +//! Declaration resolution contexts built natively from a Python scope lookup's positions. + +use crate::scopes::models::NativeDeclarationContexts; +use std::collections::HashMap; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyDict; + +use sqlbuild_scopes::scope_index::models::VisibleEntry; + +pub(crate) const ENUM_KIND: u8 = 0; +pub(crate) const CONSTANT_KIND: u8 = 1; +pub(crate) const MACRO_KIND: u8 = 2; + +pub(crate) type TableRow = ( + Vec, + HashMap>, + HashMap>, +); +pub(crate) type VisibleRow<'py> = (Bound<'py, PyAny>, VisibleEntry, Option>); + +/// One declaration position: identity, first record, name, kinds, runtime value and key. +#[derive(Debug, FromPyObject)] +pub(crate) struct PositionRow( + pub(crate) Py, + pub(crate) Py, + pub(crate) Py, + pub(crate) u8, + pub(crate) Py, + pub(crate) u8, + pub(crate) u32, +); + +/// The Python classes and enum members the contexts are built from. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ContextClasses { + pub(crate) visibility_record: Py, + pub(crate) resolution_context: Py, + pub(crate) reasons: Py, +} + +/// The Python lookup mappings a matching resource's private positions and grants come from. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct LookupMappings { + pub(crate) identity_keys: Py, + pub(crate) private_positions: Py, + pub(crate) grants_by_resource: Py, +} + +/// Dictionaries one context is assembled into, in Python's insertion order. +pub(crate) struct ContextParts<'py> { + pub(crate) enums: Bound<'py, PyDict>, + pub(crate) constants: Bound<'py, PyDict>, + pub(crate) macros: Bound<'py, PyDict>, + pub(crate) macro_records: Bound<'py, PyDict>, + pub(crate) inaccessible: [Bound<'py, PyDict>; 3], + pub(crate) visibility: [Bound<'py, PyDict>; 3], +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs index 0915a43d9a..30e3caf6b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs @@ -13,7 +13,7 @@ use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, IdentityFailure, QualifiedIdentity, ResourceIdentity, ScopePathFailure, }; -use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; +use crate::scopes::models::NativeDeclarationContexts; use sqlbuild_python_base::records::main::fields::field::field as native_field; const PATH_CACHE_PREFIX: &str = ""; @@ -26,7 +26,7 @@ const PATH_RESOURCE_SUFFIX: &str = ">"; const MODEL_KIND: &str = "model"; /// The lookup mappings and classes resolution reads beyond the classifier's own. -#[derive(FromPyObject)] +#[derive(Debug, FromPyObject)] #[pyo3(from_item_all)] pub(crate) struct ResolutionInputs { resources_by_path: Py, @@ -49,7 +49,7 @@ type Matches<'py> = (Vec>, Bound<'py, PyAny>); impl NativeDeclarationContexts { /// The context of `file_path` or `resource`, and whether it was projected rather than cached. - pub(crate) fn resolved_context<'py>( + pub fn resolved_context<'py>( &self, resolver: &Bound<'py, PyAny>, file_path: &str, @@ -271,7 +271,7 @@ impl NativeDeclarationContexts { } /// `_rebind_declaration_context`: fresh mappings, visibility rebound to `consumer`. - pub(crate) fn rebound_context<'py>( + pub fn rebound_context<'py>( &self, context: &Bound<'py, PyAny>, consumer: &Bound<'py, PyAny>, @@ -294,7 +294,7 @@ impl NativeDeclarationContexts { } /// `_rebind_visibility`: each record with `consumer` as its resource. - pub(crate) fn rebound_visibility<'py>( + pub fn rebound_visibility<'py>( &self, records_by_name: &Bound<'py, PyDict>, consumer: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/mod.rs new file mode 100644 index 0000000000..d8ddc87120 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod declaration_contexts; +pub(crate) mod declaration_resolution; +pub(crate) mod placement; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/placement.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/declarations/placement.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/getters.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/getters.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/getters.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/index/getters.rs index 7e5c5fbda2..78500096a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/getters.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::scopes::scope_index::NativeScopeIndex; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::scopes::models::NativeScopeIndex; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_native_scope_index, -}]; - -fn read_native_scope_index<'py>( +pub(crate) fn read_native_scope_index<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/mod.rs new file mode 100644 index 0000000000..6b74563908 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod getters; +pub(crate) mod scope_cache; +pub(crate) mod scope_index; +pub(crate) mod scope_lookup; +pub(crate) mod scope_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs index 548b55dc52..83357b83b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs @@ -12,9 +12,10 @@ use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; use sqlbuild_scopes::scope_cache::models::{CachedIndex, Fingerprint, FingerprintInputs}; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, index_object, read_completeness, read_index, -}; +use crate::scopes::_helpers::index::scope_objects::index_object; +use crate::scopes::_helpers::index::scope_objects::read_completeness; +use crate::scopes::_helpers::index::scope_objects::read_index; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs index 77138e1ed1..1716405bb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs @@ -1,5 +1,6 @@ //! The static scope index built natively from discovered inputs, as the Python scope models. +use crate::scopes::models::NativeScopeIndex; use std::collections::HashMap; use pyo3::exceptions::PyKeyError; @@ -7,28 +8,26 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_scopes::relationship_names::models::RelationshipScan; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; -use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; use sqlbuild_scopes::scope_index::models::{ - DeclarationIdentity, DeclarationInput, DeclarationKind, GrantRecord, GrantThrough, - RejectedPath, RelationshipFact, ResourceIdentity, ResourceInput, ResourceKind, ResourceRoot, - ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, + DeclarationIdentity, DeclarationInput, DeclarationKind, RejectedPath, ResourceIdentity, + ResourceInput, ResourceKind, ResourceRoot, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, }; -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; -use crate::bindings::_helpers::scopes::relationship_scan::{ - RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, -}; -use crate::bindings::_helpers::scopes::scope_lookup::scope_path_error; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, field, optional_item, root_object, value_of, -}; -use crate::bindings::_helpers::scopes::scope_validation::replace; +use crate::discovery::models::ModelFileTexts; +use crate::discovery::models::NativeModelFile; +use crate::scopes::_helpers::index::scope_lookup::scope_path_error; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::index::scope_objects::optional_item; +use crate::scopes::_helpers::index::scope_objects::root_object; +use crate::scopes::_helpers::index::scope_objects::value_of; +use crate::scopes::_helpers::relationships::relationship_scan::ScenarioRow; +use crate::scopes::_helpers::relationships::relationship_scan::TestFileRow; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -42,7 +41,7 @@ use sqlbuild_python_base::types::CompilerDetach; const UNINDEXABLE_SCOPE_MESSAGE: &str = "Native scope indexing could not place a discovered path or resource kind. Discovery always yields normalized project-relative paths, so this is a SQLBuild bug; please file an issue with the project layout"; const INSPECT_MODULE: &str = "inspect"; const CODE_TYPE_MODULE: &str = "types"; -const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; const PRIVATE_ROOT: &str = "models"; const MACRO_FALLBACK: &str = "macros"; const ENUM_FALLBACK: &str = "enums"; @@ -135,12 +134,12 @@ fn header_name<'py>(file: &Bound<'py, PyAny>) -> PyResult> { native_field(file, "header_values")?.call_method1("get", ("name",)) } -fn items<'py>(item: &Bound<'py, PyAny>, name: &str) -> PyResult>> { +pub(crate) fn items<'py>(item: &Bound<'py, PyAny>, name: &str) -> PyResult>> { native_field(item, name)?.try_iter()?.collect() } impl<'py> Walk<'py> { - fn new(py: Python<'py>) -> PyResult { + pub(crate) fn new(py: Python<'py>) -> PyResult { Ok(Self { classes: ScopeClasses::new(py)?, inputs: ScopeInputs::default(), @@ -670,22 +669,12 @@ fn normalized(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { }) } -fn unindexable(py: Python<'_>) -> PyResult { +pub(crate) fn unindexable(py: Python<'_>) -> PyResult { Ok(PyErr::from_value( error_class(py, ErrorId::UnindexableScope)?.call1((UNINDEXABLE_SCOPE_MESSAGE,))?, )) } -/// A built scope index and the Python records materialised from it. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct NativeScopeIndex { - native: ScopeIndex, - /// Each declaration record in input order. - declarations: Vec>, - index: Py, - grants: Py, -} - fn resource_root<'py>( classes: &ScopeClasses<'py>, kind: ResourceKind, @@ -840,7 +829,7 @@ fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult) -> PyResult { +pub(crate) fn test_file_row(test_file: &Bound<'_, PyAny>) -> PyResult { let path: Bound<'_, PyAny> = native_field(test_file, "relative_path")?; let mut blocks: Vec<(String, String, String)> = Vec::new(); for block in items(test_file, "blocks")? { @@ -856,7 +845,7 @@ fn test_file_row(test_file: &Bound<'_, PyAny>) -> PyResult { Ok((text(&path)?, blocks)) } -fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult { Ok(( field(scenario, "name")?, text(&native_field(scenario, "relative_path")?)?, @@ -864,112 +853,6 @@ fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult { )) } -#[pymethods] -impl NativeScopeIndex { - /// The static index, without grants. - #[getter] - pub(crate) fn index(&self, py: Python<'_>) -> Py { - self.index.clone_ref(py) - } - - /// Whether any macro, enum or constant is scoped, which makes relationship grants matter. - #[getter] - pub(crate) fn has_scoped_relationship_declarations(&self) -> bool { - self.native.has_scoped_relationship_declarations - } - - /// Grant every test's and scenario's relationships; return each extraction fault. - #[pyo3(signature = (*, discovered_inputs, syntax, scan_store))] - fn grant_relationships( - &mut self, - py: Python<'_>, - discovered_inputs: &Bound<'_, PyAny>, - syntax: &Bound<'_, PyAny>, - scan_store: (Bound<'_, PyAny>, String), - ) -> PyResult> { - let files: Vec = items(discovered_inputs, "test_files")? - .iter() - .map(test_file_row) - .collect::>()?; - let scenarios: Vec = items(discovered_inputs, "scenario_files")? - .iter() - .map(scenario_row) - .collect::>()?; - let (store, algorithm) = scan_store; - let settings: Bound<'_, PyDict> = PyDict::new(py); - settings.set_item("syntax", native_field(syntax, "native_mapping")?)?; - settings.set_item("store", store)?; - settings.set_item("scan_key", (algorithm, native_field(syntax, "cache_key")?))?; - let settings: RelationshipScanSettings = settings.extract()?; - let scan: RelationshipScan = scanned_relationships(py, &files, &scenarios, settings)?; - let index: &ScopeIndex = &self.native; - let facts: &[RelationshipFact] = &scan.facts; - let grants: Result, ScopeDeferral> = py - .compiler_detach(|| Ok(relationship_grants(index, facts))) - .map_err(compiler_error)?; - let Ok(grants) = grants else { - return Err(unindexable(py)?); - }; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let mut records: Vec> = Vec::with_capacity(grants.len()); - for grant in &grants { - let through: Bound<'_, PyAny> = match &grant.through { - GrantThrough::Model(name) => classes.model(RecordId::ResourceIdentity)?.call1(( - classes.member(RecordId::ResourceKind, ResourceKind::Model.as_str())?, - name.as_str(), - ))?, - GrantThrough::Declaration(position) => { - self.declarations[*position].bind(py).getattr("identity")? - } - }; - records.push( - classes.model(RecordId::GrantRecord)?.call1(( - classes.model(RecordId::ResourceIdentity)?.call1(( - classes.member(RecordId::ResourceKind, grant.resource.kind.as_str())?, - grant.resource.name.as_str(), - ))?, - self.declarations[grant.declaration] - .bind(py) - .getattr("identity")?, - through, - classes.member(RecordId::GrantKind, grant.kind.as_str())?, - ))?, - ); - } - self.grants = PyTuple::new(py, records)?.unbind(); - let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; - let faults: Vec> = scan - .faults - .into_iter() - .map(|(path, message)| { - Ok(PyTuple::new( - py, - [path_class.call1((path,))?, message.into_bound_py_any(py)?], - )? - .into_any()) - }) - .collect::>()?; - Ok(PyTuple::new(py, faults)?.unbind()) - } - - /// The index with its grants attached and relationships complete. - fn index_with_relationships(&self, py: Python<'_>) -> PyResult> { - let index: &Bound<'_, PyAny> = self.index.bind(py); - let completeness: Bound<'_, PyAny> = replace( - &native_field(index, "completeness")?, - &[("relationships", true.into_bound_py_any(py)?)], - )?; - Ok(replace( - index, - &[ - ("grants", self.grants.bind(py).clone().into_any()), - ("completeness", completeness), - ], - )? - .unbind()) - } -} - /// Python's `open_native_scope_index`: the natively built index of discovered inputs. #[pyfunction] #[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs index 4956b5a323..e0ea677f39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs @@ -18,10 +18,12 @@ use sqlbuild_scopes::scope_index::models::{ }; use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; -use crate::bindings::_helpers::scopes::placement::{QualifiedRow, qualified_identity}; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, read_index, resource_identity_object, tuple_attribute, -}; +use crate::scopes::_helpers::declarations::placement::QualifiedRow; +use crate::scopes::_helpers::declarations::placement::qualified_identity; +use crate::scopes::_helpers::index::scope_objects::read_index; +use crate::scopes::_helpers::index::scope_objects::resource_identity_object; +use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs index 1e69c3b012..016b1beb3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs @@ -1,5 +1,6 @@ //! Python `ScopeIndex` models read into the native full index, and built back from it. +use crate::scopes::models::ScopeClasses; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use sqlbuild_scopes::placement::models::{PlacementIndex, PlacementUsage}; @@ -12,45 +13,21 @@ use sqlbuild_scopes::scope_index::models::{ ResourceKind, ResourceRecord, }; -use crate::bindings::_helpers::scopes::placement::{ - DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, - ownership_root, qualified_identity, resource_identity, -}; -use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; +use crate::scopes::_helpers::declarations::placement::DeclarationRow; +use crate::scopes::_helpers::declarations::placement::IdentityRow; +use crate::scopes::_helpers::declarations::placement::QualifiedRow; +use crate::scopes::_helpers::declarations::placement::RootRow; +use crate::scopes::_helpers::declarations::placement::declaration_identity; +use crate::scopes::_helpers::declarations::placement::declaration_record; +use crate::scopes::_helpers::declarations::placement::ownership_root; +use crate::scopes::_helpers::declarations::placement::qualified_identity; +use crate::scopes::_helpers::declarations::placement::resource_identity; +use crate::scopes::_helpers::index::scope_lookup::grant_kind; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const ENUM_MODULE: &str = "enum"; -/// The native scope model and enum classes every built object comes from. -pub(crate) struct ScopeClasses<'py> { - py: Python<'py>, -} - -impl<'py> ScopeClasses<'py> { - /// The scope classes of `py`. - pub(crate) fn new(py: Python<'py>) -> PyResult { - Ok(Self { py }) - } - - /// One scope model class. - pub(crate) fn model(&self, name: RecordId) -> PyResult> { - record_class(self.py, name) - } - - /// The member of scope enum `name` with `value`; an unknown value raises `ValueError`. - pub(crate) fn member(&self, name: RecordId, value: &str) -> PyResult> { - new_record(self.py, name, (value,), None) - } - - /// The interpreter the classes belong to. - pub(crate) fn py(&self) -> Python<'py> { - self.py - } -} - /// `str(value.value)` of an enum, else `str(value)`. pub(crate) fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { let enum_type = value.py().import(ENUM_MODULE)?.getattr("Enum")?; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/mod.rs new file mode 100644 index 0000000000..dd95c3c9ea --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod checks; +pub(crate) mod declarations; +pub(crate) mod index; +pub(crate) mod relationships; +pub(crate) mod reports; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs index 35ed2fce64..b84011d0f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs @@ -27,8 +27,6 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -/// The folder a scope path is relative to when its owner names none. -pub(crate) const CURRENT_FOLDER: &str = "."; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/mod.rs new file mode 100644 index 0000000000..8959f14da6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod compile_usages; +pub(crate) mod relationship_names; +pub(crate) mod relationship_scan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs index 15b2e43d85..eb801aa90f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs @@ -9,7 +9,7 @@ use sqlbuild_scopes::relationship_names::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_scan.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_scan.rs index 7f8cd890c6..f9bf1a70e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_scan.rs @@ -9,7 +9,7 @@ use sqlbuild_scopes::relationship_names::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/mod.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/mod.rs new file mode 100644 index 0000000000..13d2d9a90a --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod scope_errors; +pub(crate) mod scope_report_objects; +pub(crate) mod scope_reports; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs index e50f0edd1b..396b548083 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs @@ -15,13 +15,16 @@ use sqlbuild_scopes::scope_report::models::{ ScopeFolder, ScopeReport, ScopeResult, ScopeSection, SourceLocation, }; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, field, optional_item, read_declaration_identity, read_diagnostic, read_qualified, -}; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::index::scope_objects::optional_item; +use crate::scopes::_helpers::index::scope_objects::read_declaration_identity; +use crate::scopes::_helpers::index::scope_objects::read_diagnostic; +use crate::scopes::_helpers::index::scope_objects::read_qualified; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -type PyObject = Py; +pub(crate) type PyObject = Py; const PATHLIB_MODULE: &str = "pathlib"; /// The error `plain_value` raises for an integer outside 64 bits. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs rename to crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs index d183c49625..cfb49a189b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs @@ -1,5 +1,8 @@ //! `sqb scope` reports over a `ScopeLookup`, their serialization and the metadata projection. +use crate::scopes::models::ExplainArguments; +use crate::scopes::models::ListArguments; +use crate::scopes::models::ReportArguments; use pyo3::exceptions::{PyIndexError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -26,44 +29,31 @@ use sqlbuild_scopes::scope_report::models::{ ScopeResult, }; -use crate::bindings::_helpers::scopes::scope_lookup::NativeScopeLookup; -use crate::bindings::_helpers::scopes::scope_objects::{ - ScopeClasses, read_completeness, read_declaration, read_index, -}; -use crate::bindings::_helpers::scopes::scope_report_objects::{ - browse_object, diagnostics_object, explanation_object, json_to_plain, json_to_python, - list_object, move_object, read_declaration_target, read_filters, read_optional_target, - read_result, read_target, report_object, -}; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_objects::read_completeness; +use crate::scopes::_helpers::index::scope_objects::read_declaration; +use crate::scopes::_helpers::index::scope_objects::read_index; +use crate::scopes::_helpers::reports::scope_report_objects::browse_object; +use crate::scopes::_helpers::reports::scope_report_objects::diagnostics_object; +use crate::scopes::_helpers::reports::scope_report_objects::explanation_object; +use crate::scopes::_helpers::reports::scope_report_objects::json_to_plain; +use crate::scopes::_helpers::reports::scope_report_objects::json_to_python; +use crate::scopes::_helpers::reports::scope_report_objects::list_object; +use crate::scopes::_helpers::reports::scope_report_objects::move_object; +use crate::scopes::_helpers::reports::scope_report_objects::read_declaration_target; +use crate::scopes::_helpers::reports::scope_report_objects::read_filters; +use crate::scopes::_helpers::reports::scope_report_objects::read_optional_target; +use crate::scopes::_helpers::reports::scope_report_objects::read_result; +use crate::scopes::_helpers::reports::scope_report_objects::read_target; +use crate::scopes::_helpers::reports::scope_report_objects::report_object; +use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -type PyObject = Py; - -/// Target, location, whether it is a directory, and the filters. -pub(crate) struct ReportArguments<'py> { - pub(crate) target: Option>, - pub(crate) at: Option>, - pub(crate) directory: bool, - pub(crate) filters: Option>, -} -/// Declaration, target, location and whether it is a directory. -pub(crate) struct ExplainArguments<'py> { - pub(crate) declaration: Bound<'py, PyAny>, - pub(crate) target: Option>, - pub(crate) at: Option>, - pub(crate) directory: bool, -} -/// Folder, filters, target and whether the target is prospective. -pub(crate) struct ListArguments<'py> { - pub(crate) folder: String, - pub(crate) filters: Option>, - pub(crate) target: Option>, - pub(crate) target_is_prospective: bool, -} +pub(crate) type PyObject = Py; const CURRENT_FOLDER: &str = "."; diff --git a/crates/sqlbuild-python-front/src/scopes/constants.rs b/crates/sqlbuild-python-front/src/scopes/constants.rs new file mode 100644 index 0000000000..a17faf551a --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/constants.rs @@ -0,0 +1,15 @@ +//! Public constants of scopes. + +use crate::scopes::_helpers::index::getters::read_native_scope_index; +use crate::scopes::models::NativeScopeIndex; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The folder a scope path is relative to when its owner names none. +pub const CURRENT_FOLDER: &str = "."; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_native_scope_index, +}]; diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/mod.rs new file mode 100644 index 0000000000..3045d21e05 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/mod.rs @@ -0,0 +1,6 @@ +pub mod register_scope_validation; +pub mod register_scope_visibility; +pub mod relocated_declarations; +pub mod scope_index_with_compile_usages; +pub mod scope_tolerant_index; +pub mod validate_scope_index; diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs new file mode 100644 index 0000000000..59bb9c9aa3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs @@ -0,0 +1,12 @@ +//! Register the scope validation functions and classes. + +use crate::scopes::_helpers::checks::scope_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope validation functions and classes. +#[inline] +pub fn register_scope_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_validation::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs new file mode 100644 index 0000000000..03b3da25b7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs @@ -0,0 +1,12 @@ +//! Register the scope visibility functions and classes. + +use crate::scopes::_helpers::checks::scope_visibility; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope visibility functions and classes. +#[inline] +pub fn register_scope_visibility(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_visibility::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs new file mode 100644 index 0000000000..1f67ce386f --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs @@ -0,0 +1,17 @@ +//! Relocated records after every move in `moves`, and per record whether it lacks a needed group. + +use crate::scopes::_helpers::checks::scope_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; +use sqlbuild_scopes::scope_index::models::ResourceIdentity; + +/// Relocated records after every move in `moves`, and per record whether it lacks a needed group. +#[inline] +pub fn relocated_declarations<'py>( + index: &Bound<'py, PyAny>, + moves: &[(ResourceIdentity, String)], +) -> PyResult, Vec)>> { + scope_validation::relocated_declarations(index, moves) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs new file mode 100644 index 0000000000..8eb10e7943 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs @@ -0,0 +1,14 @@ +//! Compile inputs' canonical index with all usage and placement facts. + +use crate::scopes::_helpers::checks::scope_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Compile inputs' canonical index with all usage and placement facts. +#[inline] +pub fn scope_index_with_compile_usages<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + scope_validation::scope_index_with_compile_usages(inputs) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs new file mode 100644 index 0000000000..82e2186ba0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs @@ -0,0 +1,19 @@ +//! The tolerant index: fault diagnostics, prospective resources and partial completeness. + +use crate::scopes::_helpers::checks::scope_tolerant; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The tolerant index: fault diagnostics, prospective resources and partial completeness. +#[inline] +pub fn scope_tolerant_index( + py: Python<'_>, + index: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, + faults: (Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult> { + scope_tolerant::scope_tolerant_index(py, index, snapshot, faults) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs new file mode 100644 index 0000000000..117154f15d --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs @@ -0,0 +1,12 @@ +//! Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. + +use crate::scopes::_helpers::checks::scope_validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; + +/// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. +#[inline] +pub fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { + scope_validation::validate_scope_index(index) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/declarations/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/declarations/mod.rs new file mode 100644 index 0000000000..b038a93a76 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/declarations/mod.rs @@ -0,0 +1 @@ +pub mod register_declaration_contexts; diff --git a/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs new file mode 100644 index 0000000000..b613d65b38 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs @@ -0,0 +1,12 @@ +//! Register the declaration contexts functions and classes. + +use crate::scopes::_helpers::declarations::declaration_contexts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the declaration contexts functions and classes. +#[inline] +pub fn register_declaration_contexts(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_contexts::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs new file mode 100644 index 0000000000..d9260694d9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs @@ -0,0 +1,14 @@ +//! Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. + +use crate::scopes::_helpers::index::scope_lookup; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. +#[inline] +pub fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { + scope_lookup::build_scope_lookup(py, index) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/index/mod.rs new file mode 100644 index 0000000000..9b4bded765 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/mod.rs @@ -0,0 +1,8 @@ +pub mod build_scope_lookup; +pub mod open_native_scope_index; +pub mod project_fingerprint; +pub mod read_scope_index_cache; +pub mod register_scope_cache; +pub mod register_scope_index; +pub mod register_scope_lookup; +pub mod write_scope_index_cache; diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs new file mode 100644 index 0000000000..2196717847 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs @@ -0,0 +1,19 @@ +//! Python's `open_native_scope_index`: the natively built index of discovered inputs. + +use crate::scopes::_helpers::index::scope_index; +use crate::scopes::models::NativeScopeIndex; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Python's `open_native_scope_index`: the natively built index of discovered inputs. +#[inline] +pub fn open_native_scope_index( + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + loaded_macros: &Bound<'_, PyAny>, + static_macros: Option>, +) -> PyResult { + scope_index::open_native_scope_index(py, discovered_inputs, loaded_macros, static_macros) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs new file mode 100644 index 0000000000..ac5ddfb553 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs @@ -0,0 +1,12 @@ +//! The scope index fingerprint of `project_dir`, `None` when it cannot be read. + +use crate::scopes::_helpers::index::scope_cache; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::PathBuf; + +/// The scope index fingerprint of `project_dir`, `None` when it cannot be read. +#[inline] +pub fn project_fingerprint(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + scope_cache::project_fingerprint(py, project_dir) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs new file mode 100644 index 0000000000..146846e797 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs @@ -0,0 +1,18 @@ +//! The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. + +use crate::scopes::_helpers::index::scope_cache; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::PathBuf; + +/// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. +#[inline] +pub fn read_scope_index_cache( + py: Python<'_>, + path: PathBuf, + fingerprint: String, +) -> PyResult>> { + scope_cache::read_scope_index_cache(py, path, fingerprint) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs new file mode 100644 index 0000000000..4dcdf09e70 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs @@ -0,0 +1,12 @@ +//! Register the scope cache functions and classes. + +use crate::scopes::_helpers::index::scope_cache; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope cache functions and classes. +#[inline] +pub fn register_scope_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_cache::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs new file mode 100644 index 0000000000..1194dee508 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs @@ -0,0 +1,12 @@ +//! Register the scope index functions and classes. + +use crate::scopes::_helpers::index::scope_index; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope index functions and classes. +#[inline] +pub fn register_scope_index(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_index::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs new file mode 100644 index 0000000000..4c3e76c30d --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs @@ -0,0 +1,12 @@ +//! Register the scope lookup functions and classes. + +use crate::scopes::_helpers::index::scope_lookup; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope lookup functions and classes. +#[inline] +pub fn register_scope_lookup(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_lookup::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs new file mode 100644 index 0000000000..1f3e8858d2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs @@ -0,0 +1,19 @@ +//! Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. + +use crate::scopes::_helpers::index::scope_cache; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::PathBuf; + +/// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. +#[inline] +pub fn write_scope_index_cache( + py: Python<'_>, + location: (PathBuf, String), + fingerprint: String, + index: &Bound<'_, PyAny>, +) -> PyResult { + scope_cache::write_scope_index_cache(py, location, fingerprint, index) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/mod.rs new file mode 100644 index 0000000000..711007d92b --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/mod.rs @@ -0,0 +1,5 @@ +pub mod checks; +pub mod declarations; +pub mod index; +pub mod relationships; +pub mod reports; diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/mod.rs new file mode 100644 index 0000000000..b0b6f3c5a6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/mod.rs @@ -0,0 +1,2 @@ +pub mod register_compile_usages; +pub mod register_relationship_names; diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs new file mode 100644 index 0000000000..2b36f57b36 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs @@ -0,0 +1,12 @@ +//! Register the compile usages functions and classes. + +use crate::scopes::_helpers::relationships::compile_usages; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the compile usages functions and classes. +#[inline] +pub fn register_compile_usages(module: &Bound<'_, PyModule>) -> PyResult<()> { + compile_usages::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs new file mode 100644 index 0000000000..03781fc7b2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs @@ -0,0 +1,12 @@ +//! Register the relationship names functions and classes. + +use crate::scopes::_helpers::relationships::relationship_names; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the relationship names functions and classes. +#[inline] +pub fn register_relationship_names(module: &Bound<'_, PyModule>) -> PyResult<()> { + relationship_names::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs new file mode 100644 index 0000000000..ff32cbaeb1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs @@ -0,0 +1,13 @@ +//! A JSON value as Python builds it; arrays become tuples or lists. + +use crate::scopes::_helpers::reports::scope_report_objects; +use crate::scopes::_helpers::reports::scope_report_objects::PyObject; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_core::json::models::JsonValue; + +/// A JSON value as Python builds it; arrays become tuples or lists. +#[inline] +pub fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { + scope_report_objects::json_to_python(py, value, tuples) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs new file mode 100644 index 0000000000..9ccc5d8479 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs @@ -0,0 +1,12 @@ +//! `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. + +use crate::scopes::_helpers::reports::scope_reports; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. +#[inline] +pub fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { + scope_reports::metadata_projection_value(index) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/mod.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/mod.rs new file mode 100644 index 0000000000..96240903c4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/mod.rs @@ -0,0 +1,12 @@ +pub mod json_to_python; +pub mod metadata_projection_value; +pub mod read_result; +pub mod register_scope_reports; +pub mod report_scope_index_errors; +pub mod scope_browse; +pub mod scope_explain; +pub mod scope_list; +pub mod scope_move_preview; +pub mod scope_rebound_usages; +pub mod scope_report; +pub mod serialize_scope_report; diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs new file mode 100644 index 0000000000..255b493c51 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs @@ -0,0 +1,14 @@ +//! A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. + +use crate::scopes::_helpers::reports::scope_report_objects; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_scopes::scope_report::models::ScopeResult; + +/// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. +#[inline] +pub fn read_result(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { + scope_report_objects::read_result(classes, result) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs new file mode 100644 index 0000000000..fe8102070b --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs @@ -0,0 +1,12 @@ +//! Register the scope reports functions and classes. + +use crate::scopes::_helpers::reports::scope_reports; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope reports functions and classes. +#[inline] +pub fn register_scope_reports(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope_reports::register(module) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs new file mode 100644 index 0000000000..fb15112112 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs @@ -0,0 +1,12 @@ +//! `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. + +use crate::scopes::_helpers::reports::scope_errors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. +#[inline] +pub fn report_scope_index_errors(index: &Bound<'_, PyAny>) -> PyResult<()> { + scope_errors::report_scope_index_errors(index) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs new file mode 100644 index 0000000000..1a134fa5a9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs @@ -0,0 +1,18 @@ +//! `browse_scope_folders`: the direct child declaration-definition folders of `folder`. + +use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::reports::scope_reports::PyObject; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. +#[inline] +pub fn scope_browse( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: (String, Option>, bool), +) -> PyResult { + scope_reports::scope_browse(py, lookup, request) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs new file mode 100644 index 0000000000..86db324665 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs @@ -0,0 +1,19 @@ +//! `explain_scope_declaration`: one qualified declaration in an optional context. + +use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::models::ExplainArguments; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `explain_scope_declaration`: one qualified declaration in an optional context. +#[inline] +pub fn scope_explain( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ExplainArguments<'_>, +) -> PyResult<(PyObject, PyObject)> { + scope_reports::scope_explain(py, lookup, request) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs new file mode 100644 index 0000000000..7460b26b08 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs @@ -0,0 +1,19 @@ +//! `list_scope_declarations`: declarations recursively beneath one definition folder. + +use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::models::ListArguments; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `list_scope_declarations`: declarations recursively beneath one definition folder. +#[inline] +pub fn scope_list( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ListArguments<'_>, +) -> PyResult { + scope_reports::scope_list(py, lookup, request) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs new file mode 100644 index 0000000000..21dbee2b9e --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs @@ -0,0 +1,19 @@ +//! `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. + +use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::reports::scope_reports::PyObject; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. +#[inline] +pub fn scope_move_preview( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + resource: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<(Option, PyObject)> { + scope_reports::scope_move_preview(py, lookup, resource, destination) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs new file mode 100644 index 0000000000..ac85c5fe37 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs @@ -0,0 +1,17 @@ +//! Scoped declarations the resources at `path` use that resolve differently from `destination`. + +use crate::scopes::_helpers::reports::scope_reports; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Scoped declarations the resources at `path` use that resolve differently from `destination`. +#[inline] +pub fn scope_rebound_usages( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + paths: (&str, &str), +) -> PyResult>> { + scope_reports::scope_rebound_usages(py, lookup, paths) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs new file mode 100644 index 0000000000..5ad16340fa --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs @@ -0,0 +1,19 @@ +//! `query_scope_report`: one existing target or prospective path; invalid paths raise. + +use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::models::ReportArguments; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `query_scope_report`: one existing target or prospective path; invalid paths raise. +#[inline] +pub fn scope_report( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ReportArguments<'_>, +) -> PyResult { + scope_reports::scope_report(py, lookup, request) +} diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs new file mode 100644 index 0000000000..91429ea6c6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs @@ -0,0 +1,13 @@ +//! `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. + +use crate::scopes::_helpers::reports::scope_reports; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. +#[inline] +pub fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { + scope_reports::serialize_scope_report(py, report) +} diff --git a/crates/sqlbuild-python-front/src/scopes/mod.rs b/crates/sqlbuild-python-front/src/scopes/mod.rs new file mode 100644 index 0000000000..f5236d29bb --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/mod.rs @@ -0,0 +1,6 @@ +//! Declaration scopes: placement, resolution, visibility, relationships and scope reports. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/scopes/models.rs b/crates/sqlbuild-python-front/src/scopes/models.rs new file mode 100644 index 0000000000..f9cdf007a1 --- /dev/null +++ b/crates/sqlbuild-python-front/src/scopes/models.rs @@ -0,0 +1,487 @@ +//! Public records of scopes. + +use crate::scopes::_helpers::checks::scope_validation::replace; +use crate::scopes::_helpers::declarations::declaration_contexts::CONSTANT_KIND; +use crate::scopes::_helpers::declarations::declaration_contexts::ContextClasses; +use crate::scopes::_helpers::declarations::declaration_contexts::ContextParts; +use crate::scopes::_helpers::declarations::declaration_contexts::ENUM_KIND; +use crate::scopes::_helpers::declarations::declaration_contexts::LookupMappings; +use crate::scopes::_helpers::declarations::declaration_contexts::MACRO_KIND; +use crate::scopes::_helpers::declarations::declaration_contexts::PositionRow; +use crate::scopes::_helpers::declarations::declaration_contexts::TableRow; +use crate::scopes::_helpers::declarations::declaration_contexts::VisibleRow; +use crate::scopes::_helpers::declarations::declaration_resolution::ResolutionInputs; +use crate::scopes::_helpers::index::scope_index::PATHLIB_MODULE; +use crate::scopes::_helpers::index::scope_index::items; +use crate::scopes::_helpers::index::scope_index::scenario_row; +use crate::scopes::_helpers::index::scope_index::test_file_row; +use crate::scopes::_helpers::index::scope_index::unindexable; +use crate::scopes::_helpers::relationships::relationship_scan::RelationshipScanSettings; +use crate::scopes::_helpers::relationships::relationship_scan::ScenarioRow; +use crate::scopes::_helpers::relationships::relationship_scan::TestFileRow; +use crate::scopes::_helpers::relationships::relationship_scan::scanned_relationships; +use pyo3::IntoPyObjectExt; +use pyo3::intern; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::relationship_names::models::RelationshipScan; +use sqlbuild_scopes::scope_index::main::classify_resource::classify_resource; +use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; +use sqlbuild_scopes::scope_index::models::ConsumerResource; +use sqlbuild_scopes::scope_index::models::GrantEntry; +use sqlbuild_scopes::scope_index::models::GrantRecord; +use sqlbuild_scopes::scope_index::models::GrantThrough; +use sqlbuild_scopes::scope_index::models::RelationshipFact; +use sqlbuild_scopes::scope_index::models::ResourceKind; +use sqlbuild_scopes::scope_index::models::ScopeDeferral; +use sqlbuild_scopes::scope_index::models::ScopeIndex; +use sqlbuild_scopes::scope_index::models::VisibilityReason; +use sqlbuild_scopes::scope_index::models::VisibilityTable; +use sqlbuild_scopes::scope_index::models::VisibleEntry; +use std::collections::HashMap; + +/// Positions, visibility table and classes of one scope lookup. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeDeclarationContexts { + pub(crate) positions: Vec, + pub(crate) table: VisibilityTable, + pub(crate) classes: ContextClasses, + pub(crate) lookup: LookupMappings, + pub(crate) resolution: ResolutionInputs, +} + +#[pymethods] +impl NativeDeclarationContexts { + #[new] + pub(crate) fn new( + positions: Vec, + table: TableRow, + classes: ContextClasses, + lookups: (LookupMappings, ResolutionInputs), + ) -> Self { + let (lookup, resolution) = lookups; + let (global, local, inherited) = table; + let identity_keys: Vec = positions.iter().map(|row| row.6).collect(); + Self { + table: VisibilityTable { + identity_keys, + global, + local, + inherited, + }, + positions, + classes, + lookup, + resolution, + } + } + + /// `resolve_declaration_context`: the context and whether it was projected, not cached. + #[pyo3(signature = (resolver, file_path, resource=None))] + pub(crate) fn resolve<'py>( + &self, + resolver: Bound<'py, PyAny>, + file_path: Bound<'py, PyAny>, + resource: Option>, + ) -> PyResult<(Bound<'py, PyAny>, bool)> { + compiler_guard(|| { + let path: String = file_path.str()?.to_string(); + self.resolved_context(&resolver, &path, resource.as_ref()) + }) + } +} + +impl NativeDeclarationContexts { + pub fn classified_context<'py>( + &self, + py: Python<'py>, + matches: Vec>, + consumer: Bound<'py, PyAny>, + ) -> PyResult> { + let mut visible: Vec> = Vec::new(); + let mut inaccessible: Vec = Vec::new(); + for record in matches { + let identity: Bound<'py, PyAny> = native_field(&record, "identity")?; + let path: String = native_field(&record, "path")?.extract()?; + let private: Vec = self + .lookup + .private_positions + .bind(py) + .call_method1(intern!(py, "get"), (&identity, PyTuple::empty(py)))? + .extract()?; + if private + .iter() + .any(|position| *position >= self.positions.len()) + { + return Err(compiler_error( + "private declaration outside the scope lookup", + )); + } + let (grants, throughs) = self.grants(py, &identity)?; + let classified = classify_resource( + &self.table, + &ConsumerResource { + private, + path, + grants, + }, + ) + .map_err(|deferral| compiler_error(deferral.reason))?; + for entry in classified.visible { + let through: Option> = + entry.through.map(|slot| throughs[slot].clone()); + visible.push((identity.clone(), entry, through)); + } + inaccessible.extend(classified.inaccessible); + } + self.assemble(py, &visible, &inaccessible, consumer) + } + + /// The resource's grants of indexed declarations, with each grant's `through` value. + pub(crate) fn grants<'py>( + &self, + py: Python<'py>, + identity: &Bound<'py, PyAny>, + ) -> PyResult<(Vec, Vec>)> { + let records: Bound<'py, PyAny> = self + .lookup + .grants_by_resource + .bind(py) + .call_method1(intern!(py, "get"), (identity, PyTuple::empty(py)))?; + let identity_keys: &Bound<'py, PyDict> = self.lookup.identity_keys.bind(py); + let mut grants: Vec = Vec::new(); + let mut throughs: Vec> = Vec::new(); + for grant in records.try_iter()? { + let grant: Bound<'py, PyAny> = grant?; + let Some(identity_key) = + identity_keys.get_item(native_field(&grant, "declaration")?)? + else { + continue; + }; + let kind: String = native_field(&grant, "kind")? + .getattr(intern!(py, "value"))? + .extract()?; + grants.push(GrantEntry { + identity_key: identity_key.extract()?, + reason: if kind == VisibilityReason::TestedMacro.as_str() { + VisibilityReason::TestedMacro + } else { + VisibilityReason::ExpectedModel + }, + through: throughs.len(), + }); + throughs.push(native_field(&grant, "through")?); + } + Ok((grants, throughs)) + } + + pub(crate) fn assemble<'py>( + &self, + py: Python<'py>, + visible: &[VisibleRow<'py>], + inaccessible: &[usize], + consumer: Bound<'py, PyAny>, + ) -> PyResult> { + let parts: ContextParts<'py> = ContextParts { + enums: PyDict::new(py), + constants: PyDict::new(py), + macros: PyDict::new(py), + macro_records: PyDict::new(py), + inaccessible: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], + visibility: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], + }; + let mut records_by_identity: HashMap>> = HashMap::new(); + for (resource, entry, through) in visible { + let record: Bound<'py, PyAny> = self.visibility_record(py, resource, entry, through)?; + records_by_identity + .entry(self.positions[entry.position].6) + .or_default() + .push(record); + } + for (_, entry, _) in visible { + self.add_visible_value(py, &parts, entry.position)?; + } + for position in inaccessible { + let row: &PositionRow = &self.positions[*position]; + if let Some(target) = parts.inaccessible.get(usize::from(row.3)) { + target.set_item(row.2.bind(py), row.1.bind(py))?; + } + } + for (_, entry, _) in visible { + let row: &PositionRow = &self.positions[entry.position]; + if let Some(target) = parts.visibility.get(usize::from(row.3)) { + let records: &[Bound<'py, PyAny>] = + records_by_identity.get(&row.6).map_or(&[], Vec::as_slice); + target.set_item(row.2.bind(py), PyTuple::new(py, records)?)?; + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("enums", &parts.enums)?; + fields.set_item("constants", &parts.constants)?; + fields.set_item("inaccessible_enums", &parts.inaccessible[0])?; + fields.set_item("inaccessible_constants", &parts.inaccessible[1])?; + fields.set_item("enum_visibility", &parts.visibility[0])?; + fields.set_item("constant_visibility", &parts.visibility[1])?; + fields.set_item("macros", &parts.macros)?; + fields.set_item("macro_records", &parts.macro_records)?; + fields.set_item("macro_visibility", &parts.visibility[2])?; + fields.set_item("inaccessible_macros", &parts.inaccessible[2])?; + fields.set_item("consumer", consumer)?; + self.classes + .resolution_context + .bind(py) + .call((), Some(&fields)) + } + + pub(crate) fn add_visible_value<'py>( + &self, + py: Python<'py>, + parts: &ContextParts<'py>, + position: usize, + ) -> PyResult<()> { + let row: &PositionRow = &self.positions[position]; + let name: &Bound<'py, PyAny> = row.2.bind(py); + let value: &Bound<'py, PyAny> = row.4.bind(py); + match row.5 { + ENUM_KIND => parts.enums.set_item(name, value), + CONSTANT_KIND => parts.constants.set_item(name, value), + MACRO_KIND => { + parts.macros.set_item(name, value)?; + parts.macro_records.set_item(name, row.1.bind(py)) + } + _ => Ok(()), + } + } + + pub(crate) fn visibility_record<'py>( + &self, + py: Python<'py>, + resource: &Bound<'py, PyAny>, + entry: &VisibleEntry, + through: &Option>, + ) -> PyResult> { + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let reason: Bound<'py, PyAny> = self + .classes + .reasons + .bind(py) + .get_item(entry.reason.as_str())? + .unwrap_or_else(|| none.clone()); + self.visibility_record_for( + resource, + self.positions[entry.position].0.bind(py), + &reason, + through.as_ref().unwrap_or(&none), + ) + } + + /// A `VisibilityRecord` of `resource`, `declaration`, `reason` and `through`. + pub fn visibility_record_for<'py>( + &self, + resource: &Bound<'py, PyAny>, + declaration: &Bound<'py, PyAny>, + reason: &Bound<'py, PyAny>, + through: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = resource.py(); + let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); + let record: Bound<'py, PyAny> = class.call1((resource, declaration, reason, through))?; + Ok(record) + } +} + +/// A built scope index and the Python records materialised from it. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeScopeIndex { + pub(crate) native: ScopeIndex, + /// Each declaration record in input order. + pub(crate) declarations: Vec>, + pub(crate) index: Py, + pub(crate) grants: Py, +} + +#[pymethods] +impl NativeScopeIndex { + /// The static index, without grants. + #[getter] + pub fn index(&self, py: Python<'_>) -> Py { + self.index.clone_ref(py) + } + + /// Whether any macro, enum or constant is scoped, which makes relationship grants matter. + #[getter] + pub fn has_scoped_relationship_declarations(&self) -> bool { + self.native.has_scoped_relationship_declarations + } + + /// Grant every test's and scenario's relationships; return each extraction fault. + #[pyo3(signature = (*, discovered_inputs, syntax, scan_store))] + pub(crate) fn grant_relationships( + &mut self, + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + syntax: &Bound<'_, PyAny>, + scan_store: (Bound<'_, PyAny>, String), + ) -> PyResult> { + let files: Vec = items(discovered_inputs, "test_files")? + .iter() + .map(test_file_row) + .collect::>()?; + let scenarios: Vec = items(discovered_inputs, "scenario_files")? + .iter() + .map(scenario_row) + .collect::>()?; + let (store, algorithm) = scan_store; + let settings: Bound<'_, PyDict> = PyDict::new(py); + settings.set_item("syntax", native_field(syntax, "native_mapping")?)?; + settings.set_item("store", store)?; + settings.set_item("scan_key", (algorithm, native_field(syntax, "cache_key")?))?; + let settings: RelationshipScanSettings = settings.extract()?; + let scan: RelationshipScan = scanned_relationships(py, &files, &scenarios, settings)?; + let index: &ScopeIndex = &self.native; + let facts: &[RelationshipFact] = &scan.facts; + let grants: Result, ScopeDeferral> = py + .compiler_detach(|| Ok(relationship_grants(index, facts))) + .map_err(compiler_error)?; + let Ok(grants) = grants else { + return Err(unindexable(py)?); + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let mut records: Vec> = Vec::with_capacity(grants.len()); + for grant in &grants { + let through: Bound<'_, PyAny> = match &grant.through { + GrantThrough::Model(name) => classes.model(RecordId::ResourceIdentity)?.call1(( + classes.member(RecordId::ResourceKind, ResourceKind::Model.as_str())?, + name.as_str(), + ))?, + GrantThrough::Declaration(position) => { + self.declarations[*position].bind(py).getattr("identity")? + } + }; + records.push( + classes.model(RecordId::GrantRecord)?.call1(( + classes.model(RecordId::ResourceIdentity)?.call1(( + classes.member(RecordId::ResourceKind, grant.resource.kind.as_str())?, + grant.resource.name.as_str(), + ))?, + self.declarations[grant.declaration] + .bind(py) + .getattr("identity")?, + through, + classes.member(RecordId::GrantKind, grant.kind.as_str())?, + ))?, + ); + } + self.grants = PyTuple::new(py, records)?.unbind(); + let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; + let faults: Vec> = scan + .faults + .into_iter() + .map(|(path, message)| { + Ok(PyTuple::new( + py, + [path_class.call1((path,))?, message.into_bound_py_any(py)?], + )? + .into_any()) + }) + .collect::>()?; + Ok(PyTuple::new(py, faults)?.unbind()) + } + + /// The index with its grants attached and relationships complete. + pub(crate) fn index_with_relationships(&self, py: Python<'_>) -> PyResult> { + let index: &Bound<'_, PyAny> = self.index.bind(py); + let completeness: Bound<'_, PyAny> = replace( + &native_field(index, "completeness")?, + &[("relationships", true.into_bound_py_any(py)?)], + )?; + Ok(replace( + index, + &[ + ("grants", self.grants.bind(py).clone().into_any()), + ("completeness", completeness), + ], + )? + .unbind()) + } +} + +/// The native scope model and enum classes every built object comes from. +pub struct ScopeClasses<'py> { + pub(crate) py: Python<'py>, +} + +impl std::fmt::Debug for ScopeClasses<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ScopeClasses") + .finish_non_exhaustive() + } +} + +impl<'py> ScopeClasses<'py> { + /// The scope classes of `py`. + pub fn new(py: Python<'py>) -> PyResult { + Ok(Self { py }) + } + + /// One scope model class. + pub fn model(&self, name: RecordId) -> PyResult> { + record_class(self.py, name) + } + + /// The member of scope enum `name` with `value`; an unknown value raises `ValueError`. + pub fn member(&self, name: RecordId, value: &str) -> PyResult> { + new_record(self.py, name, (value,), None) + } + + /// The interpreter the classes belong to. + pub fn py(&self) -> Python<'py> { + self.py + } +} + +/// Target, location, whether it is a directory, and the filters. +#[derive(Debug)] +pub struct ReportArguments<'py> { + pub target: Option>, + pub at: Option>, + pub directory: bool, + pub filters: Option>, +} + +/// Declaration, target, location and whether it is a directory. +#[derive(Debug)] +pub struct ExplainArguments<'py> { + pub declaration: Bound<'py, PyAny>, + pub target: Option>, + pub at: Option>, + pub directory: bool, +} + +/// Folder, filters, target and whether the target is prospective. +#[derive(Debug)] +pub struct ListArguments<'py> { + pub folder: String, + pub filters: Option>, + pub target: Option>, + pub target_is_prospective: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs rename to crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs index b2bfd88836..e6e78be935 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs @@ -12,7 +12,7 @@ use sqlbuild_analysis::compiler::models::{SqlTestExtractionFailure, SqlTestReque use sqlbuild_analysis::sql_test_attachment::main::authored_test_offset::authored_test_offset; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/mod.rs b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/mod.rs new file mode 100644 index 0000000000..b5cbd1d8ea --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod authored_ctes; diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/mod.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/mod.rs new file mode 100644 index 0000000000..cf178c6ba0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/mod.rs @@ -0,0 +1,3 @@ +pub mod prefixed_ctes; +pub mod register_authored_ctes; +pub mod validate_authored_sql_test_ctes; diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs new file mode 100644 index 0000000000..ee70b193f8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs @@ -0,0 +1,13 @@ +//! Extract fixture bodies from authored test or scenario CTEs in authored order. + +use crate::sql_test_glue::_helpers::authored_ctes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Extract fixture bodies from authored test or scenario CTEs in authored order. +#[inline] +pub fn prefixed_ctes<'py>(ctes: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { + authored_ctes::prefixed_ctes(ctes, prefix) +} diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs new file mode 100644 index 0000000000..06b795b1ca --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs @@ -0,0 +1,12 @@ +//! Register the authored ctes functions and classes. + +use crate::sql_test_glue::_helpers::authored_ctes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the authored ctes functions and classes. +#[inline] +pub fn register_authored_ctes(module: &Bound<'_, PyModule>) -> PyResult<()> { + authored_ctes::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs new file mode 100644 index 0000000000..4fb8a7c492 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs @@ -0,0 +1,19 @@ +//! Reject the first authored test block, then scenario, whose CTE names cannot be read. + +use crate::sql_test_glue::_helpers::authored_ctes; +use crate::sqltext::models::LexicalSyntaxInput; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// Reject the first authored test block, then scenario, whose CTE names cannot be read. +#[inline] +pub fn validate_authored_sql_test_ctes( + py: Python<'_>, + test_files: Bound<'_, PyAny>, + scenario_files: Bound<'_, PyAny>, + syntax: LexicalSyntaxInput, +) -> Result<(), PyErr> { + authored_ctes::validate_authored_sql_test_ctes(py, test_files, scenario_files, syntax) +} diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/mod.rs b/crates/sqlbuild-python-front/src/sql_test_glue/mod.rs new file mode 100644 index 0000000000..28fdd5543e --- /dev/null +++ b/crates/sqlbuild-python-front/src/sql_test_glue/mod.rs @@ -0,0 +1,4 @@ +//! Authored CTEs of SQL tests. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs new file mode 100644 index 0000000000..948a471284 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs @@ -0,0 +1,9 @@ +//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. + +use crate::sqltext::models::SqlCodeScanner; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs index e990f03a98..a31739eb01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs @@ -1,5 +1,6 @@ //! `@enum`/`@const` reference offsets scanned natively for the model loop. +use crate::sqltext::models::ScannedReference; use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; @@ -12,17 +13,8 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The reference kind of an outer SQL macro call. const SQL_MACRO_REFERENCE_KIND: u8 = 2; -/// One natively scanned reference: kind 0 is an enum member, 1 a constant, 2 a SQL macro call. -#[derive(Clone)] -pub(crate) struct ScannedReference { - pub(crate) kind: u8, - pub(crate) name: String, - pub(crate) member: Option, - pub(crate) start: usize, - pub(crate) end: usize, -} /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. -type ScanRow = (Vec, Option); +pub(crate) type ScanRow = (Vec, Option); /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs index 4fa4e68116..200ce9e1eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs @@ -1,6 +1,6 @@ //! One adapter's `SqlLexicalSyntax`, read from a Python mapping. -use pyo3::FromPyObject; +use crate::sqltext::models::LexicalSyntaxInput; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyFrozenSet; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -9,18 +9,6 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -/// One adapter's `SqlLexicalSyntax`, read from a Python mapping. -#[derive(Clone, FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct LexicalSyntaxInput { - backslash_escape_quotes: Vec, - escape_string_prefix: bool, - raw_string_prefix: bool, - triple_quoted_strings: bool, - nested_block_comments: bool, - line_comment_prefixes: Vec, -} - impl From for LexicalSyntax { fn from(input: LexicalSyntaxInput) -> Self { Self { diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs new file mode 100644 index 0000000000..4a2ee8426a --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod authored_values; +pub(crate) mod code_scanner; +pub(crate) mod ddl_effects; +pub(crate) mod declaration_references; +pub(crate) mod lexical_syntax; +pub(crate) mod model_headers; +pub(crate) mod placeholders; +pub(crate) mod sql_references; +pub(crate) mod static_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs index 0aa0183f3e..f8c8ecd8c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs @@ -1,20 +1,15 @@ //! Match, tokenize and parse model headers natively. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use crate::sqltext::models::ColumnLocation; +use crate::sqltext::types::ParsedModelHeader; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::sqltext::authored_values::optional_authored_value_to_python; +use crate::sqltext::_helpers::authored_values::optional_authored_value_to_python; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; -pub(crate) type ParsedModelHeader = ( - Option>, - Option>, - Option, -); - #[pyfunction] pub(crate) fn parse_model_headers( py: Python<'_>, @@ -37,18 +32,6 @@ pub(crate) fn parse_model_headers( .collect() } -positional_record! { - into; - /// A column name and its authored line, column, end line and end column. - pub(crate) struct ColumnLocation { - pub(crate) name: String, - pub(crate) line: usize, - pub(crate) column: usize, - pub(crate) end_line: usize, - pub(crate) end_column: usize, - } -} - pub(crate) fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/placeholders.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs index 6e04c994fb..c45eecade0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs @@ -12,7 +12,7 @@ use sqlbuild_sqltext::sql_references::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs rename to crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs index a85311b919..1e37cd63c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs @@ -1,5 +1,7 @@ //! Interpolate `@@` tokens in authored SQL and extract static SQL references. +use crate::sqltext::models::InterpolationRow; +use crate::sqltext::models::InterpolationSources; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::cell::RefCell; @@ -11,30 +13,9 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; - const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -/// One SQL text interpolated: its SQL, substitution spans, reads and error message. -pub(crate) struct InterpolationRow { - /// The interpolated SQL, or `None` when unchanged. - pub(crate) sql: Option, - pub(crate) spans: Vec, - pub(crate) reads: Vec<(&'static str, String)>, - pub(crate) error: Option, -} - -/// Variables, environment, context values (or `None`), variable renderer and lexical rules. -pub(crate) struct InterpolationSources<'py>( - pub(crate) Bound<'py, PyDict>, - pub(crate) Bound<'py, PyAny>, - pub(crate) Option>, - pub(crate) Bound<'py, PyAny>, - /// The lexical rules the SQL is authored in; `None` reads it under the generic rules. - pub(crate) Option, -); - struct PythonInterpolationHost<'py> { sources: InterpolationSources<'py>, /// A Python exception other than a rendering `ValueError`, raised after the scan. diff --git a/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs new file mode 100644 index 0000000000..4b26dde9b3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs @@ -0,0 +1,14 @@ +//! One authored value as the Python header object discovery hands to config builds. + +use crate::sqltext::_helpers::authored_values; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// One authored value as the Python header object discovery hands to config builds. +#[inline] +pub fn authored_value_to_python(py: Python<'_>, value: &AuthoredValue) -> PyResult> { + authored_values::authored_value_to_python(py, value) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs new file mode 100644 index 0000000000..78b1018b7c --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs @@ -0,0 +1,11 @@ +//! One SQL string's references and stop code, scanned without walking past malformed ones. + +use crate::sqltext::_helpers::declaration_references; +use crate::sqltext::_helpers::declaration_references::ScanRow; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// One SQL string's references and stop code, scanned without walking past malformed ones. +#[inline] +pub fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { + declaration_references::declaration_scan(sql, syntax) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs new file mode 100644 index 0000000000..dd8e5d84b4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs @@ -0,0 +1,16 @@ +//! Logical SQL refs outside comments and quotes, reporting rejected calls as P012. + +use crate::sqltext::_helpers::sql_references; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. +#[inline] +pub fn extract_references<'py>( + sql: &str, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { + sql_references::extract_references(sql, syntax) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs new file mode 100644 index 0000000000..3a0a42ef4a --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs @@ -0,0 +1,13 @@ +//! A header syntax error's detail, help and file line (when `header_line` is known). + +use crate::sqltext::_helpers::model_headers; + +/// A header syntax error's detail, help and file line (when `header_line` is known). +#[inline] +pub fn header_failure( + error: &str, + header: &str, + header_line: Option, +) -> (String, Option, Option) { + model_headers::header_failure(error, header, header_line) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs new file mode 100644 index 0000000000..37f5d41c82 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs @@ -0,0 +1,16 @@ +//! One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. + +use crate::sqltext::_helpers::static_sql; +use crate::sqltext::_helpers::static_sql::InterpolatedRow; +use crate::sqltext::models::InterpolationSources; +use pyo3::prelude::PyResult; + +/// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. +#[inline] +pub fn interpolated( + sources: InterpolationSources<'_>, + sql: &str, + file_path: &str, +) -> PyResult { + static_sql::interpolated(sources, sql, file_path) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs new file mode 100644 index 0000000000..5c13ea9fdf --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs @@ -0,0 +1,15 @@ +//! Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. + +use crate::sqltext::_helpers::static_sql; +use crate::sqltext::models::InterpolationRow; +use crate::sqltext::models::InterpolationSources; +use pyo3::prelude::PyResult; + +/// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. +#[inline] +pub fn interpolation_batch( + sources: InterpolationSources<'_>, + sqls: &[(String, String)], +) -> PyResult> { + static_sql::interpolation_batch(sources, sqls) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs new file mode 100644 index 0000000000..5788549395 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs @@ -0,0 +1,12 @@ +//! Location rows. + +use crate::sqltext::_helpers::model_headers; +use crate::sqltext::models::ColumnLocation; + +/// Location rows. +#[inline] +pub fn location_rows( + locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, +) -> Vec { + model_headers::location_rows(locations) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs new file mode 100644 index 0000000000..eb9e38b163 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs @@ -0,0 +1,14 @@ +//! Map to python. + +use crate::sqltext::_helpers::authored_values; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// Map to python. +#[inline] +pub fn map_to_python(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { + authored_values::map_to_python(py, values) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/mod.rs b/crates/sqlbuild-python-front/src/sqltext/main/mod.rs new file mode 100644 index 0000000000..7d472535a4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/mod.rs @@ -0,0 +1,20 @@ +pub mod authored_value_to_python; +pub mod declaration_scan; +pub mod extract_references; +pub mod header_failure; +pub mod interpolated; +pub mod interpolation_batch; +pub mod location_rows; +pub mod map_to_python; +pub mod model_output_columns; +pub mod optional_authored_value_to_python; +pub mod optional_record_syntax; +pub mod parse_model_headers; +pub mod record_syntax; +pub mod register_code_scanner; +pub mod register_ddl_effects; +pub mod register_model_headers; +pub mod register_placeholders; +pub mod register_sql_references; +pub mod sorted_texts; +pub mod syntax_record; diff --git a/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs new file mode 100644 index 0000000000..3c8c84f92b --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs @@ -0,0 +1,14 @@ +//! Simple top-level SELECT output names and their authored locations. + +use crate::sqltext::_helpers::model_headers; +use crate::sqltext::models::ColumnLocation; +use pyo3::prelude::PyResult; + +/// Simple top-level SELECT output names and their authored locations. +#[inline] +pub fn model_output_columns( + contents: &str, + implicit_aliases: bool, +) -> PyResult> { + model_headers::model_output_columns(contents, implicit_aliases) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs new file mode 100644 index 0000000000..6385780a54 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs @@ -0,0 +1,17 @@ +//! Optional authored value to python. + +use crate::sqltext::_helpers::authored_values; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// Optional authored value to python. +#[inline] +pub fn optional_authored_value_to_python( + py: Python<'_>, + value: Option<&AuthoredValue>, +) -> PyResult>> { + authored_values::optional_authored_value_to_python(py, value) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs new file mode 100644 index 0000000000..d329f2acc3 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs @@ -0,0 +1,13 @@ +//! The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. + +use crate::sqltext::_helpers::lexical_syntax; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. +#[inline] +pub fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { + lexical_syntax::optional_record_syntax(value) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs new file mode 100644 index 0000000000..068fe6c526 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs @@ -0,0 +1,15 @@ +//! Parse model headers. + +use crate::sqltext::_helpers::model_headers; +use crate::sqltext::types::ParsedModelHeader; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Parse model headers. +#[inline] +pub fn parse_model_headers( + py: Python<'_>, + headers: Vec, +) -> PyResult> { + model_headers::parse_model_headers(py, headers) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs new file mode 100644 index 0000000000..a990fdc113 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs @@ -0,0 +1,13 @@ +//! The lexical rules a `SqlLexicalSyntax` record holds. + +use crate::sqltext::_helpers::lexical_syntax; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// The lexical rules a `SqlLexicalSyntax` record holds. +#[inline] +pub fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { + lexical_syntax::record_syntax(instance) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs new file mode 100644 index 0000000000..f70b7bc051 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs @@ -0,0 +1,12 @@ +//! Register the code scanner functions and classes. + +use crate::sqltext::_helpers::code_scanner; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the code scanner functions and classes. +#[inline] +pub fn register_code_scanner(module: &Bound<'_, PyModule>) -> PyResult<()> { + code_scanner::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs new file mode 100644 index 0000000000..6ab4c7d18f --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs @@ -0,0 +1,12 @@ +//! Register the ddl effects functions and classes. + +use crate::sqltext::_helpers::ddl_effects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the ddl effects functions and classes. +#[inline] +pub fn register_ddl_effects(module: &Bound<'_, PyModule>) -> PyResult<()> { + ddl_effects::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs new file mode 100644 index 0000000000..abf259e94a --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs @@ -0,0 +1,12 @@ +//! Register the model headers functions and classes. + +use crate::sqltext::_helpers::model_headers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model headers functions and classes. +#[inline] +pub fn register_model_headers(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_headers::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs new file mode 100644 index 0000000000..e49345fc1b --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs @@ -0,0 +1,12 @@ +//! Register the placeholders functions and classes. + +use crate::sqltext::_helpers::placeholders; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the placeholders functions and classes. +#[inline] +pub fn register_placeholders(module: &Bound<'_, PyModule>) -> PyResult<()> { + placeholders::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs new file mode 100644 index 0000000000..98649fadff --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs @@ -0,0 +1,12 @@ +//! Register the sql references functions and classes. + +use crate::sqltext::_helpers::sql_references; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sql references functions and classes. +#[inline] +pub fn register_sql_references(module: &Bound<'_, PyModule>) -> PyResult<()> { + sql_references::register(module) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs new file mode 100644 index 0000000000..f16064c9b8 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs @@ -0,0 +1,12 @@ +//! The sorted texts of a set of strings. + +use crate::sqltext::_helpers::lexical_syntax; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The sorted texts of a set of strings. +#[inline] +pub fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { + lexical_syntax::sorted_texts(values) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs new file mode 100644 index 0000000000..6cc6dfdb00 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs @@ -0,0 +1,14 @@ +//! A `SqlLexicalSyntax` record holding `syntax`. + +use crate::sqltext::_helpers::lexical_syntax; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// A `SqlLexicalSyntax` record holding `syntax`. +#[inline] +pub fn syntax_record<'py>(py: Python<'py>, syntax: &LexicalSyntax) -> PyResult> { + lexical_syntax::syntax_record(py, syntax) +} diff --git a/crates/sqlbuild-python-front/src/sqltext/mod.rs b/crates/sqlbuild-python-front/src/sqltext/mod.rs new file mode 100644 index 0000000000..9b314966b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/mod.rs @@ -0,0 +1,6 @@ +//! Lexical SQL text: code scanning, references, placeholders, model headers and static SQL. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-front/src/sqltext/models.rs b/crates/sqlbuild-python-front/src/sqltext/models.rs new file mode 100644 index 0000000000..7ebb26aa9f --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/models.rs @@ -0,0 +1,140 @@ +//! Public records of sqltext. + +use crate::render::types::SpanRow; +use pyo3::FromPyObject; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_sqltext::code_scanner::main::code_scanner::code_scanner; +use sqlbuild_sqltext::code_scanner::main::scanner_matching_paren::scanner_matching_paren; +use sqlbuild_sqltext::code_scanner::main::scanner_non_code_end::scanner_non_code_end; +use sqlbuild_sqltext::code_scanner::models::CodeScanner; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +/// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct SqlCodeScanner { + pub(crate) scanner: CodeScanner, + pub(crate) context: String, +} + +#[pymethods] +impl SqlCodeScanner { + #[new] + #[pyo3(signature = (sql, syntax, context))] + pub fn new(sql: &str, syntax: Option, context: String) -> Self { + Self { + scanner: code_scanner(sql, syntax.map(Into::into)), + context, + } + } + + /// The end of the comment or quoted text at `index`, or `None` when code starts there. + pub fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { + scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) + } + + /// The index of the parenthesis closing the one at `open`. + pub fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { + scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) + } +} + +impl SqlCodeScanner { + /// `CompileInputError(" contains an unclosed ...")`. + pub(crate) fn error(&self, py: Python<'_>, unclosed: Unclosed) -> pyo3::prelude::PyErr { + let construct: &str = match unclosed { + Unclosed::BlockComment => "block comment", + Unclosed::Quote => "quoted string", + Unclosed::Parenthesis => "parenthesis", + }; + let message: String = format!("{} contains an unclosed {construct}", self.context); + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = + error_class(py, ErrorId::CompileInput)?.call1((message,))?; + Ok(pyo3::prelude::PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) + } +} + +/// One natively scanned reference: kind 0 is an enum member, 1 a constant, 2 a SQL macro call. +#[derive(Debug, Clone)] +pub struct ScannedReference { + pub kind: u8, + pub name: String, + pub member: Option, + pub start: usize, + pub end: usize, +} + +/// One adapter's `SqlLexicalSyntax`, read from a Python mapping. +#[derive(Debug, Clone, FromPyObject)] +#[pyo3(from_item_all)] +pub struct LexicalSyntaxInput { + pub(crate) backslash_escape_quotes: Vec, + pub(crate) escape_string_prefix: bool, + pub(crate) raw_string_prefix: bool, + pub(crate) triple_quoted_strings: bool, + pub(crate) nested_block_comments: bool, + pub(crate) line_comment_prefixes: Vec, +} + +/// A column name and its authored line, column, end line and end column. +#[derive(Debug)] +pub struct ColumnLocation { + pub name: String, + pub line: usize, + pub column: usize, + pub end_line: usize, + pub end_column: usize, +} + +impl<'py> pyo3::IntoPyObject<'py> for ColumnLocation { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.name, + self.line, + self.column, + self.end_line, + self.end_column, + ), + py, + ) + } +} + +/// One SQL text interpolated: its SQL, substitution spans, reads and error message. +#[derive(Debug)] +pub struct InterpolationRow { + /// The interpolated SQL, or `None` when unchanged. + pub sql: Option, + pub spans: Vec, + pub reads: Vec<(&'static str, String)>, + pub error: Option, +} + +/// Variables, environment, context values (or `None`), variable renderer and lexical rules. +#[derive(Debug)] +pub struct InterpolationSources<'py>( + pub Bound<'py, PyDict>, + pub Bound<'py, PyAny>, + pub Option>, + pub Bound<'py, PyAny>, + /// The lexical rules the SQL is authored in; `None` reads it under the generic rules. + pub Option, +); diff --git a/crates/sqlbuild-python-front/src/sqltext/types.rs b/crates/sqlbuild-python-front/src/sqltext/types.rs new file mode 100644 index 0000000000..bac8d76df6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/sqltext/types.rs @@ -0,0 +1,10 @@ +//! Public traits and type aliases of sqltext. + +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; + +pub type ParsedModelHeader = ( + Option>, + Option>, + Option, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs index b3082aad36..58a33d0726 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs @@ -16,7 +16,8 @@ use sqlbuild_state::events::main::migration_event_id::migration_event_id; use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; use sqlbuild_state::events::models::{EventRelation, StateCell, StateColumnType, StateValue}; -use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, presumed_utc}; +use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; +use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs index 5264dcd08a..e3e00b6dba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs @@ -19,10 +19,12 @@ use sqlbuild_state::freshness::models::{ }; use sqlbuild_state::freshness::types::TimestampParser; -use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; -use crate::bindings::_helpers::state_rows::timestamps::{ - date_time_typed, in_utc, offset_aware, presumed_utc, -}; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; +use crate::state_rows::_helpers::event_rows::timestamps::in_utc; +use crate::state_rows::_helpers::event_rows::timestamps::offset_aware; +use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs new file mode 100644 index 0000000000..fd6367a2fd --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod events; +pub(crate) mod freshness; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs index 6629ea2924..645abff126 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_cursor::algebra::models::LocalDateTime; -use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::values::local_date_time::local_date_time; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs index aabedd2d5f..94b847285c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs @@ -10,19 +10,27 @@ use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::recorded_renames::recorded_renames; use sqlbuild_state::events::models::RecordedRenameKey; -use crate::bindings::_helpers::state_rows::constants::{ - COLUMN_KIND, LOGGING_MODULE, MIGRATION_LOGGER, MIGRATION_WRITE_ATTEMPTS, - MIGRATION_WRITE_RETRY_BASE_SECONDS, MIGRATION_WRITE_RETRY_MESSAGE, MODEL_KIND, OLD_NAME_KIND, - TIME_MODULE, -}; -use crate::bindings::_helpers::state_rows::events::relation as event_relation; -use crate::bindings::_helpers::state_rows::migration_objects::{event_order, layout, state_error}; -use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; -use crate::bindings::_helpers::state_rows::migration_rows::{ - decoded_events, read_statement, write_statements, -}; -use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout, StateAccess}; -use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_objects::layout; +use crate::state_rows::_helpers::migrations::migration_objects::state_error; +use crate::state_rows::_helpers::migrations::migration_projections::old_name_views; +use crate::state_rows::_helpers::migrations::migration_rows::decoded_events; +use crate::state_rows::_helpers::migrations::migration_rows::read_statement; +use crate::state_rows::_helpers::migrations::migration_rows::write_statements; +use crate::state_rows::constants::COLUMN_KIND; +use crate::state_rows::constants::LOGGING_MODULE; +use crate::state_rows::constants::MIGRATION_LOGGER; +use crate::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; +use crate::state_rows::constants::MIGRATION_WRITE_RETRY_BASE_SECONDS; +use crate::state_rows::constants::MIGRATION_WRITE_RETRY_MESSAGE; +use crate::state_rows::constants::MODEL_KIND; +use crate::state_rows::constants::OLD_NAME_KIND; +use crate::state_rows::constants::TIME_MODULE; +use crate::state_rows::models::EventTable; +use crate::state_rows::models::EventTableLayout; +use crate::state_rows::models::StateAccess; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -187,7 +195,7 @@ fn append_event( Ok(()) } -fn access<'py>(request: &Request<'_, 'py>) -> PyResult> { +pub(crate) fn access<'py>(request: &Request<'_, 'py>) -> PyResult> { Ok(StateAccess { connection: request.get("connection")?, execute: request.get("execute")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs index f477b7d8b4..64fdb804f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs @@ -12,9 +12,12 @@ use sqlbuild_state::events::constants::{ }; use sqlbuild_state::events::models::EventOrder; -use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; -use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; -use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; +use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; +use crate::state_rows::constants::COLUMN_KIND; +use crate::state_rows::constants::MODEL_KIND; +use crate::state_rows::constants::OLD_NAME_KIND; +use crate::state_rows::models::EventTable; +use crate::state_rows::models::EventTableLayout; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs index 8119870cf4..e2e0dc9087 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs @@ -11,9 +11,9 @@ use sqlbuild_state::events::models::{ ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, }; -use crate::bindings::_helpers::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; -use crate::bindings::_helpers::state_rows::events::relation as event_relation; -use crate::bindings::_helpers::state_rows::migration_objects::event_order; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs index 05f2bf62b7..77326fddcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs @@ -15,14 +15,15 @@ use sqlbuild_state::events::main::event_read_sql::event_read_sql; use sqlbuild_state::events::main::grants_json::grants_json; use sqlbuild_state::events::main::readable_columns::readable_columns; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; -use crate::bindings::_helpers::state_rows::events::typed_insert_sql; -use crate::bindings::_helpers::state_rows::migration_objects::{ - layout, qualified_table, state_error, -}; -use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; -use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; -use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; +use crate::cursor::constants::DATETIME_MODULE; +use crate::state_rows::_helpers::event_rows::events::typed_insert_sql; +use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; +use crate::state_rows::_helpers::migrations::migration_objects::layout; +use crate::state_rows::_helpers::migrations::migration_objects::qualified_table; +use crate::state_rows::_helpers::migrations::migration_objects::state_error; +use crate::state_rows::_helpers::tables::state_tables::state_table_create_sql; +use crate::state_rows::models::EventTable; +use crate::state_rows::models::EventTableLayout; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/mod.rs new file mode 100644 index 0000000000..aa91bc5990 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod migration_io; +pub(crate) mod migration_objects; +pub(crate) mod migration_projections; +pub(crate) mod migration_rows; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/mod.rs new file mode 100644 index 0000000000..c980d67515 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod event_rows; +pub(crate) mod migrations; +pub(crate) mod tables; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/mod.rs new file mode 100644 index 0000000000..dbb106aba0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/mod.rs @@ -0,0 +1 @@ +pub(crate) mod state_tables; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs rename to crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs index c3769c7844..a5f914e949 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs @@ -16,7 +16,7 @@ use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::bindings::_helpers::state_rows::freshness::record_event_id; +use crate::state_rows::_helpers::event_rows::freshness::record_event_id; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python-front/src/state_rows/constants.rs similarity index 61% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs rename to crates/sqlbuild-python-front/src/state_rows/constants.rs index adf8039338..f7044b0732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python-front/src/state_rows/constants.rs @@ -1,13 +1,22 @@ -//! Python modules, names and messages of migration state rows. +//! Public constants of state rows. pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; -pub(crate) const MODEL_KIND: &str = "model"; -pub(crate) const COLUMN_KIND: &str = "column"; -pub(crate) const OLD_NAME_KIND: &str = "old_name"; + +pub const MODEL_KIND: &str = "model"; + +pub const COLUMN_KIND: &str = "column"; + +pub const OLD_NAME_KIND: &str = "old_name"; + pub(crate) const MIGRATION_LOGGER: &str = "sqlbuild.migrations"; + pub(crate) const LOGGING_MODULE: &str = "logging"; + pub(crate) const TIME_MODULE: &str = "time"; -pub(crate) const MIGRATION_WRITE_ATTEMPTS: i64 = 3; + +pub const MIGRATION_WRITE_ATTEMPTS: i64 = 3; + pub(crate) const MIGRATION_WRITE_RETRY_BASE_SECONDS: f64 = 0.05; + pub(crate) const MIGRATION_WRITE_RETRY_MESSAGE: &str = "migration event write attempt %s/%s failed for '%s'; retrying: %s"; diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs new file mode 100644 index 0000000000..994837c021 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs @@ -0,0 +1,16 @@ +//! Optional timestamp age policy for one source freshness observation. + +use crate::state_rows::_helpers::event_rows::freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Optional timestamp age policy for one source freshness observation. +#[inline] +pub fn age_policy<'py>( + policy: &Bound<'py, PyAny>, + data_version: &Bound<'py, PyAny>, + observed_at: &Bound<'py, PyAny>, +) -> PyResult>> { + freshness::age_policy(policy, data_version, observed_at) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs new file mode 100644 index 0000000000..7215ae0fd5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs @@ -0,0 +1,12 @@ +//! Whether `value` is a `datetime`. + +use crate::state_rows::_helpers::event_rows::timestamps; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether `value` is a `datetime`. +#[inline] +pub fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { + timestamps::date_time_typed(value) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs new file mode 100644 index 0000000000..cb8bd180b5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs @@ -0,0 +1,16 @@ +//! Whether two source freshness records are equivalent for skip decisions. + +use crate::state_rows::_helpers::event_rows::freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether two source freshness records are equivalent for skip decisions. +#[inline] +pub fn equivalent( + previous_record: &Bound<'_, PyAny>, + current_record: &Bound<'_, PyAny>, + lag_tolerance: Option<&str>, +) -> PyResult { + freshness::equivalent(previous_record, current_record, lag_tolerance) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs new file mode 100644 index 0000000000..8d28e7e853 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs @@ -0,0 +1,12 @@ +//! `value.astimezone(UTC)`. + +use crate::state_rows::_helpers::event_rows::timestamps; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `value.astimezone(UTC)`. +#[inline] +pub fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + timestamps::in_utc(value) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/mod.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/mod.rs new file mode 100644 index 0000000000..261c2dc766 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/mod.rs @@ -0,0 +1,10 @@ +pub mod age_policy; +pub mod date_time_typed; +pub mod equivalent; +pub mod in_utc; +pub mod normalize; +pub mod offset_aware; +pub mod presumed_utc; +pub mod register_events; +pub mod register_freshness; +pub mod version_hash; diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs new file mode 100644 index 0000000000..ec853a1552 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs @@ -0,0 +1,15 @@ +//! A source freshness value normalized for stable state storage. + +use crate::state_rows::_helpers::event_rows::freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A source freshness value normalized for stable state storage. +#[inline] +pub fn normalize<'py>( + value: &Bound<'py, PyAny>, + value_kind: &Bound<'py, PyAny>, +) -> PyResult> { + freshness::normalize(value, value_kind) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs new file mode 100644 index 0000000000..88ffd3edd5 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs @@ -0,0 +1,12 @@ +//! Whether `value` has a `tzinfo` that reports an offset. + +use crate::state_rows::_helpers::event_rows::timestamps; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether `value` has a `tzinfo` that reports an offset. +#[inline] +pub fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { + timestamps::offset_aware(value) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs new file mode 100644 index 0000000000..9d8d99beed --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs @@ -0,0 +1,13 @@ +//! The UTC wall time of `value`, reading a naive or offset-less value as UTC. + +use crate::state_rows::_helpers::event_rows::timestamps; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// The UTC wall time of `value`, reading a naive or offset-less value as UTC. +#[inline] +pub fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { + timestamps::presumed_utc(value) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs new file mode 100644 index 0000000000..530de6df36 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs @@ -0,0 +1,12 @@ +//! Register the events functions and classes. + +use crate::state_rows::_helpers::event_rows::events; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the events functions and classes. +#[inline] +pub fn register_events(module: &Bound<'_, PyModule>) -> PyResult<()> { + events::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs new file mode 100644 index 0000000000..61581e44e6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs @@ -0,0 +1,12 @@ +//! Register the freshness functions and classes. + +use crate::state_rows::_helpers::event_rows::freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the freshness functions and classes. +#[inline] +pub fn register_freshness(module: &Bound<'_, PyModule>) -> PyResult<()> { + freshness::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs new file mode 100644 index 0000000000..544e6287c0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs @@ -0,0 +1,15 @@ +//! Hash the stable source freshness identity used as a graph input. + +use crate::state_rows::_helpers::event_rows::freshness; +use pyo3::prelude::PyResult; + +/// Hash the stable source freshness identity used as a graph input. +#[inline] +pub fn version_hash( + source_name: &str, + strategy: &str, + value_kind: &str, + data_version: &str, +) -> PyResult { + freshness::version_hash(source_name, strategy, value_kind, data_version) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs new file mode 100644 index 0000000000..caae386fb9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs @@ -0,0 +1,11 @@ +//! The layout of the state table `kind` names: `model`, `column` or `old_name`. + +use crate::state_rows::_helpers::migrations::migration_objects; +use crate::state_rows::models::EventTableLayout; +use pyo3::prelude::PyResult; + +/// The layout of the state table `kind` names: `model`, `column` or `old_name`. +#[inline] +pub fn layout(kind: &str) -> PyResult { + migration_objects::layout(kind) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/mod.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/mod.rs new file mode 100644 index 0000000000..58584513db --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/mod.rs @@ -0,0 +1,12 @@ +pub mod layout; +pub mod newest_column; +pub mod newest_move; +pub mod old_name_retention; +pub mod old_name_views; +pub mod read_events; +pub mod read_histories; +pub mod register_migration_io; +pub mod register_migration_projections; +pub mod register_migration_rows; +pub mod stored_columns_binding; +pub mod stored_old_name_columns; diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs new file mode 100644 index 0000000000..aa92be6ad4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs @@ -0,0 +1,16 @@ +//! The latest recorded rename on `relation` naming either column, or `None`. + +use crate::state_rows::_helpers::migrations::migration_projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The latest recorded rename on `relation` naming either column, or `None`. +#[inline] +pub fn newest_column<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + (origin_column, destination_column): (&str, &str), +) -> PyResult> { + migration_projections::newest_column(events, relation, (origin_column, destination_column)) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs new file mode 100644 index 0000000000..c24e453093 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs @@ -0,0 +1,15 @@ +//! The latest recorded move naming `relation` as origin or destination, or `None`. + +use crate::state_rows::_helpers::migrations::migration_projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The latest recorded move naming `relation` as origin or destination, or `None`. +#[inline] +pub fn newest_move<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + migration_projections::newest_move(events, relation) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs new file mode 100644 index 0000000000..6396c515f7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs @@ -0,0 +1,15 @@ +//! The model's old-name view retention, the project's when unset, or `None` when disabled. + +use crate::state_rows::_helpers::migrations::migration_projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's old-name view retention, the project's when unset, or `None` when disabled. +#[inline] +pub fn old_name_retention<'py>( + config_values: &Bound<'py, PyAny>, + project_retention: &Bound<'py, PyAny>, +) -> PyResult> { + migration_projections::old_name_retention(config_values, project_retention) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs new file mode 100644 index 0000000000..92d839abeb --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs @@ -0,0 +1,16 @@ +//! One history per recorded move whose old name needs a compatibility view, in move order. + +use crate::state_rows::_helpers::migrations::migration_projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// One history per recorded move whose old name needs a compatibility view, in move order. +#[inline] +pub fn old_name_views<'py>( + moves: &Bound<'py, PyAny>, + facts: &Bound<'py, PyAny>, +) -> PyResult> { + migration_projections::old_name_views(moves, facts) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs new file mode 100644 index 0000000000..44beac2afc --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs @@ -0,0 +1,20 @@ +//! Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. + +use crate::state_rows::_helpers::migrations::migration_io; +use crate::state_rows::models::EventTableLayout; +use crate::state_rows::models::StateAccess; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. +#[inline] +pub fn read_events<'py>( + access: &StateAccess<'py>, + layout: &EventTableLayout, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&[String]>, +) -> PyResult> { + migration_io::read_events(access, layout, (database, schema), stored) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs new file mode 100644 index 0000000000..1c946a2d48 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs @@ -0,0 +1,13 @@ +//! The old-name view history of every recorded move in a schema that holds both state tables. + +use crate::state_rows::_helpers::migrations::migration_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// The old-name view history of every recorded move in a schema that holds both state tables. +#[inline] +pub fn read_histories<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + migration_io::read_histories(request) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs new file mode 100644 index 0000000000..90b03f6d18 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs @@ -0,0 +1,12 @@ +//! Register the migration io functions and classes. + +use crate::state_rows::_helpers::migrations::migration_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the migration io functions and classes. +#[inline] +pub fn register_migration_io(module: &Bound<'_, PyModule>) -> PyResult<()> { + migration_io::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs new file mode 100644 index 0000000000..65e9d95e8d --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs @@ -0,0 +1,12 @@ +//! Register the migration projections functions and classes. + +use crate::state_rows::_helpers::migrations::migration_projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the migration projections functions and classes. +#[inline] +pub fn register_migration_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { + migration_projections::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs new file mode 100644 index 0000000000..8a095b7b1e --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs @@ -0,0 +1,12 @@ +//! Register the migration rows functions and classes. + +use crate::state_rows::_helpers::migrations::migration_rows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the migration rows functions and classes. +#[inline] +pub fn register_migration_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { + migration_rows::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs new file mode 100644 index 0000000000..dfd775e2ba --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs @@ -0,0 +1,18 @@ +//! The column names of one schema's old-name view state table. + +use crate::state_rows::_helpers::migrations::migration_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// The column names of one schema's old-name view state table. +#[inline] +pub fn stored_columns_binding<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, +) -> PyResult> { + migration_io::stored_columns_binding(adapter, connection, database, schema) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs new file mode 100644 index 0000000000..6c29967353 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs @@ -0,0 +1,16 @@ +//! The column names of one schema's old-name view state table, as the adapter lists them. + +use crate::state_rows::_helpers::migrations::migration_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The column names of one schema's old-name view state table, as the adapter lists them. +#[inline] +pub fn stored_old_name_columns<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + migration_io::stored_old_name_columns(adapter, connection, (database, schema)) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/mod.rs b/crates/sqlbuild-python-front/src/state_rows/main/mod.rs new file mode 100644 index 0000000000..a9d30d5bd4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/mod.rs @@ -0,0 +1,3 @@ +pub mod event_rows; +pub mod migrations; +pub mod tables; diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/mod.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/mod.rs new file mode 100644 index 0000000000..cfe5add1dc --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/mod.rs @@ -0,0 +1,2 @@ +pub mod register_state_tables; +pub mod state_table_create_sql; diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs new file mode 100644 index 0000000000..db283b6c98 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs @@ -0,0 +1,12 @@ +//! Register the state tables functions and classes. + +use crate::state_rows::_helpers::tables::state_tables; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the state tables functions and classes. +#[inline] +pub fn register_state_tables(module: &Bound<'_, PyModule>) -> PyResult<()> { + state_tables::register(module) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs new file mode 100644 index 0000000000..20b153bb59 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs @@ -0,0 +1,24 @@ +//! Create-if-missing DDL whose column types the adapter renders. + +use crate::state_rows::_helpers::tables::state_tables; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Create-if-missing DDL whose column types the adapter renders. +#[inline] +pub fn state_table_create_sql<'py>( + qualified_name: &str, + (columns, column_types, required_columns): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult { + state_tables::state_table_create_sql( + qualified_name, + (columns, column_types, required_columns), + render_framework_type, + ) +} diff --git a/crates/sqlbuild-python-front/src/state_rows/mod.rs b/crates/sqlbuild-python-front/src/state_rows/mod.rs new file mode 100644 index 0000000000..22cf001f32 --- /dev/null +++ b/crates/sqlbuild-python-front/src/state_rows/mod.rs @@ -0,0 +1,6 @@ +//! Warehouse state rows: events, freshness, migrations, timestamps and the state tables. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs b/crates/sqlbuild-python-front/src/state_rows/models.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs rename to crates/sqlbuild-python-front/src/state_rows/models.rs index b69ffba5c7..542ffd658c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs +++ b/crates/sqlbuild-python-front/src/state_rows/models.rs @@ -1,6 +1,7 @@ -//! The layout of one append-only migration state table as the bindings use it. +//! Public records of state rows. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; /// Which migration state table a request reads or writes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -12,8 +13,8 @@ pub(crate) enum EventTable { /// Name, columns, event relation attribute and messages of one migration state table. #[derive(Clone, Copy, Debug)] -pub(crate) struct EventTableLayout { - pub table: EventTable, +pub struct EventTableLayout { + pub(crate) table: EventTable, pub name: &'static str, pub columns: &'static [&'static str], pub relation_attribute: &'static str, @@ -24,7 +25,7 @@ pub(crate) struct EventTableLayout { /// How migration state reaches the warehouse: the connection, execute callable and name renderer. #[derive(Debug)] -pub(crate) struct StateAccess<'py> { +pub struct StateAccess<'py> { pub connection: Bound<'py, PyAny>, pub execute: Bound<'py, PyAny>, pub render_qualified_name: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index b7304f2a1d..e31e9d0f07 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -41,6 +41,7 @@ sqlbuild-orchestration.workspace = true sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true @@ -49,7 +50,6 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true -stacker.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 256fbc8e10..730a898710 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::analysis_session::session::{ CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, start_model_analysis_session, }; -use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; @@ -36,6 +35,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; const LOGGING_MODULE: &str = "logging"; const DEBUG_LOGGER: &str = "sqlbuild.compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index ef9222d133..68bc94ec1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::analysis_session::session::{ ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, }; -use crate::bindings::_helpers::model_config::header_families::family_specs; use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; const BATCH_MODE: &str = "batch"; const TABLE_FUNCTION_KIND: &str = "table_fn"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs index 06178594d6..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs @@ -1,6 +1 @@ -pub(crate) mod argument_values; -pub(crate) mod ceremonial_select; -pub(crate) mod cursor_intrinsics; -pub(crate) mod function_headers; -pub(crate) mod reference_markers; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs index 2f62e267e3..0ad0079092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs @@ -2,13 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::attachments::{ - ceremonial_select, cursor_intrinsics, reference_markers, -}; +use sqlbuild_python_front::attachments::main::register_ceremonial_select::register_ceremonial_select; +use sqlbuild_python_front::attachments::main::register_cursor_intrinsics::register_cursor_intrinsics; +use sqlbuild_python_front::attachments::main::register_reference_markers::register_reference_markers; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_intrinsics::register(module)?; - ceremonial_select::register(module)?; - reference_markers::register(module)?; + register_cursor_intrinsics(module)?; + register_ceremonial_select(module)?; + register_reference_markers(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 84e2457856..1fb668c0c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -19,7 +19,6 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs index ec3d9abff9..50b57c771c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs @@ -9,12 +9,13 @@ use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use sqlbuild_model_config::header_metadata::main::parse_audit_list::parse_audit_list; use sqlbuild_model_config::header_metadata::models::ParsedAudit; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::contract_classes::contract_classes; -use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::model_config::main::headers::audit_tuple::audit_tuple; +use sqlbuild_python_front::model_config::main::model_nodes::contract_classes::contract_classes; +use sqlbuild_python_front::model_config::models::ContractClasses; +use sqlbuild_python_front::model_config::models::PyNode; /// A resource identity error as `ResourceIdentityError`, anything else as `fallback_class(message)`. fn raised( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs index b74d618669..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs @@ -1,5 +1 @@ -mod fields; -pub(crate) mod project_vars; -mod python_nodes; pub(crate) mod registration; -pub(crate) mod sql_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs index c1c11ce4c5..7369430883 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs @@ -1,7 +1,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::authored_values::fields::register(module)?; - crate::bindings::_helpers::authored_values::project_vars::register(module)?; - crate::bindings::_helpers::authored_values::python_nodes::register(module) + sqlbuild_python_front::authored_values::main::register_fields::register_fields(module)?; + sqlbuild_python_front::authored_values::main::register_project_vars::register_project_vars( + module, + )?; + sqlbuild_python_front::authored_values::main::register_python_nodes::register_python_nodes( + module, + ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 1ecd6227f4..8e3408df42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -20,9 +20,6 @@ use crate::bindings::_helpers::build_command::planning::{ }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; -use crate::bindings::_helpers::discovery::provider_secrets::{ - consumer_uses, provider_uses, require_used_secrets, sink_uses, -}; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, @@ -35,6 +32,10 @@ use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::diagnostics::models::BuildPhaseTimingTracker; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; +use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; +use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; +use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 4ee06a5390..dd9b0fb38d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -21,7 +21,6 @@ use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::policies::enforce_model_execution_limit_binding; use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; @@ -34,6 +33,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; const VIEW_MATERIALIZATION: &str = "view"; const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 294c8c5da0..59374d9439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -32,10 +32,8 @@ use crate::bindings::_helpers::consumption::recorder::{ ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, }; use crate::bindings::_helpers::cost_telemetry::context::CostContext; -use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; @@ -59,6 +57,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; +use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; +use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; use sqlbuild_state::consumption::models::ChangedPolicy; const EXECUTION_COMMAND: &str = "sqb build"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 5556b457d0..d5ae29884b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -12,15 +12,9 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -32,6 +26,12 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_front::discovery::models::SecretPolicy; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index bf71b84c81..2fee2b0bf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -25,10 +25,10 @@ use crate::bindings::_helpers::change_selection::base_cache::{ use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; use crate::bindings::_helpers::change_selection::environment_nodes::with_base_dependencies; use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; -use crate::bindings::_helpers::discovery::project_config::configured_git_base_ref; use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; use sqlbuild_python_base::planner_identity::constants::COMPACT; use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; +use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; /// Where base compiles work when the project's cache directory cannot be created. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index b843473116..0982412eeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -8,9 +8,6 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::discovery::provider_secrets::{ - consumer_uses, require_used_secrets, sink_uses, -}; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; @@ -36,6 +33,9 @@ use sqlbuild_python_base::native_host::main::connection_failure::connection_fail use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; +use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; +use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use crate::bindings::_helpers::check::check_core::{ ReadSideInputs, load_results_by_loader_name, record_python_run_state_results, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 18af7aaf5c..00ea711da8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::dbt_command::init::run_dbt_init_command; use crate::bindings::_helpers::debug_command::run_debug::run_debug_command; use crate::bindings::_helpers::diff_report::command::run_diff_command; use crate::bindings::_helpers::diff_report::support::cli_error; -use crate::bindings::_helpers::discovery::provider_secrets::MissingSecretsScope; use crate::bindings::_helpers::format_command::command::run_format_command; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::init_command::command::run_init_command; @@ -57,6 +56,7 @@ use sqlbuild_core::format_command::main::format_path_selectors::format_path_sele use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_front::discovery::models::MissingSecretsScope; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 662970cd50..a8e12a58bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -28,10 +28,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor::registration::register(module)?; plan_command::registration::register(module)?; freshness::registration::register(module)?; - crate::bindings::_helpers::native_host::consumption_state::register(module)?; - crate::bindings::_helpers::native_host::observations_state::register(module)?; + sqlbuild_python_front::native_host::main::register_consumption_state::register_consumption_state(module)?; + sqlbuild_python_front::native_host::main::register_observations_state::register_observations_state(module)?; crate::bindings::_helpers::native_host::resource_costs_state::register(module)?; - crate::bindings::_helpers::native_host::audit_results_state::register(module)?; + sqlbuild_python_front::native_host::main::register_audit_results_state::register_audit_results_state(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index a9b9a09989..f4844e130a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -27,9 +27,6 @@ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, timed_clon use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; @@ -48,6 +45,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; const CLONE_LOGGER: &str = "sqlbuild.clone"; /// The clone phase that copies relations into the destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 0e1168cf78..2820d658f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; @@ -24,6 +23,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 331227a146..184caf11c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -5,13 +5,13 @@ use pyo3::types::{PyBool, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::planner_entries::clone_plans::{PhaseTimings, timed_clone_planning}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; /// The phase name of one target compile, before the target name. const COMPILE_PHASE_PREFIX: &str = "compile_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 2fdb7a55d1..95d3841303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -19,15 +19,15 @@ use sqlbuild_host::entries::models::ThirdPartyHost; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::command_runtime::warehouses::{WarehouseInputs, apply_warehouse}; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::dbt_interop::profile_connection::resolve_raw_dbt_profile_connection; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; const ATTACH_KEY: &str = "attach"; const ATTACH_PATH_KEY: &str = "path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 9120cbd84a..8590c9746b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -7,8 +7,6 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -17,6 +15,8 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::models::SecretPolicy; /// The project directory, discovered inputs, adapter context, colour and progress stream. struct InvocationContext<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index 9e2bda19bc..4d947aefd7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod concurrency; pub(crate) mod connection; pub(crate) mod invocation; pub(crate) mod preview_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index fab0bb4d15..20515e7b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -22,17 +22,17 @@ use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, raised, }; use crate::bindings::_helpers::command_runtime::warehouse_scope::current_command_warehouse_scope; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; -use crate::bindings::_helpers::compile_driver::config_validation::session_warehouse_support; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; -use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; -use crate::bindings::_helpers::model_config::template_data::expanded_template_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; +use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; /// The `WarehouseGroup` whose warehouse is a target's `warehouses.build`; any other reads `query`. const BUILD_GROUP: &str = "build"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index 89247cc49e..c6399918c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -4,13 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyTuple; -use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; -use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; -use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; -use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; -use crate::bindings::_helpers::sql_test_glue::authored_ctes::validate_authored_sql_test_ctes; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::models::SqlTestScanCache; @@ -22,6 +16,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::render::main::declarations::declaration_scope_resolver::declaration_scope_resolver; +use sqlbuild_python_front::scopes::main::checks::validate_scope_index::validate_scope_index; +use sqlbuild_python_front::scopes::main::index::build_scope_lookup::build_scope_lookup; +use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_native_scope_index; +use sqlbuild_python_front::sql_test_glue::main::validate_authored_sql_test_ctes::validate_authored_sql_test_ctes; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index 5737802a12..a5d3007cac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod cursor_roles; pub(crate) mod declaration_scope; -pub(crate) mod effective_config; pub(crate) mod model_config; pub(crate) mod model_inputs; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index bf8764fa89..d945fb86d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -6,13 +6,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; use crate::bindings::_helpers::input_loops::loop_classes::table_backed; -use crate::bindings::_helpers::model_config::model_config_builder::NativeModelConfigBuilder; -use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::model_nodes::unicode_environment::unicode_environment; +use sqlbuild_python_front::model_config::models::NativeModelConfigBuilder; /// Capture the project inputs every model's native config build and validation read. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index b49c068239..0d2924508e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -10,18 +10,11 @@ use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_a use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::loop_classes::model_loop_classes; use crate::bindings::_helpers::input_loops::model_loop::{LoopSwitches, NativeModelLoop}; -use crate::bindings::_helpers::model_config::contract_classes::contract_classes; -use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; -use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -use crate::bindings::_helpers::render::declaration_values::ConstantRendering; -use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -29,6 +22,13 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_front::attachments::main::reserved_cursor_markers::reserved_cursor_markers; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::model_nodes::contract_classes::contract_classes; +use sqlbuild_python_front::model_config::main::model_nodes::unicode_environment::unicode_environment; +use sqlbuild_python_front::model_config::models::NativeModelValidator; +use sqlbuild_python_front::render::main::expansion::macro_expansion_classes::macro_expansion_classes; +use sqlbuild_python_front::render::models::ConstantRendering; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -36,6 +36,8 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::authored_values::main::render_project_var_text_function::render_project_var_text_function; +use sqlbuild_python_front::model_config::main::config::record_template_reads_function::record_template_reads_function; /// The explicit reference call kinds and the `SqlResourceRefKind` member each resolves to. const EXPLICIT_REFERENCE_KINDS: [(&str, &str); 3] = [("ref", "MODEL"), ("source", "SOURCE"), ("seed", "SEED")]; @@ -138,7 +140,7 @@ fn sql_hook_inputs<'py>( ("environment", unicode_environment(py)?), ( "render_project_var", - pyo3::wrap_pyfunction!(crate::bindings::_helpers::authored_values::project_vars::render_project_var_text, py)?.into_any(), + render_project_var_text_function(py)?.into_any(), ), ( "sql_resource_ref", @@ -242,11 +244,7 @@ pub(crate) fn build_model_inputs<'py>( ), ( "record_template_reads", - pyo3::wrap_pyfunction!( - crate::bindings::_helpers::model_config::template_data::record_template_reads, - py - )? - .into_any(), + record_template_reads_function(py)?.into_any(), ), ( "visible_declarations", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index ea6f752684..a6aa6e901d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -4,43 +4,28 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_attachment::{ - cursor_roles, declaration_scope, effective_config, model_config, model_inputs, - sql_expansion_context, target, + cursor_roles, declaration_scope, model_config, model_inputs, sql_expansion_context, target, }; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config_function::build_effective_connection_config_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_function::build_effective_connection_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config_function::build_effective_runtime_config_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config_function::build_effective_settings_config_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings_function::build_effective_settings_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_target_config_function::build_effective_target_config_function; +use sqlbuild_python_front::compile_attachment::main::build_effective_vars_function::build_effective_vars_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( sql_expansion_context::build_sql_expansion_context, module )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_connection, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_settings, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_vars, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_target_config, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_connection_config, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_runtime_config, - module - )?)?; - module.add_function(wrap_pyfunction!( - effective_config::build_effective_settings_config, - module - )?)?; + module.add_function(build_effective_connection_function(module)?)?; + module.add_function(build_effective_settings_function(module)?)?; + module.add_function(build_effective_vars_function(module)?)?; + module.add_function(build_effective_target_config_function(module)?)?; + module.add_function(build_effective_connection_config_function(module)?)?; + module.add_function(build_effective_runtime_config_function(module)?)?; + module.add_function(build_effective_settings_config_function(module)?)?; module.add_function(wrap_pyfunction!( model_config::native_model_config_session, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 226900a3fb..da924c246f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -8,14 +8,7 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; -use crate::bindings::_helpers::compile_attachment::effective_config::{ - build_effective_vars, target_selection, -}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; -use crate::bindings::_helpers::render::declaration_indexes::public_declaration_indexes; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -23,6 +16,12 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; +use sqlbuild_python_front::compile_attachment::main::target_selection::target_selection; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; +use sqlbuild_python_front::render::main::declarations::public_declaration_indexes::public_declaration_indexes; +use sqlbuild_python_front::render::models::SqlExpansionInputs; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. fn declaration_scope<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index b997929a6b..7bda60a51e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -4,21 +4,22 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, }; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::constants::{ COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, }; -use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_attachment::main::project_vars::project_vars; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; +use sqlbuild_python_front::model_config::main::config::run_id::run_id; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index e9a710d0b0..8ce8ee5894 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -14,8 +14,6 @@ use sqlbuild_core::cli_text::models::{ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; -use crate::bindings::_helpers::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; -use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; +use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; const METADATA_MODULE: &str = "importlib.metadata"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 86e566f8af..dd5044785d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -41,9 +41,7 @@ use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_outputs::reuse::{ ReuseCheckPy, ReuseRulesPy, check_compile_reuse, }; -use crate::bindings::_helpers::discovery::project_config::warn_deprecated_settings; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; -use crate::bindings::_helpers::macro_bridge::first_compile::keep_delta_claim; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -58,6 +56,8 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; +use sqlbuild_python_front::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; +use sqlbuild_python_front::macro_bridge::main::keep_delta_claim::keep_delta_claim; const REUSE_REQUEST: FacadeHost = FacadeHost::CompileReuseRequest; const HIT: &str = "hit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs index c63f5d157d..b2cbf7d862 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -7,11 +7,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::discovery::provider_secrets::invocation_allows_missing_secrets; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; +use sqlbuild_python_front::discovery::main::sources::invocation_allows_missing_secrets::invocation_allows_missing_secrets; /// The version of the runtime identity a stored compile records. const REUSE_FORMAT_VERSION: &str = "1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 883111a1b8..8f5b47607d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -11,7 +11,6 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; @@ -36,6 +35,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d00ffb9d14..d74df6da1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -27,12 +27,10 @@ use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_co use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::compiled_views::source_views::read_source_view; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, DeclaredShapes, declared_relations, }; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; -use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::compile_context::models::PythonProcessClock; @@ -48,6 +46,8 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::HashMap; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index a2ee44856e..c3374d41f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -3,13 +3,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; -use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; -use crate::bindings::_helpers::render::macro_expansion_result::{ - ExpansionRequest, expansion_result, -}; -use crate::bindings::_helpers::render::sql_macro_mocks::with_sql_macro_mocks; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -19,6 +12,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::validated_model_cursor_sql::validated_model_cursor_sql; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::render::main::declarations::declaration_scope_resolver::declaration_scope_resolver; +use sqlbuild_python_front::render::main::expansion::expansion_result::expansion_result; +use sqlbuild_python_front::render::main::sql_macros::with_sql_macro_mocks::with_sql_macro_mocks; +use sqlbuild_python_front::render::models::ExpansionRequest; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index cbb08a55c3..60ef674765 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::analysis_session::session::{ FamilyRow, NativeModelAnalysisSession, PivotRequestRow, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; -use crate::bindings::_helpers::model_config::header_families::HeaderFamilies; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; @@ -22,6 +21,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::models::HeaderFamilies; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 6d70cc38a6..1008452f1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; -use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; @@ -22,12 +21,13 @@ use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholde use crate::bindings::_helpers::project_assembly::syntax_validation::{ ParseOptions, validate_hook_sql_syntax, validate_sql_syntax, }; -use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_python_front::render::main::calls::top_level_macro_call_names::top_level_macro_call_names; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index d966251b77..089b7861b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -25,12 +25,6 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::contracts::spec_resolution::{ - resolve_effective_adapter_name, resolve_effective_scenario_config, -}; -use crate::bindings::_helpers::model_config::template_data::early_model_templates; -use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; -use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::semantic_checks::stage::{ SemanticCompletion, completed_semantic_project, }; @@ -44,6 +38,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::path_posix::path_posix; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; +use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; +use sqlbuild_python_front::scopes::main::checks::scope_index_with_compile_usages::scope_index_with_compile_usages; +use sqlbuild_python_front::scopes::main::reports::report_scope_index_errors::report_scope_index_errors; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 1bf2372e9f..ec03b3a6f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -10,7 +10,6 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_e use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; use sqlbuild_model_config::templates::models::DeferredSegment; -use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -19,6 +18,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::authored_values::main::render_project_var_text::render_project_var_text; const PRESERVE_TARGET_VALUE: &str = "preserve"; /// `sorted(DEFERRED_TARGET_CONTEXT_KEYS)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index ae41fdc99d..5d97463a57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -6,42 +6,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::compile_driver::config_validation::{ - ProjectConfiguration, reject_invalid_project_configuration, -}; use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; -use crate::bindings::_helpers::discovery::constants::{ - ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, -}; -use crate::bindings::_helpers::discovery::declaration_records::{ - discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, - discover_macro_files, discover_model_schema_files, discover_seed_files, - discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, -}; -use crate::bindings::_helpers::discovery::imported_providers::{ - discover_event_exporter_functions, discover_provider_classes, -}; -use crate::bindings::_helpers::discovery::imported_python_nodes::{ - discover_hook_functions, discover_materialization_files, discover_python_node_functions, -}; -use crate::bindings::_helpers::discovery::model_records::{ - ModelFileSelection, discovered_model_files, -}; -use crate::bindings::_helpers::discovery::project_config::{ - load_local_config_file, load_project_config_file, warn_deprecated_settings, -}; -use crate::bindings::_helpers::discovery::provider_secrets::{ - SecretPolicy, invocation_allows_missing_secrets, -}; -use crate::bindings::_helpers::discovery::python_function_files::discover_python_function_files; -use crate::bindings::_helpers::discovery::python_paths::reject_unsupported_python_paths; -use crate::bindings::_helpers::discovery::snapshot::in_snapshot; -use crate::bindings::_helpers::discovery::sql_test_records::{ - discover_scenario_files, discover_test_files, -}; -use crate::bindings::_helpers::discovery::yaml_records::{ - discover_schema_files, discover_source_files, -}; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produced}; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -55,6 +20,40 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; +use sqlbuild_python_front::compile_driver::models::ProjectConfiguration; +use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_ENV; +use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_SETTING; +use sqlbuild_python_front::discovery::constants::TRUE_ENV_VALUES; +use sqlbuild_python_front::discovery::main::project::load_local_config_file::load_local_config_file; +use sqlbuild_python_front::discovery::main::project::load_project_config_file::load_project_config_file; +use sqlbuild_python_front::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; +use sqlbuild_python_front::discovery::main::python_files::discover_event_exporter_functions::discover_event_exporter_functions; +use sqlbuild_python_front::discovery::main::python_files::discover_hook_functions::discover_hook_functions; +use sqlbuild_python_front::discovery::main::python_files::discover_materialization_files::discover_materialization_files; +use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; +use sqlbuild_python_front::discovery::main::python_files::discover_python_function_files::discover_python_function_files; +use sqlbuild_python_front::discovery::main::python_files::discover_python_node_functions::discover_python_node_functions; +use sqlbuild_python_front::discovery::main::python_files::reject_unsupported_python_paths::reject_unsupported_python_paths; +use sqlbuild_python_front::discovery::main::records::discover_adapter_file::discover_adapter_file; +use sqlbuild_python_front::discovery::main::records::discover_audit_files::discover_audit_files; +use sqlbuild_python_front::discovery::main::records::discover_constant_files::discover_constant_files; +use sqlbuild_python_front::discovery::main::records::discover_enum_files::discover_enum_files; +use sqlbuild_python_front::discovery::main::records::discover_macro_files::discover_macro_files; +use sqlbuild_python_front::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use sqlbuild_python_front::discovery::main::records::discover_scenario_files::discover_scenario_files; +use sqlbuild_python_front::discovery::main::records::discover_schema_files::discover_schema_files; +use sqlbuild_python_front::discovery::main::records::discover_seed_files::discover_seed_files; +use sqlbuild_python_front::discovery::main::records::discover_source_files::discover_source_files; +use sqlbuild_python_front::discovery::main::records::discover_sql_function_files::discover_sql_function_files; +use sqlbuild_python_front::discovery::main::records::discover_sql_hook_files::discover_sql_hook_files; +use sqlbuild_python_front::discovery::main::records::discover_test_files::discover_test_files; +use sqlbuild_python_front::discovery::main::records::discovered_model_files::discovered_model_files; +use sqlbuild_python_front::discovery::main::records::retained_discovery_session::retained_discovery_session; +use sqlbuild_python_front::discovery::main::snapshots::in_snapshot::in_snapshot; +use sqlbuild_python_front::discovery::main::sources::invocation_allows_missing_secrets::invocation_allows_missing_secrets; +use sqlbuild_python_front::discovery::models::ModelFileSelection; +use sqlbuild_python_front::discovery::models::SecretPolicy; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; /// The setting that turns delta compiles, and the project image they read, off. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index f7248baa18..6bad877a8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -6,19 +6,18 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; -use crate::bindings::_helpers::compile_attachment::effective_config::{ - build_effective_connection, build_effective_settings, build_effective_vars, -}; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection::build_effective_connection; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings::build_effective_settings; +use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::records::{compile_inputs_view, inputs_with}; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::input_loops::attachment::{attach_checks, attach_resources}; -use crate::bindings::_helpers::model_config::template_data::run_id; -use crate::bindings::_helpers::render::declaration_indexes::{ - public_declaration_indexes, public_model_schema_index, -}; +use sqlbuild_python_front::model_config::main::config::run_id::run_id; +use sqlbuild_python_front::render::main::declarations::public_declaration_indexes::public_declaration_indexes; +use sqlbuild_python_front::render::main::declarations::public_model_schema_index::public_model_schema_index; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index e0b34f994a..e7112cca71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -8,7 +8,6 @@ pub(crate) mod assembly_models; pub(crate) mod assembly_project; pub(crate) mod assembly_sql_tests; pub(crate) mod command; -pub(crate) mod config_validation; pub(crate) mod deferral; pub(crate) mod diff; pub(crate) mod discovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 2b619310b6..e2f94f9638 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::compile_command::reuse_recording::ArtifactRecorde use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, write_static_target, }; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::manifest::builder::build_manifest; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index c1f3620e1c..eb1a3b50b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; @@ -19,6 +18,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; /// The current build's phase timing tracker, when one is recording. pub(crate) fn timing_tracker(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 9e397246e4..488081bd09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -20,19 +20,18 @@ use crate::bindings::_helpers::compile_driver::targets::{ requires_explicit_targets, }; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; -use crate::bindings::_helpers::sqltext::lexical_syntax::{optional_record_syntax, syntax_record}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; +use sqlbuild_python_front::sqltext::main::syntax_record::syntax_record; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; use crate::bindings::_helpers::compiled_views::records::project_with; use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; @@ -42,6 +41,8 @@ use sqlbuild_python_base::compile_driver::constants::PROMOTION_CONFLICTS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Optional keywords of the project graph builders. const GRAPH_OPTIONAL: [&str; 7] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index ebe177bb4a..e665d8bde7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -22,9 +22,9 @@ use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::model_config::template_data::expanded_template_data; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ @@ -41,11 +41,7 @@ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databrick const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; use crate::bindings::_helpers::compile_attachment::target::resolved_target; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::discovery::guidance::setting_help; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -53,6 +49,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::discovery::main::failures::setting_help::setting_help; const DEFAULTS_SECTION: &str = "defaults"; /// One optional `(database, schema)` namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index dc6026816b..f9b2367126 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -19,7 +19,7 @@ use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, Ru use crate::bindings::_helpers::compiled_project::constants::{ JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, }; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. type Catalog = (BTreeMap, BTreeSet); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index c4f635c57d..6f644d93cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -56,11 +56,11 @@ use crate::bindings::_helpers::compiled_views::seed_views::{ use crate::bindings::_helpers::compiled_views::source_views::{ NativeCompiledSource, NativeSourceInput, SourceRecord, }; -use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use sqlbuild_project::types::InferredRow; use sqlbuild_project::types::ObjectKeyRow; use sqlbuild_project::types::SourceNamespace; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use sqlbuild_python_front::render::models::ConstantRendering; pub(crate) const CONTRACT_KEY: &str = "contract"; const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs index 1ee1ad8cc2..76bdb01204 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -13,7 +13,6 @@ use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_splices; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::snapshot::{ dependency_keys, destination, insert_node, @@ -21,6 +20,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. pub(crate) fn model_nodes( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs index 984526d334..5b08d91959 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::compiled_views::model_input_view::{ InputViewClasses, reference_tuple, view_classes, }; use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; const AUDIT_RESOURCE: &str = "audit"; const AUDIT_CLASSES: &str = "audits"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs index 4796268646..ca2d959b72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs @@ -30,12 +30,12 @@ use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSes use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledViewClasses; use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; -use crate::bindings::_helpers::render::macro_call_rows::span_expansion; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::render::main::calls::span_expansion::span_expansion; const COMPILE_PHASE: &str = "compile"; const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 47b1933493..24e937b063 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -1,9 +1,9 @@ //! A compiled model, read from the native compiled project that owns it. -use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; use sqlbuild_project::models as project_models; use sqlbuild_project::models::{DestinationFact, ModelMetadataTexts}; use sqlbuild_project::types::InferredRow; +use sqlbuild_python_front::model_config::models::HeaderAudits; use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -29,10 +29,10 @@ use crate::bindings::_helpers::compiled_views::model_input_view::{ }; use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::RelationKeyRow; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::models::HeaderColumns; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index 7b57e33aa6..df851fe4ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -20,14 +20,17 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::{ }; use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::{RelationKeyRow, relation_key_row}; -use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; -use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; -use crate::bindings::_helpers::model_config::header_families::{HeaderFamilies, family_specs}; -use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; -use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_tuple}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::records::projected_text::projected_text; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::HeaderFamilies; +use sqlbuild_python_front::render::main::calls::span_tuple::span_tuple; +use sqlbuild_python_front::render::types::SpanRow; const MODEL_RELATION_KIND: &str = "ref"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs index 8ed4f3cdce..19829fd9a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs @@ -21,8 +21,8 @@ use crate::bindings::_helpers::compiled_views::model_input_view::{ use crate::bindings::_helpers::compiled_views::source_views::{ SourceKeyFacts, SourceViews, read_source_view, }; -use crate::bindings::_helpers::discovery::project_facts::seed_csv_settings; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; const MATERIALIZED_CONFIG: &str = "materialized"; const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs index 905f2ca018..97b34ea561 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs @@ -10,9 +10,9 @@ use sqlbuild_state::consumption::models::{ }; use crate::bindings::_helpers::compiled_project::job_catalog::triggered_jobs; -use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; const DEFINITION: &str = "definition"; const SOURCE_DATA: &str = "source_data"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index 7d430dc305..54655ad6ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -23,10 +23,10 @@ use uuid::Uuid; use crate::bindings::_helpers::consumption::facts::{ BuildFacts, MODEL, SEED, build_facts, enum_text, }; -use crate::bindings::_helpers::native_host::consumption_state::{ - ConsumptionConnection, Location, NodeCompleteHook, -}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::native_host::models::NodeCompleteHook; /// Succeeded rows are flushed once this many executions are pending. const FLUSH_EXECUTIONS: usize = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 809fb4a1de..2b1e918e9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -9,9 +9,9 @@ use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; use crate::bindings::_helpers::analysis::relation_names::{ RelationRow, extract_sql_relation_names, }; -use crate::bindings::_helpers::native_host::consumption_state::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_front::native_host::main::optional_text::optional_text; /// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 7abdec05e8..8ac95d3422 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::refactor::contract_compare::compare_contracts; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -38,6 +37,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::refactor::main::compare_contracts::compare_contracts; const GENERATE_ACTION: &str = "generate"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index a0f0fcd71d..9352b77d16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -7,9 +7,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::format_command::files::write_file_atomically; -use crate::bindings::_helpers::refactor::contract_edits::edit_contract_model; -use crate::bindings::_helpers::refactor::contract_edits::edit_contract_source; -use crate::bindings::_helpers::refactor::contract_edits::remaining_contract_findings; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -18,6 +15,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::refactor::main::edit_contract_model::edit_contract_model; +use sqlbuild_python_front::refactor::main::edit_contract_source::edit_contract_source; +use sqlbuild_python_front::refactor::main::remaining_contract_findings::remaining_contract_findings; const UTF8: &str = "utf-8"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 9b2a843b48..361d2c5738 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -24,11 +24,12 @@ use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; use crate::bindings::_helpers::contracts::model_contracts::{ ConflictRow, DiagnosticRow, evaluate_project_model_contracts, project_promotion_conflicts, }; -use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location_rows}; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::sqltext::main::location_rows::location_rows; +use sqlbuild_python_front::sqltext::models::ColumnLocation; const PATHLIB_MODULE: &str = "pathlib"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 3565cb6f91..375db96d18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,6 +1,4 @@ pub(crate) mod contract_diagnostics; -pub(crate) mod effective_config; pub(crate) mod exception_types; pub(crate) mod model_contracts; pub(crate) mod registration; -pub(crate) mod spec_resolution; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index 30f503d77e..a1f676c557 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -2,21 +2,21 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::authored_values::sql_values; -use crate::bindings::_helpers::contracts::{ - contract_diagnostics, effective_config, exception_types, spec_resolution, -}; -use crate::bindings::_helpers::discovery::project_config; +use crate::bindings::_helpers::contracts::{contract_diagnostics, exception_types}; use sqlbuild_python_base::contracts::main::register_resource_identities::register_resource_identities; use sqlbuild_python_base::contracts::main::register_state_literals::register_state_literals; +use sqlbuild_python_front::authored_values::main::register_sql_values::register_sql_values; +use sqlbuild_python_front::contracts::main::register_effective_config::register_effective_config; +use sqlbuild_python_front::contracts::main::register_spec_resolution::register_spec_resolution; +use sqlbuild_python_front::discovery::main::project::register_project_config::register_project_config; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { contract_diagnostics::register(module)?; - effective_config::register(module)?; + register_effective_config(module)?; exception_types::register(module)?; - project_config::register(module)?; + register_project_config(module)?; register_resource_identities(module)?; - spec_resolution::register(module)?; - sql_values::register(module)?; + register_spec_resolution(module)?; + register_sql_values(module)?; register_state_literals(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 56ba668b30..b19e980701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -11,9 +11,7 @@ use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_target_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::cost_report::functions::read_cost_run; use crate::bindings::_helpers::cost_report::functions::resolve_cost_run; use crate::bindings::_helpers::cost_report::functions::write_cost_run; @@ -26,6 +24,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; /// Collection budget of an explicit `sqb cost` refresh, which a user waits for on purpose. const REFRESH_BUDGET_SECONDS: f64 = 300.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs index 8f03efd327..f6145f6d98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs @@ -17,8 +17,8 @@ use crate::bindings::_helpers::cost_report::records::{ cost_run, cost_runs, record_object, record_tuple, }; use crate::bindings::_helpers::cost_report::store; -use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; const BREAKDOWN_OPTIONS: (&[&str], &[&str]) = ( &["record", "use_color"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs index bc34d1abf0..0f8c8fc14c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -17,7 +17,6 @@ use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJso use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; use crate::bindings::_helpers::cost_telemetry::values::process_object; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -27,6 +26,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 0b3e650146..708bc7c78b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -21,7 +21,6 @@ use crate::bindings::_helpers::cost_telemetry::allocation::{ }; use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; /// Default budget of one collection, which the at-exit collection keeps short. const DEADLINE_SECONDS: f64 = 30.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs index a731e96140..5a415eec9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -7,10 +7,10 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyTuple}; use sqlbuild_core::cost_allocation::models::PyDecimal; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; -use crate::bindings::_helpers::state_rows::timestamps::offset_aware; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; const DECIMAL_MODULE: &str = "decimal"; /// The methods an adapter with warehouse cost telemetry implements. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs index a1ea62d802..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs @@ -1,7 +1 @@ -pub(crate) mod conversion; -pub(crate) mod durations; -pub(crate) mod intervals; -pub(crate) mod overrides; pub(crate) mod registration; -pub(crate) mod scalars; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs index ed10c4959d..23f5f77014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs @@ -2,11 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cursor::{durations, intervals, scalars}; +use sqlbuild_python_front::cursor::main::durations::register_durations::register_durations; +use sqlbuild_python_front::cursor::main::intervals::register_intervals::register_intervals; +use sqlbuild_python_front::cursor::main::scalars::register_scalars::register_scalars; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - scalars::register(module)?; - intervals::register(module)?; - durations::register(module)?; + register_scalars(module)?; + register_intervals(module)?; + register_durations(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 4711e6a15a..f6212e2fb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -5,8 +5,6 @@ use pyo3::types::PyDict; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dbt_interop::execution::render_dbt_execution_summary_footer; use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::plan::render_dbt_interop_plan; @@ -26,6 +24,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; const OPTIONS: (&[&str], &[&str]) = ( &["request", "invocation"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index 914f1ba3fe..e226cf5c8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::dbt_interop::cli::resolve_dbt_executable; use crate::bindings::_helpers::dbt_interop::cli::route_dbt_interop_args; use crate::bindings::_helpers::dbt_interop::graph::build_dbt_combined_graph; @@ -18,6 +17,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::DbtCompileReferenceResolver; use crate::bindings::_helpers::dbt_interop::runtime_config::{ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 138bd0bf77..3ba6d26f14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::command_runtime::connection::{ anchor_local_database_path, with_dbt_profile, }; use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_driver::deferral::resolve_model_deferral_inputs; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::planner_entries::display_plan::build_display_only_sqlbuild_plan; @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); const ATTACH_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 3c21e86d6d..5be09d62b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -34,10 +34,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; -use crate::bindings::_helpers::discovery::configuration::discover_configured_providers; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -51,6 +48,9 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::project::discover_configured_providers::discover_configured_providers; +use sqlbuild_python_front::discovery::models::SecretPolicy; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( &[], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 1389abde3a..f34d93ed9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -21,8 +21,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::diff_report::facts::diff_facts; use crate::bindings::_helpers::diff_report::output::publish_diff_output; @@ -33,7 +31,6 @@ use crate::bindings::_helpers::diff_report::support::{ call, caused, cli_error, error_text, is_exception, matches_class, matches_native, none, object, request_facts, user_error_parts, }; -use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -43,6 +40,9 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; const DEFAULT_DIFF_MAX_FULL_ROWS: i64 = 10_000_000; const ADAPTER_ERRORS: AdapterHost = AdapterHost::AdapterUserError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 90f7869f5f..29afbc7e3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -16,9 +16,7 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, none, object, progress_line, }; @@ -36,6 +34,8 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const GENERIC_DIALECT: &str = "generic"; pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs deleted file mode 100644 index 9e24509e7e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! The fields discovery reports for SQL hook entries. - -/// The fields a SQL hook entry reports, in output order. -pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ - "statement", - "name", - "relative_path", - "definition_sql", - "kwargs", - "description", -]; - -/// The flag of the static commands that lets them compile without provider secrets. -pub(crate) const ALLOW_MISSING_SECRETS_FLAG: &str = "--allow-missing-secrets"; -/// The environment variable that sets `--allow-missing-secrets`. -pub(crate) const ALLOW_MISSING_SECRETS_ENV: &str = "SQLBUILD_ALLOW_MISSING_SECRETS"; -/// The `[settings]` key of `sqlbuild_local.toml` that sets `--allow-missing-secrets`. -pub(crate) const ALLOW_MISSING_SECRETS_SETTING: &str = "allow_missing_secrets"; - -/// Environment values that switch a boolean `SQLBUILD_*` variable on. -pub(crate) const TRUE_ENV_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index a83fee866c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,38 +1 @@ -pub(crate) mod configuration; -pub(crate) mod constants; -pub(crate) mod declaration_files; -pub(crate) mod declaration_records; -pub(crate) mod description_inputs; -pub(crate) mod faults; -pub(crate) mod getters; -pub(crate) mod guidance; -pub(crate) mod header_declarations; -pub(crate) mod header_keys; -pub(crate) mod header_values; -pub(crate) mod hook_entries; -pub(crate) mod imported_modules; -pub(crate) mod imported_providers; -pub(crate) mod imported_python_nodes; -pub(crate) mod key_checks; -pub(crate) mod locations; -pub(crate) mod model_file_record; -pub(crate) mod model_records; -pub(crate) mod orchestration_files; -pub(crate) mod orchestration_values; -pub(crate) mod project_config; -pub(crate) mod project_facts; -pub(crate) mod project_files; -pub(crate) mod project_validation; -pub(crate) mod provider_secrets; -pub(crate) mod python_function_files; -pub(crate) mod python_hook_files; -pub(crate) mod python_module_files; -pub(crate) mod python_paths; -pub(crate) mod raised; pub(crate) mod registration; -pub(crate) mod scope_snapshot; -pub(crate) mod snapshot; -pub(crate) mod sql_test_blocks; -pub(crate) mod sql_test_records; -pub(crate) mod yaml_entries; -pub(crate) mod yaml_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs deleted file mode 100644 index c07a7efcb5..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ /dev/null @@ -1,630 +0,0 @@ -//! A discovered SQL model file as a native record whose Python attributes are built on demand. - -use std::path::PathBuf; -use std::sync::Arc; - -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use pyo3::exceptions::{PyRuntimeError, PyTypeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pymethods}; -use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; -use sqlbuild_discovery::models::LineColumnSpan; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_sqltext::compiler::models::AuthoredValue; - -const FIELD_NAMES: [&str; 10] = [ - "file_path", - "relative_path", - "contents", - "header_values", - "header_column_locations", - "output_column_locations", - "query_sql", - "enum_declarations", - "constant_declarations", - "extract_implicit_alias_columns", -]; -const STEM_ATTRIBUTE: &str = "stem"; - -/// Authored locations: native rows until Python reads them, then a cached dict. -struct Locations { - rows: Vec<(String, LineColumnSpan)>, - projected: PyOnceLock>, -} - -impl Locations { - fn from_rows(rows: Vec<(String, LineColumnSpan)>) -> Self { - Self { - rows, - projected: PyOnceLock::new(), - } - } - - /// These locations for another record: the rows, and the dict once Python has read it. - fn shared(&self, py: Python<'_>) -> Self { - let projected: PyOnceLock> = PyOnceLock::new(); - if let Some(dict) = self.projected.get(py) { - let _ = projected.set(py, dict.clone_ref(py)); - } - Self { - rows: self.rows.clone(), - projected, - } - } - - fn from_dict(py: Python<'_>, dict: Py) -> Self { - let projected: PyOnceLock> = PyOnceLock::new(); - let _ = projected.set(py, dict); - Self { - rows: Vec::new(), - projected, - } - } - - /// Python's `{name: SourceLocation(...)}`: a repeated name keeps its first position and last value. - fn dict<'py>(&self, path: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = path.py(); - Ok(self - .projected - .get_or_try_init(py, || { - let dict: Bound<'py, PyDict> = PyDict::new(py); - for (name, span) in &self.rows { - let location: Bound<'py, PyAny> = new_record( - py, - RecordId::SourceLocation, - (path, span.line, span.column, span.end_line, span.end_column), - None, - )?; - dict.set_item(name, location)?; - } - Ok::<_, pyo3::PyErr>(dict.unbind()) - })? - .bind(py) - .clone()) - } -} - -/// The names a model file is known by, as Python's `pathlib` spells them. -#[derive(Clone)] -pub(crate) struct ModelFileTexts { - /// `file_path.stem`, the model's name. - pub(crate) name: String, - /// `relative_path.as_posix()`. - pub(crate) posix_path: String, - /// `str(relative_path)`. - pub(crate) relative_path: String, - /// `str(file_path)`. - pub(crate) file_path: String, -} - -/// A model file's paths: native text, with the Python path objects built on first read. -struct ModelFilePaths { - native_file_path: PathBuf, - /// `None` when a path is not text Python can encode, so readers raise as Python does. - texts: Option, - /// `(file_path, relative_path)` as Python path objects. - objects: PyOnceLock<(Py, Py)>, - /// The relative path the objects are built from, unless they were given. - relative: Option, -} - -impl ModelFilePaths { - /// Paths given as Python objects; their text is read once, here. - fn given(py: Python<'_>, file_path: Py, relative_path: Py) -> PyResult { - let native_file_path: PathBuf = file_path.bind(py).extract()?; - let relative: PathBuf = relative_path.bind(py).extract()?; - let texts: Option = - native_file_path - .to_str() - .zip(relative.to_str()) - .map(|(file_text, relative_text)| ModelFileTexts { - name: model_file_stem(&posix(file_text)).to_owned(), - posix_path: posix(relative_text), - relative_path: relative_text.to_owned(), - file_path: file_text.to_owned(), - }); - let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); - let _ = objects.set(py, (file_path, relative_path)); - Ok(Self { - native_file_path, - texts, - objects, - relative: None, - }) - } - - /// Paths native discovery named; their Python path objects are built only when read. - fn discovered(relative_path: &str, file_path: String) -> Self { - let name: String = model_file_stem(relative_path).to_owned(); - Self { - native_file_path: PathBuf::from(&file_path), - texts: Some(ModelFileTexts { - name, - posix_path: relative_path.to_owned(), - relative_path: relative_path.replace('/', std::path::MAIN_SEPARATOR_STR), - file_path, - }), - objects: PyOnceLock::new(), - relative: Some(PathBuf::from(relative_path)), - } - } - - fn shared(&self, py: Python<'_>) -> Self { - let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); - if let Some((file_path, relative_path)) = self.objects.get(py) { - let _ = objects.set(py, (file_path.clone_ref(py), relative_path.clone_ref(py))); - } - Self { - native_file_path: self.native_file_path.clone(), - texts: self.texts.clone(), - objects, - relative: self.relative.clone(), - } - } - - /// `(file_path, relative_path)` as the `pathlib` objects discovery names the file by. - fn objects<'py>(&self, py: Python<'py>) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let (file_path, relative_path) = self.objects.get_or_try_init(py, || { - let Some(relative) = &self.relative else { - return Err(PyRuntimeError::new_err("a model file has no paths")); - }; - Ok::<_, pyo3::PyErr>(( - self.native_file_path.as_path().into_pyobject(py)?.unbind(), - relative.as_path().into_pyobject(py)?.unbind(), - )) - })?; - Ok((file_path.bind(py).clone(), relative_path.bind(py).clone())) - } - - /// The file's names, or the error Python raises reading a name that is not UTF-8 text. - fn texts(&self, py: Python<'_>) -> PyResult<&ModelFileTexts> { - if let Some(texts) = &self.texts { - return Ok(texts); - } - let (file_path, relative_path) = self.objects(py)?; - for path in [file_path, relative_path] { - path.str()?.to_str()?; - } - Err(PyRuntimeError::new_err("a model file path is not text")) - } -} - -/// `text` with the platform's path separator spelled `/`, as `PurePath.as_posix` spells it. -fn posix(text: &str) -> String { - text.replace(std::path::MAIN_SEPARATOR, "/") -} - -/// `text` as the Python string `cache` shares with every reader, built on first read. -pub(crate) fn projected_text<'py>( - py: Python<'py>, - cache: &PyOnceLock>, - text: &str, -) -> Bound<'py, PyString> { - cache - .get_or_init(py, || PyString::new(py, text).unbind()) - .bind(py) - .clone() -} - -/// A model header's authored values: native entries, with the Python dict built on first read. -struct HeaderValues { - native: Option>, - projected: PyOnceLock>, -} - -impl HeaderValues { - fn native(values: Vec<(String, AuthoredValue)>) -> Self { - Self { - native: Some(values.into()), - projected: PyOnceLock::new(), - } - } - - fn given(py: Python<'_>, values: Py) -> Self { - let projected: PyOnceLock> = PyOnceLock::new(); - let _ = projected.set(py, values); - Self { - native: None, - projected, - } - } - - fn shared(&self, py: Python<'_>) -> Self { - let projected: PyOnceLock> = PyOnceLock::new(); - if let Some(dict) = self.projected.get(py) { - let _ = projected.set(py, dict.clone_ref(py)); - } - Self { - native: self.native.clone(), - projected, - } - } - - fn dict<'py>(&self, py: Python<'py>) -> PyResult> { - Ok(self - .projected - .get_or_try_init(py, || { - let values: &[(String, AuthoredValue)] = self.native.as_deref().unwrap_or_default(); - map_to_python(py, values)? - .into_bound(py) - .downcast_into::() - .map(Bound::unbind) - .map_err(pyo3::PyErr::from) - })? - .bind(py) - .clone()) - } -} - -/// A discovered SQL model file and its raw contents. -#[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] -pub(crate) struct NativeModelFile { - paths: ModelFilePaths, - contents: Arc, - /// `contents` and `query_sql` as the Python strings readers share, built on first read. - projected_texts: (PyOnceLock>, PyOnceLock>), - header_values: HeaderValues, - header_column_locations: Locations, - output_column_locations: Locations, - query_sql: Arc, - enum_declarations: Py, - constant_declarations: Py, - extract_implicit_alias_columns: bool, -} - -/// What native discovery read for one model file. -pub(crate) struct ModelFileParts { - pub(crate) contents: String, - pub(crate) header_values: Vec<(String, AuthoredValue)>, - pub(crate) header_column_locations: Vec<(String, LineColumnSpan)>, - pub(crate) output_column_locations: Vec<(String, LineColumnSpan)>, - pub(crate) query_sql: String, - pub(crate) extract_implicit_alias_columns: bool, -} - -/// How discovery names a model file: natively, or by Python paths for names Python escapes. -pub(crate) enum DiscoveredPaths { - /// The `/`-separated relative path and `str(project_dir / relative_path)`. - Native(String, String), - /// `(file_path, relative_path)` built by Python. - Python(Py, Py), -} - -/// A model file before its header declarations are attached. -pub(crate) struct PendingModelFile { - paths: ModelFilePaths, - parts: ModelFileParts, -} - -impl PendingModelFile { - pub(crate) fn new( - py: Python<'_>, - paths: DiscoveredPaths, - parts: ModelFileParts, - ) -> PyResult { - let paths: ModelFilePaths = match paths { - DiscoveredPaths::Native(relative_path, file_path) => { - ModelFilePaths::discovered(&relative_path, file_path) - } - DiscoveredPaths::Python(file_path, relative_path) => { - ModelFilePaths::given(py, file_path, relative_path)? - } - }; - Ok(Self { paths, parts }) - } - - /// The model's name, its file stem. - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - match &self.paths.texts { - Some(texts) => Ok(texts.name.clone()), - None => field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), - } - } - - /// The relative path as the Python path object the record keeps. - pub(crate) fn relative_path<'py>(&self, py: Python<'py>) -> PyResult> { - Ok(self.paths.objects(py)?.1) - } - - /// The authored header value of `key`. - pub(crate) fn header_value(&self, key: &str) -> Option<&AuthoredValue> { - self.parts - .header_values - .iter() - .rev() - .find(|(name, _)| name == key) - .map(|(_, value)| value) - } - - pub(crate) fn finished(self, declarations: (Py, Py)) -> NativeModelFile { - let (enum_declarations, constant_declarations) = declarations; - let parts: ModelFileParts = self.parts; - NativeModelFile { - paths: self.paths, - contents: parts.contents.into(), - projected_texts: (PyOnceLock::new(), PyOnceLock::new()), - header_values: HeaderValues::native(parts.header_values), - header_column_locations: Locations::from_rows(parts.header_column_locations), - output_column_locations: Locations::from_rows(parts.output_column_locations), - query_sql: parts.query_sql.into(), - enum_declarations, - constant_declarations, - extract_implicit_alias_columns: parts.extract_implicit_alias_columns, - } - } -} - -/// The keyword fields Python constructs a model file record with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct ModelFileFields { - file_path: Py, - relative_path: Py, - contents: Py, - header_values: Py, - header_column_locations: Py, - output_column_locations: Py, - query_sql: Py, - #[pyo3(default)] - enum_declarations: Option>, - #[pyo3(default)] - constant_declarations: Option>, - #[pyo3(default = true)] - extract_implicit_alias_columns: bool, -} - -impl NativeModelFile { - pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { - self.header_column_locations - .dict(&self.paths.objects(py)?.1) - } - - /// The header's authored values as the Python dict config builds read. - pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> PyResult> { - self.header_values.dict(py) - } - - /// The header columns' authored locations, unless Python constructed this record. - pub(crate) fn header_location_rows(&self) -> &[(String, LineColumnSpan)] { - &self.header_column_locations.rows - } - - /// The header's native authored values, unless Python constructed this record. - pub(crate) fn native_header_values(&self) -> Option<&Arc<[(String, AuthoredValue)]>> { - self.header_values.native.as_ref() - } - - pub(crate) fn declaration_tuples<'py>( - &self, - py: Python<'py>, - ) -> (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>) { - ( - self.enum_declarations.bind(py), - self.constant_declarations.bind(py), - ) - } - - /// The file path as a native path. - pub(crate) fn file_path_buf(&self) -> PathBuf { - self.paths.native_file_path.clone() - } - - /// The file's names, or the error Python raises reading a name that is not UTF-8 text. - pub(crate) fn path_texts(&self, py: Python<'_>) -> PyResult { - self.paths.texts(py).cloned() - } - - /// `relative_path.as_posix()`. - pub(crate) fn posix_path(&self, py: Python<'_>) -> PyResult { - Ok(self.paths.texts(py)?.posix_path.clone()) - } - - /// `str(relative_path)`. - pub(crate) fn relative_text(&self, py: Python<'_>) -> PyResult { - Ok(self.paths.texts(py)?.relative_path.clone()) - } - - /// `str(relative_path.parent)`, the folder project graphs group the model by. - pub(crate) fn folder(&self, py: Python<'_>) -> PyResult { - Ok(self - .paths - .texts(py)? - .relative_path - .rsplit_once(std::path::MAIN_SEPARATOR) - .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())) - } - - /// `(file_path, relative_path)` as Python path objects. - pub(crate) fn path_objects<'py>( - &self, - py: Python<'py>, - ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - self.paths.objects(py) - } - - /// The authored query below the header. - pub(crate) fn query_text(&self) -> &Arc { - &self.query_sql - } - - /// The whole authored file. - pub(crate) fn contents_text(&self) -> &Arc { - &self.contents - } - - fn fields<'py>(&self, py: Python<'py>) -> PyResult>> { - let (file_path, relative_path) = self.paths.objects(py)?; - Ok(vec![ - file_path, - relative_path.clone(), - self.contents(py).into_any(), - self.header_values.dict(py)?.into_any(), - self.header_column_locations - .dict(&relative_path)? - .into_any(), - self.output_column_locations - .dict(&relative_path)? - .into_any(), - self.query_sql(py).into_any(), - self.enum_declarations.bind(py).clone().into_any(), - self.constant_declarations.bind(py).clone().into_any(), - pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) - .to_owned() - .into_any(), - ]) - } -} - -#[pymethods] -impl NativeModelFile { - #[new] - #[pyo3(signature = (**fields))] - fn new(py: Python<'_>, fields: Option>) -> PyResult { - let Some(fields) = fields else { - return Err(PyTypeError::new_err( - "DiscoveredSqlModelFile requires its fields", - )); - }; - for key in fields.keys() { - let key: String = key.extract()?; - if !FIELD_NAMES.contains(&key.as_str()) { - return Err(PyTypeError::new_err(format!( - "DiscoveredSqlModelFile got an unexpected keyword argument '{key}'" - ))); - } - } - let fields: ModelFileFields = fields.extract()?; - Ok(Self { - paths: ModelFilePaths::given(py, fields.file_path, fields.relative_path)?, - contents: fields.contents.bind(py).to_str()?.into(), - projected_texts: (PyOnceLock::new(), PyOnceLock::new()), - header_values: HeaderValues::given(py, fields.header_values), - header_column_locations: Locations::from_dict(py, fields.header_column_locations), - output_column_locations: Locations::from_dict(py, fields.output_column_locations), - query_sql: fields.query_sql.bind(py).to_str()?.into(), - enum_declarations: fields - .enum_declarations - .unwrap_or_else(|| PyTuple::empty(py).unbind()), - constant_declarations: fields - .constant_declarations - .unwrap_or_else(|| PyTuple::empty(py).unbind()), - extract_implicit_alias_columns: fields.extract_implicit_alias_columns, - }) - } - - /// This file with edited contents and query, as `dataclasses.replace` would make it. - pub(crate) fn with_text( - &self, - py: Python<'_>, - contents: Py, - query_sql: Py, - ) -> PyResult { - Ok(Self { - paths: self.paths.shared(py), - contents: contents.bind(py).to_str()?.into(), - projected_texts: (PyOnceLock::new(), PyOnceLock::new()), - header_values: self.header_values.shared(py), - header_column_locations: self.header_column_locations.shared(py), - output_column_locations: self.output_column_locations.shared(py), - query_sql: query_sql.bind(py).to_str()?.into(), - enum_declarations: self.enum_declarations.clone_ref(py), - constant_declarations: self.constant_declarations.clone_ref(py), - extract_implicit_alias_columns: self.extract_implicit_alias_columns, - }) - } - - #[getter] - pub(crate) fn file_path(&self, py: Python<'_>) -> PyResult> { - Ok(self.paths.objects(py)?.0.unbind()) - } - - #[getter] - pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { - Ok(self.paths.objects(py)?.1.unbind()) - } - - #[getter] - pub(crate) fn contents<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { - projected_text(py, &self.projected_texts.0, &self.contents) - } - - #[getter] - pub(crate) fn header_values(&self, py: Python<'_>) -> PyResult> { - Ok(self.header_values.dict(py)?.unbind()) - } - - #[getter] - pub(crate) fn header_column_locations<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.header_locations(py) - } - - #[getter] - pub(crate) fn output_column_locations<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.output_column_locations - .dict(&self.paths.objects(py)?.1) - } - - #[getter] - pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { - projected_text(py, &self.projected_texts.1, &self.query_sql) - } - - #[getter] - pub(crate) fn enum_declarations(&self, py: Python<'_>) -> Py { - self.enum_declarations.clone_ref(py) - } - - #[getter] - pub(crate) fn constant_declarations(&self, py: Python<'_>) -> Py { - self.constant_declarations.clone_ref(py) - } - - #[getter] - pub(crate) fn extract_implicit_alias_columns(&self) -> bool { - self.extract_implicit_alias_columns - } - - fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult> { - let Ok(other) = other.downcast::() else { - return Ok(py.NotImplemented()); - }; - let other: &NativeModelFile = other.get(); - for (left, right) in self.fields(py)?.iter().zip(other.fields(py)?) { - if !left.eq(right)? { - return false.into_py_any(py); - } - } - true.into_py_any(py) - } - - fn __repr__(&self, py: Python<'_>) -> PyResult { - let fields: Vec = FIELD_NAMES - .iter() - .zip(self.fields(py)?) - .map(|(name, value)| Ok(format!("{name}={}", value.repr()?))) - .collect::>()?; - Ok(format!("DiscoveredSqlModelFile({})", fields.join(", "))) - } - - /// The keyword fields `copy` and `pickle` rebuild this record from. - fn __getnewargs_ex__<'py>( - slf: &Bound<'py, Self>, - ) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyDict>)> { - let py: Python<'py> = slf.py(); - let keywords: Bound<'py, PyDict> = PyDict::new(py); - for (name, value) in FIELD_NAMES.iter().zip(slf.get().fields(py)?) { - keywords.set_item(name, value)?; - } - Ok((PyTuple::empty(py), keywords)) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 37d4db2725..652232047c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -2,36 +2,51 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::discovery::{ - declaration_files, declaration_records, description_inputs, guidance, header_declarations, - header_values, hook_entries, imported_python_nodes, key_checks, locations, model_records, - project_files, project_validation, python_function_files, python_hook_files, - python_module_files, raised, scope_snapshot, snapshot, sql_test_records, yaml_entries, - yaml_records, -}; +use sqlbuild_python_front::discovery::main::failures::register_guidance::register_guidance; +use sqlbuild_python_front::discovery::main::failures::register_raised::register_raised; +use sqlbuild_python_front::discovery::main::headers::register_header_declarations::register_header_declarations; +use sqlbuild_python_front::discovery::main::headers::register_header_values::register_header_values; +use sqlbuild_python_front::discovery::main::headers::register_hook_entries::register_hook_entries; +use sqlbuild_python_front::discovery::main::headers::register_key_checks::register_key_checks; +use sqlbuild_python_front::discovery::main::project::register_locations::register_locations; +use sqlbuild_python_front::discovery::main::project::register_project_files::register_project_files; +use sqlbuild_python_front::discovery::main::project::register_project_validation::register_project_validation; +use sqlbuild_python_front::discovery::main::python_files::register_imported_python_nodes::register_imported_python_nodes; +use sqlbuild_python_front::discovery::main::python_files::register_python_function_files::register_python_function_files; +use sqlbuild_python_front::discovery::main::python_files::register_python_hook_files::register_python_hook_files; +use sqlbuild_python_front::discovery::main::python_files::register_python_module_files::register_python_module_files; +use sqlbuild_python_front::discovery::main::records::register_declaration_records::register_declaration_records; +use sqlbuild_python_front::discovery::main::records::register_description_inputs::register_description_inputs; +use sqlbuild_python_front::discovery::main::records::register_model_records::register_model_records; +use sqlbuild_python_front::discovery::main::records::register_sql_test_records::register_sql_test_records; +use sqlbuild_python_front::discovery::main::records::register_yaml_records::register_yaml_records; +use sqlbuild_python_front::discovery::main::snapshots::register_scope_snapshot::register_scope_snapshot; +use sqlbuild_python_front::discovery::main::snapshots::register_snapshot::register_snapshot; +use sqlbuild_python_front::discovery::main::sources::register_declaration_files::register_declaration_files; +use sqlbuild_python_front::discovery::main::sources::register_yaml_entries::register_yaml_entries; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_files::register(module)?; - snapshot::register(module)?; - scope_snapshot::register(module)?; - declaration_files::register(module)?; - declaration_records::register(module)?; - python_function_files::register(module)?; - python_hook_files::register(module)?; - python_module_files::register(module)?; - model_records::register(module)?; - sql_test_records::register(module)?; - yaml_records::register(module)?; - raised::register(module)?; - locations::register(module)?; - header_declarations::register(module)?; - key_checks::register(module)?; - project_validation::register(module)?; - yaml_entries::register(module)?; - header_values::register(module)?; - guidance::register(module)?; - description_inputs::register(module)?; - hook_entries::register(module)?; - imported_python_nodes::register(module)?; + register_project_files(module)?; + register_snapshot(module)?; + register_scope_snapshot(module)?; + register_declaration_files(module)?; + register_declaration_records(module)?; + register_python_function_files(module)?; + register_python_hook_files(module)?; + register_python_module_files(module)?; + register_model_records(module)?; + register_sql_test_records(module)?; + register_yaml_records(module)?; + register_raised(module)?; + register_locations(module)?; + register_header_declarations(module)?; + register_key_checks(module)?; + register_project_validation(module)?; + register_yaml_entries(module)?; + register_header_values(module)?; + register_guidance(module)?; + register_description_inputs(module)?; + register_hook_entries(module)?; + register_imported_python_nodes(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs index cd72890a7f..0585f065b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs @@ -14,11 +14,12 @@ use sqlbuild_state::execution_history::models::{ }; use sqlbuild_state::execution_history::types::UtcTime; -use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_time; const UTC_SUFFIX: &str = "Z"; const UTC_OFFSET: &str = "+00:00"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index c7ecfe0ed6..2f51322b96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -38,9 +38,6 @@ use sqlbuild_state::execution_history::constants::{ DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, SCHEMA_VERSION, }; -use crate::bindings::_helpers::attachments::cursor_intrinsics::{ - MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, -}; use crate::bindings::_helpers::compile_command::constants::{ REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; @@ -51,7 +48,9 @@ use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; use crate::bindings::_helpers::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; -use crate::bindings::_helpers::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; +use sqlbuild_python_front::attachments::constants::MICROBATCH_END_SENTINEL; +use sqlbuild_python_front::attachments::constants::MICROBATCH_START_SENTINEL; +use sqlbuild_python_front::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; /// Text constants exported under their Python names. pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs index cecee1955a..80d54a4cf9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs @@ -3,13 +3,13 @@ use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; use crate::bindings::_helpers::exported_constants::constants::{ EXPORTED_INTEGERS, EXPORTED_TEXT_SETS, EXPORTED_TEXT_TUPLES, EXPORTED_TEXTS, }; use sqlbuild_cli::launcher::main::install_native_build_identity::install_native_build_identity; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::constants::GRAIN_NAMES; /// Registers after the native enums, since `GRAIN_ORDER` is keyed by `CursorGrain` members. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 86eccff280..20ba1bd9b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -18,7 +18,6 @@ use sqlbuild_rules::lint_text::models::{ FormatFault, FormatFile, FormattedBodies, SuppressionFinding, }; -use crate::bindings::_helpers::discovery::description_inputs::externally_described_models; use crate::bindings::_helpers::format_command::cache::FormatCache; use crate::bindings::_helpers::format_command::files::{ FormatChange, PendingWrite, persist_changes, @@ -46,6 +45,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::discovery::main::records::externally_described_models::externally_described_models; const DESCRIPTION_PRESENT: &str = "description-present"; /// The fix that rewrites descriptions escaping or doubling double quotes as `"""` strings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index fa4dbbd098..539fb0eec0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -15,16 +15,6 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::discovery::declaration_records::discover_model_schema_files; -use crate::bindings::_helpers::discovery::model_records::discover_model_files; -use crate::bindings::_helpers::discovery::project_config::load_local_config_file; -use crate::bindings::_helpers::discovery::project_config::load_project_config_file; -use crate::bindings::_helpers::discovery::snapshot::directory_snapshot_scope; -use crate::bindings::_helpers::discovery::sql_test_records::discover_test_files; -use crate::bindings::_helpers::discovery::yaml_records::{ - discover_schema_files, discover_source_files, -}; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -37,6 +27,15 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::project::load_local_config_file::load_local_config_file; +use sqlbuild_python_front::discovery::main::project::load_project_config_file::load_project_config_file; +use sqlbuild_python_front::discovery::main::records::discover_model_files::discover_model_files; +use sqlbuild_python_front::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use sqlbuild_python_front::discovery::main::records::discover_schema_files::discover_schema_files; +use sqlbuild_python_front::discovery::main::records::discover_source_files::discover_source_files; +use sqlbuild_python_front::discovery::main::records::discover_test_files::discover_test_files; +use sqlbuild_python_front::discovery::main::snapshots::directory_snapshot_scope::directory_snapshot_scope; const SELECTION_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 6a6e07528e..8c6b61c763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_project_conn use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::compiled_project::snapshot_execution::VerifiedRun; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; @@ -32,6 +31,7 @@ use sqlbuild_python_base::native_host::main::connection_failure::connection_fail use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index f2b9aa9219..a7712a43d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::freshness::observation::{ Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, }; -use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, @@ -25,6 +24,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::state_rows::main::event_rows::age_policy::age_policy; +use sqlbuild_python_front::state_rows::main::event_rows::equivalent::equivalent; const ADAPTER_STRATEGY: &str = "adapter"; const UNAVAILABLE_REASON: &str = "unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index aeba5c4d31..169e044634 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -12,8 +12,6 @@ use crate::bindings::_helpers::freshness::support::{ text, }; use crate::bindings::_helpers::references::source_relations::render_source_read; -use crate::bindings::_helpers::state_rows::freshness::{normalize, version_hash}; -use crate::bindings::_helpers::state_rows::timestamps::{in_utc, offset_aware}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -27,6 +25,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::normalize::normalize; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; +use sqlbuild_python_front::state_rows::main::event_rows::version_hash::version_hash; const ADAPTER_STRATEGY: &str = "adapter"; const COLUMN_STRATEGY: &str = "column"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index eb97aa8dd2..aabf7ddd0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -21,7 +21,6 @@ use crate::bindings::_helpers::freshness::observation::{ use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::text; use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -31,6 +30,8 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::is_warehouse_access_refused::is_warehouse_access_refused; +use sqlbuild_python_front::state_rows::main::event_rows::age_policy::age_policy; +use sqlbuild_python_front::state_rows::main::event_rows::equivalent::equivalent; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 0879f31789..317e8a32e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -11,13 +11,9 @@ use sqlbuild_state::freshness::constants::{ use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; -use crate::bindings::_helpers::native_host::consumption_state::state_table_name; -use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; -use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -25,6 +21,10 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::ensure_event_id_column::ensure_event_id_column; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; +use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; +use sqlbuild_python_front::state_rows::main::tables::state_table_create_sql::state_table_create_sql; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index da4822caa8..93d3fb0850 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -4,12 +4,14 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyString}; -use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::exception_types::{ SourceFreshnessInputError, SourceFreshnessObservationError, }; -use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; pub(crate) const INPUT_ERROR: fn(String) -> PyErr = SourceFreshnessInputError::new_err; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index 4ace206d30..cfebc0428e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -8,8 +8,6 @@ use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNod use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use crate::bindings::_helpers::discovery::imported_modules::provider_usages; -use crate::bindings::_helpers::discovery::imported_python_nodes::node_identity; use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; use crate::bindings::_helpers::graph::views::compiled_graph_keys; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -19,6 +17,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::main::python_files::node_identity::node_identity; +use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; const NAME_KEY: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index 5be7d8d2ce..7c03d6d5f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -37,15 +37,16 @@ use crate::bindings::_helpers::input_loops::model_loop_named::{ use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, file_declarations, }; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::header_audits::{AuditInstance, HeaderAudits}; -use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; -use crate::bindings::_helpers::model_config::header_metadata::thresholds; -use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use sqlbuild_project::models::AttachedTarget; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::headers::thresholds::thresholds; +use sqlbuild_python_front::model_config::models::AuditInstance; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::NativeModelValidator; +use sqlbuild_python_front::model_config::models::PyNode; const THRESHOLDS_KEY: &str = "thresholds"; const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 1054b80b3c..19050d2648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -29,7 +29,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use crate::bindings::_helpers::analysis_session::query_columns::query_columns_error; -use crate::bindings::_helpers::attachments::function_headers::parsed_header; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ FunctionRow, FunctionSettings, }; @@ -44,12 +43,13 @@ use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, expanded_template, parse_checked, scoped_declarations, }; -use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::PythonRuntime; use sqlbuild_project::models::ReferenceRow; use sqlbuild_project::types::NamedTypeRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; +use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; +use sqlbuild_python_front::model_config::models::NativeModelValidator; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 603ec3c3c2..962dbc097b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -48,7 +48,6 @@ use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInpu use crate::bindings::_helpers::compiled_views::source_views::{ AttachedSource, NativeSourceInput, retain_sources, }; -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::input_loops::audit_loop::{AttachmentOwner, attached_audits}; use crate::bindings::_helpers::input_loops::function_loop::{ attached_functions, checked_table_function_arities, @@ -91,28 +90,32 @@ use crate::bindings::_helpers::input_loops::model_loop_visibility::{ }; use crate::bindings::_helpers::input_loops::source_loop::attached_sources; use crate::bindings::_helpers::input_loops::sql_test_loop::TestScope; -use crate::bindings::_helpers::model_config::config_classes::ConfigClasses; -use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; -use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; -use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; -use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, - expand_declarations, -}; -use crate::bindings::_helpers::render::declaration_values::ConstantRendering; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; -use crate::bindings::_helpers::render::macro_call_rows::span_row; -use crate::bindings::_helpers::render::sql_macro_host::{SqlCallOrigin, SqlMacroSource}; -use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -use crate::bindings::_helpers::sqltext::static_sql::{ - InterpolationRow as SqlInterpolationRow, InterpolationSources, interpolation_batch, -}; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::discovery::models::ModelFileTexts; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; +use sqlbuild_python_front::model_config::models::ConfigClasses; +use sqlbuild_python_front::model_config::models::ContractClasses; +use sqlbuild_python_front::model_config::models::NativeModelValidator; +use sqlbuild_python_front::render::main::calls::span_row::span_row; +use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; +use sqlbuild_python_front::render::models::ConstantRendering; +use sqlbuild_python_front::render::models::DeclarationExpansion; +use sqlbuild_python_front::render::models::DeclarationExpansionClasses; +use sqlbuild_python_front::render::models::ExpandedDeclarations; +use sqlbuild_python_front::render::models::ExpansionConsumer; +use sqlbuild_python_front::render::models::SqlCallOrigin; +use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::types::SpanRow; +use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; +use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_batch; +use sqlbuild_python_front::sqltext::models::InterpolationRow as SqlInterpolationRow; +use sqlbuild_python_front::sqltext::models::InterpolationSources; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; @@ -659,7 +662,7 @@ impl NativeModelLoop { }; let interpolated: String = applied_interpolation(context, interpolation)?; let (references, stop) = scan.clone().unwrap_or_default(); - let expansion_classes: ExpansionClasses<'py> = ExpansionClasses { + let expansion_classes: DeclarationExpansionClasses<'py> = DeclarationExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs index 2cb2ebd0e7..502891c5c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs @@ -13,15 +13,17 @@ use sqlbuild_model_config::config_presence::main::contains_template::contains_te use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_config_value; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, config_error}; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::config_classes::{ConfigClasses, config_fields, view}; -use crate::bindings::_helpers::model_config::config_nodes::carried_node; -use crate::bindings::_helpers::model_config::model_config_builder::{ - BuiltConfig, ConfigBuild, NativeModelConfigBuilder, -}; -use crate::bindings::_helpers::model_config::typed_config::BuiltConfigValues; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; +use sqlbuild_python_front::model_config::main::config::config_fields::config_fields; +use sqlbuild_python_front::model_config::main::config::view::view; +use sqlbuild_python_front::model_config::models::BuiltConfig; +use sqlbuild_python_front::model_config::models::ConfigBuild; +use sqlbuild_python_front::model_config::models::ConfigClasses; +use sqlbuild_python_front::model_config::models::NativeModelConfigBuilder; +use sqlbuild_python_front::model_config::models::PyNode; +use sqlbuild_python_front::model_config::types::BuiltConfigValues; const COLUMNS_KEY: &str = "columns"; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs index 94c5f85d75..ed82993a64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs @@ -9,11 +9,13 @@ use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::input_loops::model_loop::LoopContext; use crate::bindings::_helpers::input_loops::model_loop_schema::DeclaredColumns; -use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_node}; -use crate::bindings::_helpers::model_config::header_metadata::{HeaderColumn, HeaderColumns}; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_node::config_node; +use sqlbuild_python_front::model_config::models::HeaderColumn; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::types::PyConfigNode; const CASES_ATTRIBUTE: &str = "cases"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index a14e4b5d5d..0becc419a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -18,7 +18,6 @@ use sqlbuild_model_config::model_validation::models::ColumnMigration; use sqlbuild_model_config::types::NodeKind; use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, compile_error, config_error, }; @@ -29,20 +28,24 @@ use crate::bindings::_helpers::input_loops::model_loop_inputs::present; use crate::bindings::_helpers::input_loops::model_loop_schema::{ DeclaredColumns, ModelSchema, merged_columns, schema_enums, }; -use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; -use crate::bindings::_helpers::model_config::header_families::HeaderFamilies; -use crate::bindings::_helpers::model_config::header_metadata::{ - HeaderColumn, HeaderColumns, HeaderValue, NativeColumnSpec, native_schema_column, - parsed_audit_identity, -}; -use crate::bindings::_helpers::model_config::typed_config::{config_node_of, typed_values}; use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; use sqlbuild_python_base::compile_driver::constants::NATIVE_HEADER_COLUMNS; use sqlbuild_python_base::compile_driver::constants::PARSED_HEADER_COLUMNS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::native_config_error::native_config_error; +use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; +use sqlbuild_python_front::model_config::main::headers::native_schema_column::native_schema_column; +use sqlbuild_python_front::model_config::main::headers::parsed_audit_identity::parsed_audit_identity; +use sqlbuild_python_front::model_config::main::model_nodes::column_migration_error::column_migration_error; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use sqlbuild_python_front::model_config::models::HeaderColumn; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::HeaderFamilies; +use sqlbuild_python_front::model_config::models::NativeColumnSpec; +use sqlbuild_python_front::model_config::types::HeaderValue; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; @@ -222,7 +225,7 @@ pub(crate) fn header_schema<'py>( let resolved: EnumColumns<'py> = enum_columns(context, (&enums, source.visible), source.values, columns)?; let fact: ModelSchemaFact = ModelSchemaFact { - audit_count: u32::try_from(audits.len()).unwrap_or(u32::MAX), + audit_count: u32::try_from(audits.count()).unwrap_or(u32::MAX), columns: resolved.facts, dynamic_columns: family_facts, type_enforcement, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs index 2e44b18827..5ac32c69f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs @@ -18,7 +18,6 @@ use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use crate::bindings::_helpers::attachments::argument_values::argument_pairs; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, validation_outcome, @@ -35,15 +34,18 @@ use crate::bindings::_helpers::input_loops::model_loop_named::{ SQL_HOOK_DECLARATION, named_usages, }; use crate::bindings::_helpers::input_loops::model_loop_visibility::ModelVisibility; -use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, - expand_declarations, -}; -use crate::bindings::_helpers::render::sql_macro_host::SqlMacroSource; -use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::attachments::main::argument_pairs::argument_pairs; +use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; +use sqlbuild_python_front::render::models::DeclarationExpansion; +use sqlbuild_python_front::render::models::DeclarationExpansionClasses; +use sqlbuild_python_front::render::models::ExpandedDeclarations; +use sqlbuild_python_front::render::models::ExpansionConsumer; +use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; +use sqlbuild_python_front::sqltext::main::interpolated::interpolated; +use sqlbuild_python_front::sqltext::models::InterpolationSources; const REFERENCES_METHOD: &str = "references"; const STATEMENT_ATTRIBUTE: &str = "statement"; @@ -489,7 +491,7 @@ impl<'py> HookRun<'_, '_, 'py> { scope: &Scope<'_, 'py>, ) -> PyResult> { let (references, stop) = declaration_scan(sql, Some(&self.model_loop.syntax)); - let classes: ExpansionClasses<'py> = ExpansionClasses { + let classes: DeclarationExpansionClasses<'py> = DeclarationExpansionClasses { usage_record: self.context.classes.usage_record.clone(), compile_error: self.context.classes.compile_error.clone(), }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index 164c1bf4bf..d75cfcf5b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -22,21 +22,25 @@ use crate::bindings::_helpers::compiled_project::project::{ use crate::bindings::_helpers::compiled_views::model_input_view::{ InputProjections, ModelInputHandles, ModelInputRecord, SchemaHandles, SchemaRecord, }; -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::input_loops::model_loop_header::AttachedHeader; use crate::bindings::_helpers::input_loops::model_loop_inputs::{ HOOK_KEYS, reference_fact, stripped_values, }; -use crate::bindings::_helpers::model_config::config_nodes::carried_node; -use crate::bindings::_helpers::model_config::typed_config::{retain_values, typed_values}; -use crate::bindings::_helpers::render::declaration_expansion::{ExpandedDeclarations, span_rows}; -use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; -use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; -use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_front::discovery::models::ModelFileTexts; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; +use sqlbuild_python_front::model_config::main::config::retain_values::retain_values; +use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; +use sqlbuild_python_front::render::main::calls::span_row::span_row; +use sqlbuild_python_front::render::main::declarations::span_rows::span_rows; +use sqlbuild_python_front::render::models::ExpandedDeclarations; +use sqlbuild_python_front::render::models::ExpansionOutput; +use sqlbuild_python_front::render::types::SpanRow; +use sqlbuild_python_front::sqltext::models::ScannedReference; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs index ba2a752650..99dff69f7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs @@ -14,12 +14,13 @@ use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, VisibleDeclarations, compile_error, }; use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; -use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, StateInput, expanded, -}; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::render::main::expansion::expanded::expanded; +use sqlbuild_python_front::render::models::ExpansionOutput; +use sqlbuild_python_front::render::models::NativeMacroExpansion; +use sqlbuild_python_front::render::models::StateInput; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; /// Fields of `_VisibleModelDeclarations` a `DeclarationResolutionContext` is built from. const CONTEXT_FIELDS: [&str; 9] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs index 77d6b680e6..d0dfcb0890 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs @@ -58,7 +58,7 @@ pub(crate) const AUDIT_DECLARATION: NamedKind = NamedKind { label: "Generic audit", role: "audits/generic", }; -use crate::bindings::_helpers::scopes::compile_usages::CURRENT_FOLDER; +use sqlbuild_python_front::scopes::constants::CURRENT_FOLDER; use sqlbuild_scopes::scope_index::main::declared_scope_path_message::declared_scope_path_message; /// The usage of the indexed declaration `name`, if any, reporting S006 when it is hidden. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs index 1ce7960b6a..0a25bfd463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs @@ -8,8 +8,8 @@ use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::input_loops::model_loop_inputs::present; -use crate::bindings::_helpers::model_config::header_metadata::HeaderColumn; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::models::HeaderColumn; const MODEL_SCHEMA_KEY: &str = "model_schema"; const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs index 140e7581b2..0c032f1090 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs @@ -9,9 +9,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, VisibleDeclarations}; -use crate::bindings::_helpers::render::sql_macro_host::MacroScopes; -use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::render::types::MacroScopes; +use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; const NAME_FIELD: &str = "name"; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 0cfad4fc9c..4ea914fa10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -28,13 +28,13 @@ use crate::bindings::_helpers::compiled_project::attachment_inputs::{ }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::NativeModelFile; const MISSING_DESCRIPTION_CODE: &str = "P010"; const DESCRIPTION_LENGTH_CODE: &str = "P027"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index fd40d9e070..fb42efb29e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -17,24 +17,28 @@ use crate::bindings::_helpers::input_loops::model_loop::{ use crate::bindings::_helpers::input_loops::model_loop_macros::{ argument_references as macro_argument_references, expanded_with, }; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; -use crate::bindings::_helpers::model_config::config_templates::{ - PythonHost, Stop, TemplateSources, expanded, template_error, -}; use crate::bindings::_helpers::project_assembly::resources::{ syntax_error_message, syntax_error_messages, }; -use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, - expand_declarations, -}; -use crate::bindings::_helpers::render::sql_macro_host::SqlMacroSource; -use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; -use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::expanded::expanded; +use sqlbuild_python_front::model_config::main::config::native_config_error::native_config_error; +use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; +use sqlbuild_python_front::model_config::main::config::template_error::template_error; +use sqlbuild_python_front::model_config::models::PythonHost; +use sqlbuild_python_front::model_config::models::Stop; +use sqlbuild_python_front::model_config::models::TemplateSources; +use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; +use sqlbuild_python_front::render::models::DeclarationExpansion; +use sqlbuild_python_front::render::models::DeclarationExpansionClasses; +use sqlbuild_python_front::render::models::ExpandedDeclarations; +use sqlbuild_python_front::render::models::ExpansionConsumer; +use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; +use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; +use sqlbuild_python_front::sqltext::main::interpolated::interpolated; +use sqlbuild_python_front::sqltext::models::InterpolationSources; const MACRO_TOKEN: char = '@'; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; @@ -233,7 +237,7 @@ pub(crate) fn expanded_authored_sql<'py>( let mut argument_references: Vec = Vec::new(); let mut expanded_sql: String = interpolated_sql.clone(); if !references.is_empty() || stop.is_some() { - let classes: ExpansionClasses<'py> = ExpansionClasses { + let classes: DeclarationExpansionClasses<'py> = DeclarationExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 7655836248..5a75d64f90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -48,10 +48,10 @@ use crate::bindings::_helpers::input_loops::model_loop_calls::{ use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, scoped_declarations, }; -use crate::bindings::_helpers::render::sql_macro_tests::without_with_list_calls; use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::render::main::sql_macros::without_with_list_calls::without_with_list_calls; const MODEL_MODE: &str = "model"; const MACRO_MODE: &str = "macro"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 0b9d9d3994..96bee69a6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -31,7 +31,6 @@ use crate::bindings::_helpers::cli::machine_output::{ }; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, @@ -47,6 +46,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; const COMMAND_NAME: &str = "janitor"; /// The code of a cleanup that needs a confirmation stdin cannot give. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 90fc1ab2cc..4da294beb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -16,12 +16,11 @@ use crate::bindings::_helpers::janitor::janitor_facts::{ use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, }; -use crate::bindings::_helpers::state_rows::migration_io::{ - read_histories as read_view_histories, stored_columns_binding, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_front::state_rows::main::migrations::read_histories::read_histories as read_view_histories; +use sqlbuild_python_front::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index a37b585ab6..a484b06665 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -14,7 +14,6 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ @@ -29,6 +28,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index b23f41d303..1b11fbaa7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -33,7 +33,6 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; @@ -56,6 +55,7 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs index acad1cfa58..6e9b4b0015 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs @@ -17,10 +17,10 @@ use sqlbuild_core::loader_connection::models::LoaderConnection; use sqlbuild_host::entries::models::ThirdPartyHost; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::model_config::template_data::expanded_template_data; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; const INGESTR_ERROR: ThirdPartyHost = ThirdPartyHost::IngestrIntegrationError; const INGESTR_SOURCE_CONFIG: ThirdPartyHost = ThirdPartyHost::IngestrSourceConfig; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 69dc9d3b6a..4169796e01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -20,16 +20,8 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_target_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; -use crate::bindings::_helpers::discovery::provider_secrets::{ - SecretPolicy, consumer_uses, require_used_secrets, sink_uses, -}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -52,6 +44,16 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; +use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; +use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; +use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; +use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_front::discovery::models::SecretPolicy; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const EXECUTION_COMMAND: &str = "sqb load"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 4fb9ffbf60..8df8b56350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -17,18 +17,12 @@ use sqlbuild_core::load_command::main::select_load_entries::select_load_entries use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compile_attachment::effective_config::{ - build_effective_connection_config, build_effective_runtime_config, -}; use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; use crate::bindings::_helpers::compile_driver::targets::{ expanded_loader_namespace, managed_write_schema, preserved_loader_schemas, }; -use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::model_config::template_data::early_model_templates; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -38,6 +32,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::contracts::main::loader_destination_parts::loader_destination_parts; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const COMPILE_INPUT_CODE: &str = "P001"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 25abafa3f4..7b5c76219a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -22,9 +22,6 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; -use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; -use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; @@ -35,6 +32,10 @@ use crate::bindings::_helpers::planner_scope::values::project_graph; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node_rows; +use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; +use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 2c70680a96..c1507efb2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -27,13 +27,11 @@ use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, wareho use crate::bindings::_helpers::command_runtime::warehouses::{ WarehouseInputs, configured_warehouses, }; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; -use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::{ @@ -52,6 +50,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; /// Each audit and SQL test the snapshot publishes, with the node keys it checks. fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs index 8623ff9f42..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs @@ -1,4 +1 @@ -pub(crate) mod first_compile; -pub(crate) mod module_digests; pub(crate) mod registration; -pub(crate) mod store_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs index eddce4b97a..a6f2ad480b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::macro_bridge::{first_compile, module_digests, store_keys}; +use sqlbuild_python_front::macro_bridge::main::register_first_compile::register_first_compile; +use sqlbuild_python_front::macro_bridge::main::register_module_digests::register_module_digests; +use sqlbuild_python_front::macro_bridge::main::register_store_keys::register_store_keys; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - first_compile::register(module)?; - module_digests::register(module)?; - store_keys::register(module) + register_first_compile(module)?; + register_module_digests(module)?; + register_store_keys(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs index ae49a505f9..a6ed3997df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs @@ -10,9 +10,9 @@ use sqlbuild_cursor::algebra::models::{Scalar, SortKey}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_state::microbatches::models::{BoundInterval, CursorBound}; -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::as_scalar; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::main::scalars::as_scalar::as_scalar; +use sqlbuild_python_front::cursor::models::CursorSession; /// `MicrobatchStateError(message)`. pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs deleted file mode 100644 index c5f3c730f4..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Python objects read in place as authored header values. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; -use pyo3::types::{ - PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, - PyStringMethods, PyTuple, PyTupleMethods, -}; -use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// One Python object viewed as an authored value; parsed results keep the object itself. -#[derive(Clone, Debug)] -pub(crate) struct PyNode<'py>(pub(crate) Bound<'py, PyAny>); - -impl AuthoredNode for PyNode<'_> { - fn kind(&self) -> NodeKind { - let value = &self.0; - if value.is_none() { - NodeKind::Null - } else if let Ok(flag) = value.downcast::() { - NodeKind::Bool(flag.is_true()) - } else if value.is_instance_of::() { - value - .lt(0) - .map_or(NodeKind::Other, |negative| NodeKind::Int { negative }) - } else if value.is_instance_of::() { - NodeKind::Str - } else if value.is_instance_of::() { - NodeKind::List - } else if value.is_instance_of::() { - NodeKind::Tuple - } else if value.is_instance_of::() { - NodeKind::Map - } else { - NodeKind::Other - } - } - - fn integer(&self) -> Option { - if !self.0.is_instance_of::() - && self.0.is_instance_of::() - && let Ok(number) = self.0.extract::() - { - Some(number) - } else { - None - } - } - - fn number(&self) -> Option { - let value = &self.0; - let is_number: bool = value.is_instance_of::() - || (value.is_instance_of::() && !value.is_instance_of::()); - if !is_number { - return None; - } - if let Ok(number) = value.extract::() { - return Some(number); - } - let Ok(number) = self.python_str().parse::() else { - return None; - }; - Some(number) - } - - fn text(&self) -> Option { - self.with_text(str::to_owned) - } - - /// Read a string's text; lone surrogates, rejected where text enters a compile, read as U+FFFD. - fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { - match self.0.downcast::() { - Ok(value) => Some(read(&value.to_string_lossy())), - Err(_) => None, - } - } - - fn is_text(&self, text: &str) -> bool { - self.with_text(|value| value == text).unwrap_or(false) - } - - fn items(&self) -> Vec { - if let Ok(items) = self.0.downcast::() { - items.iter().map(PyNode).collect() - } else if let Ok(items) = self.0.downcast::() { - items.iter().map(PyNode).collect() - } else { - Vec::new() - } - } - - fn entries(&self) -> Vec<(Self, Self)> { - if let Ok(entries) = self.0.downcast::() { - entries.iter().map(entry_nodes).collect() - } else { - Vec::new() - } - } - - fn python_str(&self) -> String { - self.0 - .str() - .map(|text| text.to_string_lossy().into_owned()) - .unwrap_or_default() - } - - fn python_repr(&self) -> String { - self.0 - .repr() - .map(|text| text.to_string_lossy().into_owned()) - .unwrap_or_default() - } - - fn is_date_like(&self) -> bool { - let py = self.0.py(); - py.import("datetime") - .and_then(|module| native_field(&module, "date")) - .and_then(|date| self.0.is_instance(&date)) - .unwrap_or(false) - } -} - -fn entry_nodes<'py>( - (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> (PyNode<'py>, PyNode<'py>) { - (PyNode(key), PyNode(value)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs deleted file mode 100644 index 462efe8530..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ /dev/null @@ -1,422 +0,0 @@ -//! `${...}` templates in authored Python config values and effective vars, expanded natively. - -use std::cell::RefCell; - -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; -use pyo3::types::{ - PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, - PyTuple, PyTupleMethods, -}; -use pyo3::{FromPyObject, PyErr}; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::templates::errors::TemplateError; -use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; -use sqlbuild_model_config::templates::main::expand_template_text::expand_template_text; -use sqlbuild_model_config::templates::main::template_error_message::template_error_message; -use sqlbuild_model_config::templates::models::{ - ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, -}; -use sqlbuild_model_config::templates::types::TemplateHost; - -use crate::bindings::_helpers::authored_values::project_vars::project_var_text; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::errors::NativeConfigError; - -const TEMPLATE_OPEN_TOKEN: &str = "${"; -const ENVIRONMENT_READ: &str = "env"; -const CONTEXT_READ: &str = "ctx"; -const EFFECTIVE_VARS_LABEL: &str = "effective vars"; -const VALUE_PATH: &str = "its value"; -const BUILTINS_MODULE: &str = "builtins"; -const ASCII_FUNCTION: &str = "ascii"; -const ENCODE: &str = "encode"; -const UTF32_LE: &str = "utf-32-le"; -const SURROGATE_PASS: &str = "surrogatepass"; - -/// Variables, the process environment and context values, all read through Python. -pub(crate) struct PythonHost<'py> { - py: Python<'py>, - sources: TemplateSources<'py>, - label: String, - effective: Option>, - reads: RefCell>, - raised: RefCell>, -} - -/// Effective vars resolved on demand, as Python's `_EffectiveVarsResolver` resolves them. -struct EffectiveVars<'py> { - resolved: Bound<'py, PyDict>, - resolving: RefCell>, -} - -impl<'py> PythonHost<'py> { - /// Read variables, the environment and context values from these Python objects. - pub(crate) fn new(py: Python<'py>, sources: TemplateSources<'py>, label: &str) -> Self { - Self { - py, - sources, - label: label.to_owned(), - effective: None, - reads: RefCell::new(Vec::new()), - raised: RefCell::new(None), - } - } - - /// Return the `(kind, name)` reads in order. - pub(crate) fn into_reads(self) -> Vec<(&'static str, String)> { - self.reads.into_inner() - } - - /// Hold a Python error until the expansion returns, and stop evaluation. - fn hold(&self, error: PyErr) -> TemplateFailure { - self.raised.borrow_mut().get_or_insert(error); - TemplateFailure::Host - } - - /// The stop for `failure`, taking the held Python error of a host failure. - pub(crate) fn stop(&self, failure: TemplateFailure) -> Stop { - match failure { - TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { - Stop::Failure(error) - } - TemplateFailure::Host => { - Stop::Python(self.raised.borrow_mut().take().unwrap_or_else(|| { - PyRuntimeError::new_err("template expansion stopped without an error") - })) - } - } - } - - fn text_of(&self, value: &Bound<'py, PyAny>) -> Result { - value - .downcast::() - .map_err(|error| self.hold(error.into()))? - .to_str() - .map(str::to_owned) - .map_err(|error| self.hold(error)) - } - - /// Resolve effective var `name`, keeping Python's resolving stack on errors. - fn resolve( - &self, - effective: &EffectiveVars<'py>, - name: &str, - ) -> Result>, TemplateFailure> { - let Some(raw) = self - .sources - .0 - .get_item(name) - .map_err(|error| self.hold(error))? - else { - return Ok(None); - }; - if let Some(value) = effective - .resolved - .get_item(name) - .map_err(|error| self.hold(error))? - { - return Ok(Some(value)); - } - if effective.resolving.borrow().iter().any(|key| key == name) { - let mut cycle = effective.resolving.borrow().clone(); - cycle.push(name.to_owned()); - return Err(TemplateFailure::Invalid(TemplateError::Message(format!( - "{EFFECTIVE_VARS_LABEL} contain a cyclic reference: {}", - cycle.join(" -> ") - )))); - } - effective.resolving.borrow_mut().push(name.to_owned()); - let value = if raw.is_instance_of::() { - let text = self.text_of(&raw)?; - let options = TemplateOptions { - allow_context: false, - preserve_context_tokens: false, - preserve_unknown_context: false, - }; - PyString::new(self.py, &expand_template_text(self, &text, options)?).into_any() - } else { - raw - }; - effective.resolving.borrow_mut().pop(); - effective - .resolved - .set_item(name, &value) - .map_err(|error| self.hold(error))?; - Ok(Some(value)) - } -} - -/// The variables, the `os.environ` mapping and the context values one expansion reads. -#[derive(FromPyObject)] -pub(crate) struct TemplateSources<'py>( - pub(crate) Bound<'py, PyDict>, - pub(crate) Bound<'py, PyAny>, - pub(crate) Bound<'py, PyDict>, -); - -/// A template error, or a Python error raised while reading values or building containers. -pub(crate) enum Stop { - Failure(TemplateError), - Python(PyErr), -} - -/// The `CompileInputError` Python raises for a template error in the context `label`. -pub(crate) fn template_error(error: &TemplateError, label: &str) -> ConfigError { - ConfigError::compile(template_error_message(error, label)) -} - -impl<'py> TemplateHost for PythonHost<'py> { - type Value = Bound<'py, PyAny>; - - fn variable(&self, name: &str) -> Result, TemplateFailure> { - if let Some(effective) = &self.effective { - return self.resolve(effective, name); - } - self.sources - .0 - .get_item(name) - .map_err(|error| self.hold(error)) - } - - fn environment(&self, name: &str) -> Result, TemplateFailure> { - self.reads - .borrow_mut() - .push((ENVIRONMENT_READ, name.to_owned())); - let environment: &Bound<'py, PyAny> = &self.sources.1; - if environment - .contains(name) - .map_err(|error| self.hold(error))? - { - environment - .get_item(name) - .map(Some) - .map_err(|error| self.hold(error)) - } else { - Ok(None) - } - } - - fn context(&self, name: &str) -> Result, TemplateFailure> { - self.reads - .borrow_mut() - .push((CONTEXT_READ, name.to_owned())); - match self - .sources - .2 - .get_item(name) - .map_err(|error| self.hold(error))? - { - None => Ok(ContextValue::Unknown), - Some(value) if value.is_none() => Ok(ContextValue::Unavailable), - Some(value) => Ok(ContextValue::Value(value)), - } - } - - fn text(&self, text: &str) -> Result { - Ok(PyString::new(self.py, text).into_any()) - } - - fn boolean(&self, flag: bool) -> Result { - Ok(PyBool::new(self.py, flag).to_owned().into_any()) - } - - fn null(&self) -> Self::Value { - self.py.None().into_bound(self.py) - } - - fn scalar(&self, value: &Self::Value) -> Result { - if value.is_none() { - return Ok(Scalar::Null); - } - if let Ok(flag) = value.downcast_exact::() { - return Ok(Scalar::Bool(flag.is_true())); - } - let text = value.str().map_err(|error| self.hold(error))?; - self.text_of(text.as_any()).map(Scalar::Text) - } - - fn render(&self, value: &Self::Value, label: &str) -> Result { - if value.is_exact_instance_of::() { - return self.text_of(value); - } - match project_var_text(value, label) { - Ok(text) => Ok(text), - Err(error) if error.is_instance_of::(self.py) => Err( - TemplateFailure::Invalid(TemplateError::Message(error.value(self.py).to_string())), - ), - Err(error) => Err(self.hold(error)), - } - } - - fn label(&self) -> &str { - &self.label - } -} - -/// The expanded value, or the native error expansion stopped at, and the reads in order. -pub(crate) type TemplateOutcome<'py> = ( - Result, Py>, - Vec<(&'static str, String)>, -); - -/// `value` with every template expanded under `options`, or the error, with its reads. -pub(crate) fn expand_config_templates<'py>( - py: Python<'py>, - value: &Bound<'py, PyAny>, - sources: TemplateSources<'py>, - (options, label): (TemplateOptions, &str), -) -> PyResult> { - let host = PythonHost::new(py, sources, label); - let result = expanded(&host, value, options); - outcome(py, host, result, label) -} - -/// The first `(var name, value path, UTF-8 byte offset)` of a lone surrogate in raw vars. -pub(crate) fn lone_surrogate_site<'py>( - raw_values: &Bound<'py, PyDict>, -) -> PyResult, String, usize)>> { - for (key, value) in raw_values.iter() { - if let Some((path, offset)) = surrogate_path(&value, VALUE_PATH.to_owned())? { - return Ok(Some((key.str()?, path, offset))); - } - } - Ok(None) -} - -/// Python's former `_reject_lone_surrogates` walk: dict keys then items, lists and tuples. -fn surrogate_path(value: &Bound<'_, PyAny>, path: String) -> PyResult> { - if let Ok(mapping) = value.downcast::() { - for (key, item) in mapping.iter() { - if let Some(site) = surrogate_path(&key, format!("{path} key"))? { - return Ok(Some(site)); - } - let key_ascii = PyModule::import(value.py(), BUILTINS_MODULE)? - .getattr(ASCII_FUNCTION)? - .call1((&key,))? - .str()? - .to_string_lossy() - .into_owned(); - if let Some(site) = surrogate_path(&item, format!("{path}[{key_ascii}]"))? { - return Ok(Some(site)); - } - } - return Ok(None); - } - if value.is_instance_of::() || value.is_instance_of::() { - for (index, item) in value.try_iter()?.enumerate() { - if let Some(site) = surrogate_path(&item?, format!("{path}[{index}]"))? { - return Ok(Some(site)); - } - } - return Ok(None); - } - let Ok(text) = value.downcast::() else { - return Ok(None); - }; - Ok(first_surrogate_offset(text)?.map(|offset| (path, offset))) -} - -/// The UTF-8 byte offset of the first lone surrogate in `text`. -fn first_surrogate_offset(text: &Bound<'_, PyString>) -> PyResult> { - let encoded = text.call_method1(ENCODE, (UTF32_LE, SURROGATE_PASS))?; - let mut offset = 0; - for unit in encoded.extract::>()?.as_chunks::<4>().0 { - let code_point = u32::from_le_bytes([unit[0], unit[1], unit[2], unit[3]]); - match char::from_u32(code_point) { - Some(character) => offset += character.len_utf8(), - None => return Ok(Some(offset)), - } - } - Ok(None) -} - -/// The resolved effective vars, or the native error resolution stopped at, with the reads. -pub(crate) fn resolve_effective_vars<'py>( - py: Python<'py>, - raw_values: &Bound<'py, PyDict>, - environment: Bound<'py, PyAny>, -) -> PyResult> { - let mut host = PythonHost::new( - py, - TemplateSources(raw_values.clone(), environment, PyDict::new(py)), - EFFECTIVE_VARS_LABEL, - ); - let resolved = PyDict::new(py); - host.effective = Some(EffectiveVars { - resolved: resolved.clone(), - resolving: RefCell::new(Vec::new()), - }); - let mut result: Result, Stop> = Ok(resolved.clone().into_any()); - for key in raw_values.keys() { - let name = key.str()?.to_str()?.to_owned(); - if let Err(failure) = host.variable(&name) { - result = Err(host.stop(failure)); - break; - } - } - outcome(py, host, result, EFFECTIVE_VARS_LABEL) -} - -fn outcome<'py>( - py: Python<'py>, - host: PythonHost<'py>, - result: Result, Stop>, - label: &str, -) -> PyResult> { - let value = match result { - Ok(value) => Ok(value), - Err(Stop::Failure(error)) => Err(native_config_error(py, template_error(&error, label))?), - Err(Stop::Python(error)) => return Err(error), - }; - Ok((value, host.into_reads())) -} - -/// Expand every template in `value` and the containers inside it. -pub(crate) fn expanded<'py>( - host: &PythonHost<'py>, - value: &Bound<'py, PyAny>, - options: TemplateOptions, -) -> Result, Stop> { - let py = host.py; - if let Ok(text) = value.downcast::() { - let text = text.to_str().map_err(Stop::Python)?; - if !text.contains(TEMPLATE_OPEN_TOKEN) { - return Ok(value.clone()); - } - return match expand_template_string(host, text, options) { - Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), - Ok(StringExpansion::Value(result)) => Ok(result), - Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), - Err(failure) => Err(host.stop(failure)), - }; - } - if let Ok(mapping) = value.downcast::() { - let result = PyDict::new(py); - for (key, item) in mapping.iter() { - result - .set_item(key, expanded(host, &item, options)?) - .map_err(Stop::Python)?; - } - return Ok(result.into_any()); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyList::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyTuple::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - Ok(value.clone()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs deleted file mode 100644 index 80b69f4d79..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! The exact config error handed to Python. - -use pyo3::pyclass; - -/// One exact error: the Python class to raise, its message, code, help and named config key. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -pub(crate) struct NativeConfigError { - pub(crate) class_name: &'static str, - pub(crate) message: String, - pub(crate) code: Option<&'static str>, - pub(crate) help: Option, - pub(crate) key: Option, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs deleted file mode 100644 index ac61d18106..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs +++ /dev/null @@ -1,445 +0,0 @@ -//! Retained audit contracts; Python-authored factory results remain opaque values. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::types::{PyString, PyStringMethods, PyTuple}; -use sqlbuild_attachments::audits::models::ArgumentValue; -use sqlbuild_core::json::models::{JsonInteger, JsonValue}; -use sqlbuild_discovery::model_files::main::config_value::config_value; -use sqlbuild_discovery::model_files::models::NativeAudit; -use sqlbuild_model_config::config_values::models::ConfigNode; -use sqlbuild_model_config::header_metadata::constants::{ - ACCEPTED_VALUES_ARGUMENT, ACCEPTED_VALUES_AUDIT, -}; -use sqlbuild_model_config::header_metadata::models::ParsedAudit; -use sqlbuild_sqltext::compiler::models::AuthoredValue; -use std::sync::Arc; - -use crate::bindings::_helpers::attachments::argument_values::{ - argument_pairs, config_argument_value, -}; -use crate::bindings::_helpers::model_config::config_nodes::{ - PyConfigNode, config_node, config_object, -}; -use crate::bindings::_helpers::model_config::contract_classes::contract_classes; -use crate::bindings::_helpers::model_config::header_metadata::{ - ContractClasses, HeaderValue, audit_instance, thresholds, -}; -use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; -use sqlbuild_python_base::native_host::models::AuditProjection; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::result_output::main::json_value::json_value; - -/// Native-built specs never become Python audit records just to be read by attachment. -pub(crate) enum AuditInstance<'py> { - Native(Arc>, Bound<'py, PyAny>), - Python(Bound<'py, PyAny>), -} - -impl<'py> AuditInstance<'py> { - /// Equality with the builtin generated enum audit, without constructing argument containers. - pub(crate) fn accepts_enum(&self, values: &[PyConfigNode]) -> PyResult { - let Self::Native(spec, location) = self else { - return Ok(false); - }; - let py = location.py(); - if spec.definition_name.exact_text(py)? != ACCEPTED_VALUES_AUDIT - || spec.thresholds.is_some() - { - return Ok(false); - } - for name in [ - "name", - "description", - "severity", - "run_scope", - "minimum_samples", - "evidence_limit", - "meta", - ] { - if !self.field(name)?.is_none() { - return Ok(false); - } - } - if !self.field("always_run")?.eq(false)? || spec.arguments.len() != 1 { - return Ok(false); - } - let (key, authored) = &spec.arguments[0]; - let matches = match key { - ConfigNode::Str(key) => key == ACCEPTED_VALUES_ARGUMENT, - ConfigNode::Host(_) => { - let key = key.object(py)?; - key.is_instance_of::() && key.eq(ACCEPTED_VALUES_ARGUMENT)? - } - _ => false, - }; - if !matches { - return Ok(false); - } - let ConfigNode::Tuple(authored) = authored else { - return Ok(false); - }; - if authored.len() != values.len() { - return Ok(false); - } - for (left, right) in authored.iter().zip(values) { - if !left.object(py)?.eq(right.object(py)?)? { - return Ok(false); - } - } - Ok(true) - } - - pub(crate) fn meta_present(&self) -> PyResult { - match self { - Self::Native(spec, _) => Ok(spec - .meta - .as_ref() - .is_some_and(|value| !matches!(value, ConfigNode::Null))), - Self::Python(value) => Ok(!field(value, "meta")?.is_none()), - } - } - - pub(crate) fn threshold_policy(&self) -> PyResult<(bool, bool)> { - match self { - Self::Native(spec, _) => Ok(( - spec.thresholds.is_some(), - spec.thresholds.is_some_and(|value| value.error.is_some()), - )), - Self::Python(value) => { - let thresholds = field(value, "thresholds")?; - Ok(( - !thresholds.is_none(), - !thresholds.is_none() && !field(&thresholds, "error")?.is_none(), - )) - } - } - } - - pub(crate) fn field(&self, name: &str) -> PyResult> { - let (spec, location) = match self { - Self::Native(spec, location) => (spec, location), - Self::Python(value) => return field(value, name), - }; - let py = location.py(); - let value = match name { - "definition_name" => return spec.definition_name.object(py), - "name" => &spec.name, - "description" => &spec.description, - "run_scope" => &spec.run_scope, - "minimum_samples" => &spec.minimum_samples, - "evidence_limit" => &spec.evidence_limit, - "meta" => &spec.meta, - "always_run" => { - return match &spec.always_run { - Some(value) => value.object(py), - None => false.into_bound_py_any(py), - }; - } - "severity" => { - let classes = contract_classes(py)?; - let classes: ContractClasses<'_> = classes.extract()?; - return match &spec.severity { - Some(value) => Ok(classes - .severities - .get_item(value.object(py)?)? - .unwrap_or_else(|| py.None().into_bound(py))), - None => Ok(py.None().into_bound(py)), - }; - } - "thresholds" => { - let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; - return thresholds(py, &classes, spec.thresholds.as_ref()); - } - "location" => return Ok(location.clone()), - _ => return Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), - }; - value - .as_ref() - .map(|value| value.object(py)) - .transpose() - .map(|value| value.unwrap_or_else(|| py.None().into_bound(py))) - } - - pub(crate) fn argument(&self, name: &str) -> PyResult>> { - match self { - Self::Native(spec, location) => { - for (key, value) in &spec.arguments { - let matches = match key { - ConfigNode::Str(key) => key == name, - ConfigNode::Host(_) => { - let key = key.object(location.py())?; - key.is_instance_of::() && key.eq(name)? - } - _ => false, - }; - if matches { - return value.object(location.py()).map(Some); - } - } - Ok(None) - } - Self::Python(value) => field(value, "arguments")?.cast::()?.get_item(name), - } - } - - pub(crate) fn arguments(&self) -> PyResult> { - match self { - Self::Native(spec, location) => { - let mut arguments = Vec::with_capacity(spec.arguments.len()); - for (key, value) in &spec.arguments { - let key = match key { - ConfigNode::Str(key) => key.clone(), - ConfigNode::Host(_) => { - let key = key.object(location.py())?; - let Ok(key) = key.cast::() else { - continue; - }; - key.to_string_lossy().into_owned() - } - _ => continue, - }; - arguments.push((key, config_argument_value(location.py(), value)?)); - } - Ok(arguments) - } - Self::Python(value) => argument_pairs(field(value, "arguments")?.cast()?), - } - } - - pub(crate) fn meta(&self) -> PyResult> { - match self { - Self::Native(spec, location) => spec - .meta - .as_ref() - .map(|value| config_json_value(location.py(), value)) - .transpose() - .map(Option::flatten), - Self::Python(value) => { - let meta = field(value, "meta")?; - if meta.is_none() { - Ok(None) - } else { - json_value(&meta) - } - } - } - } -} - -/// Native discovery facts keep their authored values rather than building audit records. -pub(crate) fn authored_audit( - py: Python<'_>, - spec: &NativeAudit, -) -> PyResult> { - let value = |value: &AuthoredValue| match config_value(value) { - Some(value) => Ok(value), - None => config_node(authored_value_to_python(py, value)?.bind(py)), - }; - Ok(ParsedAudit { - definition_name: ConfigNode::Str(spec.definition_name.clone()), - arguments: spec - .arguments - .iter() - .map(|(key, item)| Ok((ConfigNode::Str(key.clone()), value(item)?))) - .collect::>()?, - name: spec.name.clone().map(ConfigNode::Str), - description: spec.description.clone().map(ConfigNode::Str), - severity: spec.severity.clone().map(ConfigNode::Str), - run_scope: spec.run_scope.clone().map(ConfigNode::Str), - always_run: spec.always_run.map(ConfigNode::Bool), - thresholds: spec.thresholds, - minimum_samples: spec.minimum_samples.as_ref().map(value).transpose()?, - evidence_limit: spec.evidence_limit.as_ref().map(value).transpose()?, - meta: spec.meta.as_ref().map(value).transpose()?, - }) -} - -/// The parsed specs and genuine Python audit objects of one model or column. -pub(crate) struct HeaderAudits { - specs: Vec>, - native_generated: Vec>, - public: AuditProjection, -} - -impl HeaderAudits { - pub(crate) fn new( - py: Python<'_>, - classes: &ContractClasses<'_>, - specs: &[ParsedAudit], - location: &Bound<'_, PyAny>, - ) -> PyResult { - let custom = if classes.uses_default_classes()? { - None - } else { - Some( - specs - .iter() - .map(|spec| audit_instance(py, classes, spec, location).map(Bound::unbind)) - .collect::>()?, - ) - }; - Ok(Self { - specs: specs - .iter() - .map(|spec| retain_audit(py, spec)) - .collect::>()?, - native_generated: Vec::new(), - public: AuditProjection { - custom, - generated: Vec::new(), - location: location.clone().unbind(), - records: PyOnceLock::new(), - }, - }) - } - - pub(crate) fn extend(&mut self, generated: Vec>) { - self.public - .generated - .extend(generated.into_iter().map(Bound::unbind)); - } - - pub(crate) fn push_native(&mut self, spec: ParsedAudit) { - self.native_generated.push(spec); - } - - pub(crate) fn len(&self) -> usize { - self.specs.len() + self.public.generated.len() + self.native_generated.len() - } - - pub(crate) fn native_specs(&self) -> &[ParsedAudit] { - if self.public.custom.is_some() { - &[] - } else { - &self.specs - } - } - - pub(crate) fn python_specs<'py>(&self, py: Python<'py>) -> Vec> { - self.public - .custom - .iter() - .flatten() - .chain(&self.public.generated) - .map(|spec| spec.bind(py).clone()) - .collect() - } - - pub(crate) fn instances<'py>(&self, py: Python<'py>) -> Vec> { - self.native_specs() - .iter() - .map(|spec| { - AuditInstance::Native( - Arc::new(spec.clone()), - self.public.location.bind(py).clone(), - ) - }) - .chain(self.python_specs(py).into_iter().map(AuditInstance::Python)) - .chain(self.native_generated.iter().map(|spec| { - AuditInstance::Native(Arc::new(spec.clone()), py.None().into_bound(py)) - })) - .collect() - } - - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { - self.public - .records - .get_or_try_init(py, || { - let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; - let mut audits = self - .native_specs() - .iter() - .map(|spec| audit_instance(py, &classes, spec, self.public.location.bind(py))) - .collect::>>()?; - audits.extend(self.python_specs(py)); - for spec in &self.native_generated { - audits.push(audit_instance( - py, - &classes, - spec, - &py.None().into_bound(py), - )?); - } - PyTuple::new(py, audits).map(Bound::unbind) - }) - .map(|value| value.bind(py).clone()) - } - - pub(crate) fn key_rows(&self, py: Python<'_>) -> PyResult)>> { - let mut rows = Vec::with_capacity(self.len()); - for spec in self.instances(py) { - let column = spec.argument("column")?; - rows.push(( - spec.field("definition_name")?.extract()?, - column - .filter(|value| value.is_instance_of::()) - .map(|value| value.extract()) - .transpose()?, - )); - } - Ok(rows) - } -} - -/// Preserve parsed leaves without rebuilding their public argument dictionaries. -pub(crate) fn retain_audit( - py: Python<'_>, - spec: &ParsedAudit, -) -> PyResult> { - let optional = |node: &Option| node.as_ref().map(|node| node.retained(py)).transpose(); - Ok(ParsedAudit { - definition_name: spec.definition_name.retained(py)?, - arguments: spec - .arguments - .iter() - .map(|(key, value)| Ok((key.retained(py)?, value.retained(py)?))) - .collect::>()?, - name: optional(&spec.name)?, - description: optional(&spec.description)?, - severity: optional(&spec.severity)?, - run_scope: optional(&spec.run_scope)?, - always_run: optional(&spec.always_run)?, - thresholds: spec.thresholds, - minimum_samples: optional(&spec.minimum_samples)?, - evidence_limit: optional(&spec.evidence_limit)?, - meta: optional(&spec.meta)?, - }) -} - -/// Read metadata without projecting retained containers; opaque user values keep Python semantics. -fn config_json_value(py: Python<'_>, value: &PyConfigNode) -> PyResult> { - Ok(Some(match value { - ConfigNode::Null => JsonValue::Null, - ConfigNode::Bool(value) => JsonValue::Bool(*value), - ConfigNode::Int(value) => JsonValue::Integer(JsonInteger::from(*value)), - ConfigNode::Float(value) => JsonValue::Float(*value), - ConfigNode::Str(value) => JsonValue::String(value.clone()), - ConfigNode::List(items) | ConfigNode::Tuple(items) => { - let mut result: Vec = Vec::with_capacity(items.len()); - for item in items { - let Some(value) = config_json_value(py, item)? else { - return Ok(None); - }; - result.push(value); - } - JsonValue::Array(result) - } - ConfigNode::Map(entries) => { - let mut result: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); - for (key, item) in entries { - let ConfigNode::Str(key) = key else { - return json_value(&config_object(py, value)?); - }; - let Some(value) = config_json_value(py, item)? else { - return Ok(None); - }; - result.push((key.clone(), value)); - } - JsonValue::Object(result) - } - ConfigNode::Host(_) => return json_value(&config_object(py, value)?), - })) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs deleted file mode 100644 index e3d1f1019f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Native dynamic-column contracts with a shared, lazy public projection. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyTuple; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; - -use sqlbuild_python_base::native_host::models::FamilyProjection; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::types::RecordId; - -/// Parsed family specs stay native until a public schema attribute needs records. -pub(crate) struct HeaderFamilies { - pub(crate) specs: Vec, - public: FamilyProjection, -} - -impl HeaderFamilies { - pub(crate) fn new(class: &Bound<'_, PyAny>, specs: Vec) -> PyResult { - let result = Self { - specs, - public: FamilyProjection::new( - class, - !class.is(&record_class( - class.py(), - RecordId::SchemaDynamicColumnFamily, - )?), - ), - }; - if result.public.custom { - let _ = result.project(class.py())?; - } - Ok(result) - } - - pub(crate) fn read(&self, py: Python<'_>) -> PyResult> { - if self.public.custom { - family_specs(self.project(py)?.as_any()) - } else { - Ok(self.specs.clone()) - } - } - - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { - self.public.project(py, &self.specs) - } -} - -/// A Python-authored family contract crosses into native ownership once. -pub(crate) fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { - families - .try_iter()? - .map(|family| { - let family = family?; - Ok(DynamicFamily { - name: field(&family, "name")?.extract()?, - pivot_column: field(&family, "pivot_column")?.extract()?, - value_column: field(&family, "value_column")?.extract()?, - aggregate: field(&family, "aggregate")?.extract()?, - data_type: field(&family, "type")?.extract()?, - name_pattern: field(&family, "name_pattern")?.extract()?, - }) - }) - .collect() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index ce22851d76..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -1,22 +1 @@ -pub(crate) mod authored_nodes; -pub(crate) mod column_migrations; -pub(crate) mod config_classes; -pub(crate) mod config_errors; -pub(crate) mod config_nodes; -pub(crate) mod config_templates; -pub(crate) mod config_values; -pub(crate) mod contract_classes; -pub(crate) mod described_models; -pub(crate) mod errors; -pub(crate) mod getters; -pub(crate) mod header_audits; -pub(crate) mod header_columns; -pub(crate) mod header_families; -pub(crate) mod header_metadata; -pub(crate) mod model_config_builder; -pub(crate) mod model_validator; -pub(crate) mod project_defaults; pub(crate) mod registration; -pub(crate) mod template_data; -pub(crate) mod typed_config; -pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs deleted file mode 100644 index 6ff8e6da57..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ /dev/null @@ -1,458 +0,0 @@ -//! The native model config build handed Python inputs once, and its results projected for Python. - -use std::collections::HashMap; - -use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{PyErr, pyclass, pymethods}; -use sqlbuild_discovery::model_files::main::config_value::config_value; -use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; -use sqlbuild_model_config::config_presence::models::MacroPath; -use sqlbuild_model_config::config_values::models::ConfigNode; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::model_config_build::main::build_model_config::build_model_config; -use sqlbuild_model_config::model_config_build::main::inherited_config::inherited_config; -use sqlbuild_model_config::model_config_build::main::model_context_values::model_context_values; -use sqlbuild_model_config::model_config_build::models::{ - BuiltModelConfig, ConfigBuildHalt, ConfigBuildInputs, HeaderEntryFacts, ModelHeader, -}; -use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; -use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; -use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; -use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; -use sqlbuild_sqltext::compiler::models::AuthoredValue; - -use crate::bindings::_helpers::authored_values::project_vars::project_var_text; -use crate::bindings::_helpers::model_config::config_classes::{ConfigClasses, built_config}; -use crate::bindings::_helpers::model_config::config_errors::config_exception; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::config_nodes::{ - HostHandle, PyConfigEntries, PyConfigNode, carried_node, config_dict, config_entries, - config_node, config_object, hook_entry, is_exact_container, -}; -use crate::bindings::_helpers::model_config::errors::NativeConfigError; -use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; -use crate::bindings::_helpers::model_config::typed_config::{BuiltConfigValues, built_values}; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; - -/// The config keys a model's context values read. -const CONTEXT_KEYS: [&str; 3] = ["database", "schema", "alias"]; -const BUILD_STOPPED: &str = "a config value read as text without an error"; - -/// A header's public mapping, parsed facts and native authored values. -type HeaderBuildInputs<'a, 'py> = ( - &'a Bound<'py, PyDict>, - Option<&'a HeaderFacts>, - Option<&'a [(String, AuthoredValue)]>, -); - -/// The project-wide inputs every model's config build reads, converted once per compile. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeModelConfigBuilder { - inputs: ConfigBuildInputs, - path_keys: Vec, - environment: Py, -} - -/// One model's built config before Python wraps it in `CompileModelConfig`. -pub(crate) struct BuiltConfig<'py> { - pub(crate) values: Bound<'py, PyDict>, - pub(crate) nodes: Option, - pub(crate) header_keys: Vec, - pub(crate) logical_schema: Option>, - pub(crate) layer_schema_configured: bool, - pub(crate) logical_database: Option>, - pub(crate) retention: RetentionOverride, - pub(crate) table_type: TableTypeOverride, - pub(crate) reads: Vec<(&'static str, String)>, -} - -/// A native build's outcome: the built config, or the config error it raises. -pub(crate) enum ConfigBuild<'py> { - Built(BuiltConfig<'py>), - Rejected(Py), -} - -/// `((target name, run id), (database, schema) or None)` of the active target. -pub(crate) type TargetInputs<'py> = ( - (Bound<'py, PyAny>, Bound<'py, PyAny>), - Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, -); - -/// The process environment and project-var rendering, read through Python. -struct PythonConfigHost<'py> { - environment: Bound<'py, PyAny>, -} - -impl ConfigBuildHost for PythonConfigHost<'_> { - type Handle = HostHandle; - type Error = PyErr; - - fn environment(&self, name: &str) -> PyResult> { - if !self.environment.contains(name)? { - return Ok(None); - } - Ok(Some(config_node(&self.environment.get_item(name)?)?)) - } - - fn render(&self, value: &PyConfigNode, label: &str) -> PyResult> { - let py: Python<'_> = self.environment.py(); - let object: Bound<'_, PyAny> = config_object(py, value)?; - if let Ok(text) = object.downcast_exact::() { - return Ok(Ok(text.to_str()?.to_owned())); - } - match project_var_text(&object, label) { - Ok(text) => Ok(Ok(text)), - Err(error) if error.is_instance_of::(py) => { - Ok(Err(error.value(py).to_string())) - } - Err(error) => Err(error), - } - } - - fn text_error(&self, value: &PyConfigNode) -> PyErr { - let py: Python<'_> = self.environment.py(); - let read = || -> PyResult<()> { - let object: Bound<'_, PyAny> = config_object(py, value)?; - let text: Bound<'_, PyString> = match object.downcast::() { - Ok(text) => text.clone(), - Err(_) => object.str()?, - }; - let _ = text.to_str()?; - Ok(()) - }; - match read() { - Ok(()) => PyRuntimeError::new_err(BUILD_STOPPED), - Err(error) => error, - } - } - - fn python_str(&self, value: &PyConfigNode, exact: bool) -> PyResult { - let text: Bound<'_, PyString> = config_object(self.environment.py(), value)?.str()?; - if exact { - return Ok(text.to_str()?.to_owned()); - } - Ok(text.to_string_lossy().into_owned()) - } - - fn typed_hook_entry(&self, value: &PyConfigNode) -> PyResult { - hook_entry(&config_object(self.environment.py(), value)?) - } -} - -/// Header keys whose values the build reads item by item, which are always converted. -const INSPECTED_KEYS: [&str; 7] = [ - "tags", - "row_diff_exclude_columns", - "row_diff_tolerances", - "pre_hooks", - "post_hooks", - "time_travel_retention", - "table_type", -]; - -/// Whether `key`'s container may be carried whole: no template or macro in it, items never read. -fn carried_whole(key: &str, facts: Option<&[(String, HeaderEntryFacts)]>) -> bool { - !INSPECTED_KEYS.contains(&key) - && facts - .and_then(|facts| facts.iter().find(|(name, _)| name == key)) - .is_some_and(|(_, facts)| !facts.template && facts.macro_path == MacroPath::Absent) -} - -/// A header's entries as config nodes; containers the build never looks into stay opaque. -fn header_entries( - header: &Bound<'_, PyDict>, - facts: Option<&[(String, HeaderEntryFacts)]>, - authored: Option<&[(String, AuthoredValue)]>, -) -> PyResult { - let authored: HashMap<&str, &AuthoredValue> = authored - .into_iter() - .flatten() - .map(|(key, value)| (key.as_str(), value)) - .collect(); - let mut entries: PyConfigEntries = Vec::with_capacity(header.len()); - for (key, value) in header.iter() { - let native: Option = match key.downcast_exact::() { - Ok(key) => match key.to_str() { - Ok(key) => authored.get(key).and_then(|value| config_value(value)), - Err(_) => None, - }, - Err(_) => None, - }; - if let Some(native) = native { - entries.push((config_node(&key)?, native)); - continue; - } - let whole: bool = is_exact_container(&value) - && key - .downcast_exact::() - .is_ok_and(|key| key.to_str().is_ok_and(|key| carried_whole(key, facts))); - let node: PyConfigNode = if whole { - carried_node(&value)? - } else { - config_node(&value)? - }; - entries.push((config_node(&key)?, node)); - } - Ok(entries) -} - -/// One header entry's parsed facts as the build reads them. -fn entry_facts((key, entry): &(String, EntryFacts)) -> (String, HeaderEntryFacts) { - let facts: HeaderEntryFacts = HeaderEntryFacts { - template: entry.template, - macro_path: entry.macro_path.clone(), - }; - (key.clone(), facts) -} - -fn optional_node(value: Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - config_node(&value).map(Some) -} - -#[pymethods] -impl NativeModelConfigBuilder { - #[new] - pub(crate) fn new( - layers: (Bound<'_, PyAny>, Bound<'_, PyDict>, Bound<'_, PyTuple>), - sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), - target: TargetInputs<'_>, - ) -> PyResult { - let (run, target_namespace) = target; - let (defaults, path_defaults, _hook_entry_types) = layers; - let mut path_keys: Vec = Vec::with_capacity(path_defaults.len()); - let mut path_values: Vec<(String, PyConfigEntries)> = - Vec::with_capacity(path_defaults.len()); - for (key, values) in path_defaults.iter() { - let key: String = key.extract()?; - path_values.push((key.clone(), config_entries(values.downcast::()?)?)); - path_keys.push(key); - } - let (target_name, run_id) = run; - Ok(Self { - inputs: ConfigBuildInputs { - defaults: config_entries(&project_defaults_mapping(&defaults)?)?, - path_defaults: path_values, - variables: config_entries(&sources.0)?, - run_id: config_node(&run_id)?, - target_name: optional_node(target_name)?.unwrap_or(ConfigNode::Null), - target_namespace: target_namespace - .map(|(database, schema)| -> PyResult<_> { - Ok((config_node(&database)?, config_node(&schema)?)) - }) - .transpose()?, - }, - path_keys, - environment: sources.1.unbind(), - }) - } - - /// Return the model's `CompileModelConfig` or the first config error. - fn model_config<'py>( - &self, - py: Python<'py>, - header: Bound<'py, PyDict>, - model: (Option, String), - classes: ConfigClasses<'py>, - ) -> PyResult> { - let (matched_path_default, model_name) = model; - let matched_path_default: Option<&str> = matched_path_default.as_deref(); - compiler_guard(|| { - match self.built(py, (&header, None, None), matched_path_default, &model_name)? { - ConfigBuild::Built(built) => { - Ok(built_config(&classes, &built, matched_path_default)?.unbind()) - } - ConfigBuild::Rejected(error) => Ok(error.into_any()), - } - }) - } -} - -/// The Python error a halt raises, or the config error it is rejected with. -fn halted<'py>( - py: Python<'py>, - halt: ConfigBuildHalt, -) -> PyResult> { - Ok(match halt { - ConfigBuildHalt::Config(error) => Ok(error), - ConfigBuildHalt::Host(error) => Err(error), - ConfigBuildHalt::MissingPathDefault(key) => Err(PyKeyError::new_err(key)), - ConfigBuildHalt::KeyNotText(key) => { - Err(match config_object(py, &key)?.downcast::() { - Ok(_) => PyRuntimeError::new_err(BUILD_STOPPED), - Err(error) => error.into(), - }) - } - }) -} - -/// The built values as a new `dict`; entries still the header's own keep the header's objects. -fn own_values_dict<'py>( - header: &Bound<'py, PyDict>, - built: &BuiltModelConfig, -) -> PyResult> { - let py: Python<'py> = header.py(); - let mapping: Bound<'py, PyDict> = PyDict::new(py); - for (key, value) in &built.values { - let own: bool = own_key(key, &built.own_header_keys); - let key: Bound<'py, PyAny> = config_object(py, key)?; - let value: Bound<'py, PyAny> = match own.then(|| header.get_item(&key)).transpose()? { - Some(Some(value)) => value, - _ => config_object(py, value)?, - }; - mapping.set_item(key, value)?; - } - Ok(mapping) -} - -/// Whether `key` is a string key named among the header's own keys. -fn own_key(key: &PyConfigNode, own: &[String]) -> bool { - let text: Option<&str> = match key { - ConfigNode::Str(text) => Some(text), - ConfigNode::Host(host) if host.facts.strict_text => host.facts.text.as_deref(), - _ => None, - }; - text.is_some_and(|text| own.iter().any(|name| name == text)) -} - -/// One model's build projected for Python. -fn projected_build<'py>( - header: &Bound<'py, PyDict>, - built: BuiltModelConfig, -) -> PyResult> { - let py: Python<'py> = header.py(); - Ok(BuiltConfig { - values: own_values_dict(header, &built)?, - nodes: built_values(built.values), - header_keys: built.header_keys, - logical_schema: projected(py, built.logical_schema)?, - layer_schema_configured: built.layer_schema_configured, - logical_database: projected(py, built.logical_database)?, - retention: built.retention, - table_type: built.table_type, - reads: built.reads, - }) -} - -fn projected<'py>( - py: Python<'py>, - value: Option, -) -> PyResult>> { - value.map(|value| config_object(py, &value)).transpose() -} - -impl NativeModelConfigBuilder { - fn host<'py>(&self, py: Python<'py>) -> PythonConfigHost<'py> { - PythonConfigHost { - environment: self.environment.bind(py).clone(), - } - } - - /// The selected path-default key, or the conflict error Python raises. - pub(crate) fn selected_path_default( - &self, - py: Python<'_>, - model_path: &str, - ) -> PyResult, Py>> { - Ok(match select_path_default(model_path, &self.path_keys) { - PathDefaultChoice::Selected(key) => Ok(key), - PathDefaultChoice::Conflict(error) => Err(native_config_error(py, error)?), - }) - } - - /// Build one model's config natively. - pub(crate) fn built<'py>( - &self, - py: Python<'py>, - (header, facts, authored): HeaderBuildInputs<'_, 'py>, - matched_path_default: Option<&str>, - model_name: &str, - ) -> PyResult> { - let facts: Option> = - facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); - let entries: PyConfigEntries = header_entries(header, facts.as_deref(), authored)?; - let model_header = ModelHeader { - entries: &entries, - facts: facts.as_deref(), - }; - let host: PythonConfigHost<'py> = self.host(py); - match build_model_config( - (&host, &self.inputs), - model_header, - matched_path_default, - model_name, - ) { - Ok(built) => Ok(ConfigBuild::Built(projected_build(header, built)?)), - Err(halt) => match halted(py, halt)? { - Ok(error) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), - Err(error) => Err(error), - }, - } - } - - /// `build_layered_model_values` with an empty header: what a path default's models inherit. - pub(crate) fn inherited_values<'py>( - &self, - py: Python<'py>, - matched_path_default: Option<&str>, - ) -> PyResult, Py>> { - let host: PythonConfigHost<'py> = self.host(py); - match inherited_config((&host, &self.inputs), matched_path_default) { - Ok(values) => Ok(Ok(config_dict(py, &values)?)), - Err(halt) => match halted(py, halt)? { - Ok(error) => Ok(Err(native_config_error(py, error)?)), - Err(error) => Err(error), - }, - } - } - - /// `build_model_context_values` for a model's config values. - pub(crate) fn model_context_values<'py>( - &self, - values: &Bound<'py, PyDict>, - model_name: &str, - include_target_values: bool, - ) -> PyResult> { - let mut read: PyConfigEntries = Vec::with_capacity(CONTEXT_KEYS.len()); - for key in CONTEXT_KEYS { - if let Some(value) = values.get_item(key)? { - read.push((ConfigNode::Str(key.to_owned()), config_node(&value)?)); - } - } - let context: PyConfigEntries = - model_context_values(&self.inputs, &read, model_name, include_target_values); - config_dict(values.py(), &context) - } - - /// The active target's `(database, schema)`, or `(None, None)` without a target. - pub(crate) fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { - match &self.inputs.target_namespace { - Some((database, schema)) => PyTuple::new( - py, - [config_object(py, database)?, config_object(py, schema)?], - ), - None => PyTuple::new(py, [py.None(), py.None()]), - } - } -} - -/// The Python exception a native config error raises as. -pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { - let raised = || -> PyResult { - let error: Py = native_config_error(py, error)?; - Ok(PyErr::from_value(config_exception( - error.bind(py).as_any(), - None, - )?)) - }; - raised().unwrap_or_else(|error| error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs deleted file mode 100644 index 6690095c67..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! The Python model validators, run natively up to the first error they raise. - -use sqlbuild_python_base::positional_record; -use std::collections::HashSet; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{IntoPyObject, pyclass, pymethods}; -use sqlbuild_attachments::functions::models::KnownResources; -use sqlbuild_model_config::model_validation::main::validate_model_config::validate_model_config; -use sqlbuild_model_config::model_validation::main::validate_model_references::validate_model_references; -use sqlbuild_model_config::model_validation::models::{ - ModelReference, ModelValidationFacts, ProjectValidationFacts, ValidationStop, -}; - -use crate::bindings::_helpers::discovery::project_facts::{column_name, each, stem}; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; - -const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const RETURNS_KEY: &str = "returns"; -/// The only key of a table function's `returns` mapping. -const TABLE_RETURNS_KEY: &str = "table"; - -positional_record! { - extract; - /// Model, seed, source, function and table function names. - struct ResourceNames { - models: HashSet, - seeds: HashSet, - sources: HashSet, - functions: HashSet, - table_functions: HashSet, - } -} - -positional_record! { - extract; - /// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. - struct ModelFacts { - references: Vec<(String, String, bool)>, - declared_columns: Option>, - retention_unmanaged: bool, - table_type_declared: bool, - } -} - -/// The project facts every model's validation reads, captured once per compile. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeModelValidator { - project: ProjectValidationFacts, -} - -impl NativeModelValidator { - /// The project's discovered resource names, which every reference resolves against. - pub(crate) fn known_resources(&self) -> KnownResources<'_> { - KnownResources { - models: &self.project.models, - seeds: &self.project.seeds, - sources: &self.project.sources, - functions: &self.project.functions, - table_functions: &self.project.table_functions, - } - } -} - -/// Model, seed, source, function and table function names discovered in `inputs`. -fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { - let model_stem = |file: &Bound<'_, PyAny>| stem(file, FILE_PATH_ATTRIBUTE); - let mut models: HashSet = each(inputs, "model_files", model_stem)? - .into_iter() - .collect(); - if let Some(injection) = - sqlbuild_python_base::compile_context::main::delta::injection::injection() - { - models.extend(injection.model_names.iter().cloned()); - } - let mut seeds: HashSet = HashSet::new(); - for schema in native_field(inputs, "schema_files")?.try_iter()? { - seeds.extend(each(&schema?, "seed_entries", column_name)?); - } - let mut sources: HashSet = HashSet::new(); - for file in native_field(inputs, "source_files")?.try_iter()? { - sources.extend(each(&file?, "source_entries", column_name)?); - } - let mut functions: HashSet = HashSet::new(); - let mut table_functions: HashSet = HashSet::new(); - for file in native_field(inputs, "sql_function_files")?.try_iter()? { - let file: Bound<'_, PyAny> = file?; - let name: String = model_stem(&file)?; - if returns_table(&file)? { - table_functions.insert(name.clone()); - } - functions.insert(name); - } - functions.extend(each(inputs, "python_function_files", model_stem)?); - Ok(ResourceNames { - models, - seeds, - sources, - functions, - table_functions, - }) -} - -/// Whether a SQL function's header `returns` is a mapping whose only key is `table`. -fn returns_table(file: &Bound<'_, PyAny>) -> PyResult { - let headers: Bound<'_, PyAny> = native_field(file, "header_values")?; - let returns: Bound<'_, PyAny> = mapping_get(&headers, RETURNS_KEY)?; - let Ok(returns) = returns.downcast::() else { - return Ok(false); - }; - let keys: Vec> = returns.keys().into_iter().collect(); - Ok(keys.len() == 1 && keys[0].eq(TABLE_RETURNS_KEY)?) -} - -#[pymethods] -impl NativeModelValidator { - /// The validator for the resources and materializations `inputs` discovered. - #[staticmethod] - pub(crate) fn from_inputs( - inputs: Bound<'_, PyAny>, - microbatch_concurrency: bool, - ) -> PyResult { - let names: ResourceNames = discovered_names(&inputs)?; - let custom_materializations: HashSet = - each(&inputs, "materialization_files", |file| { - native_field(file, "name")?.extract() - })? - .into_iter() - .collect(); - Self::new(names, custom_materializations, microbatch_concurrency) - } - - /// The discovered model names references resolve against. - fn model_names(&self) -> HashSet { - self.project.models.clone() - } - - #[new] - fn new( - names: ResourceNames, - custom_materializations: HashSet, - microbatch_concurrency: bool, - ) -> PyResult { - let ResourceNames { - models, - seeds, - sources, - functions, - table_functions, - } = names; - Ok(Self { - project: ProjectValidationFacts { - custom_materializations, - microbatch_concurrency, - models, - seeds, - sources, - functions, - table_functions, - }, - }) - } - - /// `None` when accepted, the first error, or the first externally rejected reference. - fn validate( - &self, - py: Python<'_>, - values: Bound<'_, PyDict>, - model: (String, String, String), - facts: ModelFacts, - ) -> PyResult> { - compiler_guard(|| { - let (model_name, query_sql, relative_path) = model; - let ModelFacts { - references, - declared_columns, - retention_unmanaged, - table_type_declared, - } = facts; - let references: Vec = references - .into_iter() - .map(|(kind, name, externally_rejected)| ModelReference { - kind, - name, - externally_rejected, - }) - .collect(); - let entries: Vec<(PyNode<'_>, PyNode<'_>)> = values - .iter() - .map(|(key, value)| (PyNode(key), PyNode(value))) - .collect(); - let facts = ModelValidationFacts { - model_name: &model_name, - relative_path: &relative_path, - references: &references, - declared_columns: declared_columns.as_deref(), - query_sql: &query_sql, - retention_unmanaged, - table_type_declared, - }; - match validate_model_config(entries, &self.project, &facts) { - Ok(()) => Ok(py.None()), - Err(stop) => stop_object(py, stop), - } - }) - } - - /// Check references alone: `None`, the first error, or a rejected dbt reference's index. - fn references( - &self, - py: Python<'_>, - model: (String, String), - references: Vec<(String, String, bool)>, - ) -> PyResult> { - compiler_guard(|| { - let (model_name, relative_path) = model; - let references: Vec = references - .into_iter() - .map(|(kind, name, externally_rejected)| ModelReference { - kind, - name, - externally_rejected, - }) - .collect(); - let facts = ModelValidationFacts { - model_name: &model_name, - relative_path: &relative_path, - references: &references, - declared_columns: None, - query_sql: "", - retention_unmanaged: false, - table_type_declared: false, - }; - match validate_model_references(&self.project, &facts) { - Ok(()) => Ok(py.None()), - Err(stop) => stop_object(py, stop), - } - }) - } -} - -fn stop_object(py: Python<'_>, stop: ValidationStop) -> PyResult> { - match stop.into_error() { - Err(index) => Ok(index.into_pyobject(py)?.into_any().unbind()), - Ok(error) => Ok(native_config_error(py, error)?.into_any()), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 4e9095b798..c744b3e4f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -2,18 +2,21 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::model_config::{ - config_errors, config_values, header_columns, model_config_builder, model_validator, - template_data, unicode_environment, -}; +use sqlbuild_python_front::model_config::main::config::register_config_errors::register_config_errors; +use sqlbuild_python_front::model_config::main::config::register_config_values::register_config_values; +use sqlbuild_python_front::model_config::main::config::register_template_data::register_template_data; +use sqlbuild_python_front::model_config::main::headers::register_header_columns::register_header_columns; +use sqlbuild_python_front::model_config::main::model_nodes::register_model_config_builder::register_model_config_builder; +use sqlbuild_python_front::model_config::main::model_nodes::register_model_validator::register_model_validator; +use sqlbuild_python_front::model_config::main::model_nodes::register_unicode_environment::register_unicode_environment; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - config_errors::register(module)?; - config_values::register(module)?; - header_columns::register(module)?; - model_config_builder::register(module)?; - model_validator::register(module)?; - template_data::register(module)?; - unicode_environment::register(module)?; + register_config_errors(module)?; + register_config_values(module)?; + register_header_columns(module)?; + register_model_config_builder(module)?; + register_model_validator(module)?; + register_template_data(module)?; + register_unicode_environment(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 794ea5865f..a3c45ce10a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,9 +1,6 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. -pub(crate) mod audit_results_state; -pub(crate) mod consumption_state; pub(crate) mod model_lookup; -pub(crate) mod observations_state; pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index c29ee9cf82..6216853b0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -12,10 +12,6 @@ use sqlbuild_orchestration::sensors::models::{ }; use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; -use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; -use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -23,6 +19,11 @@ use sqlbuild_python_base::native_host::constants::SENSOR_CONTEXT_PARAMETER; use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; +use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; +use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; +use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_front::discovery::models::SecretPolicy; pub(crate) fn evaluate_sensor( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index ef8daac20d..5c789c7eaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -17,9 +17,6 @@ use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; -use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -32,6 +29,10 @@ use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callbac use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; +use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; +use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; +use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; +use sqlbuild_python_front::discovery::models::SecretPolicy; /// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. pub(crate) struct CalendarRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs index d095503ec2..1b6af182fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs @@ -18,13 +18,17 @@ use sqlbuild_state::resource_costs::models::{ResourceCostKey, RunCostAttempt}; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use crate::bindings::_helpers::execution_history::conversion::{parsed_iso, utc}; -use crate::bindings::_helpers::native_host::consumption_state::{ - Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, integer_literal, literal, - optional_text, state_table_name, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::constants::ROWS_PER_STATEMENT; +use sqlbuild_python_front::native_host::main::execute::execute; +use sqlbuild_python_front::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use sqlbuild_python_front::native_host::main::integer_literal::integer_literal; +use sqlbuild_python_front::native_host::main::literal::literal; +use sqlbuild_python_front::native_host::main::optional_text::optional_text; +use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; +use sqlbuild_python_front::native_host::models::Location; /// Rate source of a price the project configured, the only one shown as USD. const CONFIGURED_RATE_SOURCE: &str = "configured"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 6ef103c2b3..6aebbf4d28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -33,7 +33,6 @@ use sqlbuild_state::state_tables::main::event_id_index::event_id_index; use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; -use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; @@ -50,6 +49,8 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; +use sqlbuild_python_front::authored_values::main::sql_value::sql_value; +use sqlbuild_python_front::authored_values::models::SqlValueClasses; /// One read-only adapter session serves every page of one synchronization. pub(crate) struct HistorySession<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index cbb914b201..069c096f37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -10,10 +10,6 @@ use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KI use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; use crate::bindings::_helpers::consumption::relation_reads::relation; -use crate::bindings::_helpers::native_host::consumption_state::{ConsumptionConnection, Location}; -use crate::bindings::_helpers::native_host::observations_state::{ - ObservationAppend, append_observations, -}; use crate::bindings::_helpers::observations::source_capture::take_source_capture; use crate::bindings::_helpers::planner_phases::objects::config_str; use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; @@ -21,6 +17,10 @@ use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::main::append_observations::append_observations; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::native_host::models::ObservationAppend; /// Dependency kind of a source in a model's `dep_rows`. const SOURCE_DEPENDENCY: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs index 7eafd8aa8d..758efed569 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs @@ -13,9 +13,9 @@ use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::native_host::main::host_error::host_error; -use crate::bindings::_helpers::macro_bridge::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_front::macro_bridge::main::text::text; const SURROGATE_PASS: &str = "surrogatepass"; const UTF8: &str = "utf-8"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 88b9602038..c60b3afdf9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspe use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; @@ -37,6 +36,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index 389a15a7f6..16aae0f96a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -11,7 +11,6 @@ use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::rules::catalogue::build_rule_catalogue; @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const LIST_ACTION: &str = "list"; const SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs index 8906c6ff58..bef1e1810a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -3,7 +3,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; use crate::bindings::_helpers::plan_output::display::{ aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, }; @@ -12,6 +11,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::constants::GRAIN_NAMES; const MICROBATCH_MODE: &str = "microbatch"; const TIMESTAMP_CURSOR: &str = "timestamp"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 6dc7b553fd..4cd6d088fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -16,9 +16,6 @@ use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_wa use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; -use crate::bindings::_helpers::cursor::scalars::{ - render_value, to_exclusive, to_inclusive, try_parse, -}; use crate::bindings::_helpers::plan_output::values::{optional_text, text}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -26,6 +23,10 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::main::scalars::to_exclusive::to_exclusive; +use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index e1894604bc..983a0772dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; -use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, cursor_details, owner, }; @@ -20,6 +19,7 @@ use crate::bindings::_helpers::plan_output::values::{ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; /// The plan JSON key of what `--changed` would run, hold back and skip. pub(crate) const CHANGED_KEY: &str = "changed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 9eace562d7..d950e7e477 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -12,7 +12,6 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, }; @@ -31,6 +30,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs index 0955e0d23e..62a673e840 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs @@ -4,9 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::plan_output::values::{dict, enum_value, items}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; /// `render(value=value)`. fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 0e6f46fa55..149d4844fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -6,13 +6,14 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; -use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_analysis::planner_changes::constants::{ CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 8277a9adf2..3ecf29754a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, @@ -40,6 +39,8 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const ENABLED_KEY: &str = "enabled"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index dbaa7ce819..bed57d5494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -4,7 +4,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; @@ -18,6 +17,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 601dd8040a..908a16a460 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_entries::models::{ @@ -51,6 +50,7 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::microbatch_guidance; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 60e731100f..9508c1d437 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -19,9 +19,7 @@ use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_iden use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::microbatches::conversion::timestamp; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_identity::hashing::{ baselined_function_definition_hash, function_definition_hash, }; @@ -34,6 +32,9 @@ use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 4ecc9ae781..6e6385e0df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -12,7 +12,6 @@ use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::state_rows::migration_projections::newest_column; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_v use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::state_rows::main::migrations::newest_column::newest_column; pub(super) fn decide<'py>( runtime: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index a2d6d2b308..5c552ee897 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,7 +1,6 @@ //! Boundary conversions for the column migration planner. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -10,6 +9,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index dd83f32cf8..ef732eccbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::model_reads::inferred_rows; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_migrations::models::{ CompatibilityCheck, MigrationCompatibilityResult, @@ -19,6 +18,8 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { config_value(model.py(), &model_values(model)?, key)?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index e156cd1cc0..1b93b679da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -5,12 +5,12 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; -use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index ac0cf2e9d8..ccafab948a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::planner_migrations::model_handover::{ use crate::bindings::_helpers::planner_migrations::models::{ CompatibilityCheck, MigrationDecisionRequest, }; -use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 8aee69e3e9..fdaea4ab1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -5,11 +5,11 @@ use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::state_rows::main::event_rows::presumed_utc::presumed_utc; pub(super) fn applies( model: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index b736ba60e6..12d630e1e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,9 +2,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::model_config::typed_config::{ - config_node_of, config_value, model_values, -}; use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; @@ -29,6 +26,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const MIGRATE_FROM_KEY: &str = "migrate_from"; const MIGRATE_FORCE_KEY: &str = "migrate_force"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index cf7cc1c969..13face79d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -4,7 +4,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; -use crate::bindings::_helpers::state_rows::migration_projections::old_name_retention; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -12,6 +11,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index a4e9bdcca7..f47da108cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -18,11 +18,6 @@ use crate::bindings::_helpers::planner_migrations::constants::{ GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; -use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; -use crate::bindings::_helpers::state_rows::migration_objects::layout; -use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; -use crate::bindings::_helpers::state_rows::models::StateAccess; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -30,6 +25,14 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_front::state_rows::constants::COLUMN_KIND; +use sqlbuild_python_front::state_rows::constants::MODEL_KIND; +use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; +use sqlbuild_python_front::state_rows::main::migrations::layout::layout; +use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; +use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; +use sqlbuild_python_front::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; +use sqlbuild_python_front::state_rows::models::StateAccess; /// A lowercased schema and relation name. type RelationKey = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs index 6b0802146c..9abc53ca3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs @@ -4,11 +4,12 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const MIGRATE_FROM_KEY: &str = "migrate_from"; /// `database.schema.name`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index bb31c108c0..6cef425326 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,6 +1,5 @@ //! Write-hash reconciliation and plan entry construction. -use crate::bindings::_helpers::command_runtime::concurrency::resolve_configured_concurrency; use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; use crate::bindings::_helpers::planner_entries::models::{ @@ -18,6 +17,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; /// The execution scope's changes with honest write hashes, and the actions merged from them. pub(crate) fn reconcile_execution_changes<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index 545e4ca0b1..cacbbee94f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,6 +10,8 @@ use pyo3::types::PyBool; use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const FULL_REFRESH_KEY: &str = "full_refresh"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 58f6a1e316..de5c5ebcdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -4,7 +4,6 @@ use sqlbuild_host::entries::models::HostId; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; use pyo3::{BoundObject, IntoPyObject}; @@ -18,6 +17,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// A Python `str`. pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index b405173d37..3cf8f14a69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -4,8 +4,6 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::model_config::typed_config::{config_value, typed_values}; use crate::bindings::_helpers::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, @@ -30,6 +28,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index e31243a0dd..4c686ff2ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -2,8 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; -use crate::bindings::_helpers::cursor::scalars::try_parse; use crate::bindings::_helpers::planner_resolution::maximum_safety::apply_maximum_start_policy; use crate::bindings::_helpers::planner_resolution::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -22,6 +20,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::conversion::algebra_error::algebra_error; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; +use sqlbuild_python_front::cursor::models::CursorSession; pub(super) fn optional_scalar<'py>( session: &CursorSession<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 35b509e3ab..30246d89b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -4,9 +4,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::typed_config::{ - config_cursor_bound, config_text_object, model_values, -}; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; @@ -34,6 +31,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py: Python<'py> = model.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 017ae15c63..428d39f17b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -1,9 +1,6 @@ //! Selected-model inspection and untyped override conversion. use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::model_config::typed_config::{ - config_text_is, config_text_object, model_values, -}; use crate::bindings::_helpers::planner_resolution::values::{method, string}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -18,6 +15,9 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs index 04d13b095b..50a8bcd9a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -1,6 +1,5 @@ //! Model-level cursor safety policy overrides over project configuration. -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -8,6 +7,8 @@ use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_v use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const POLICY_DISABLED: &str = "disabled"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index 4df5d150db..ae4eb4b540 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -4,12 +4,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::{parse_value, sentinel_to_token}; -use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; -use crate::bindings::_helpers::model_config::typed_config::{ - config_cursor_bound as typed_cursor_bound, config_text_object, model_values, -}; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; @@ -44,6 +38,14 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_time; +use sqlbuild_python_front::cursor::main::values::replaced::replaced; +use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::cmp::Ordering; const INCREMENTAL: &str = "incremental"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs index f44d0979aa..052858c4ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -1,6 +1,5 @@ //! Shared safety horizon conversion and Python evidence construction. -use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; @@ -15,6 +14,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::models::CursorSession; pub(super) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { new_plain_error(py, class, message) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs index 8029693977..599890d6d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs @@ -1,6 +1,5 @@ //! Prominent warnings for applied future-end and automatic-start cursor safety. -use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::planner_resolution::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyTuple}; @@ -9,6 +8,7 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index 04b2818ecf..46dcb20ea1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -1,7 +1,5 @@ //! Empty-window decisions and snapshot grain conversion. -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::parse_value; use crate::bindings::_helpers::planner_resolution::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -16,6 +14,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; +use sqlbuild_python_front::cursor::models::CursorSession; #[pyfunction] #[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs index 1c42350546..f2dbdb5769 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -1,6 +1,5 @@ //! Future cursor safety evidence and exception boundary. -use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::planner_resolution::cursor_safety::{ horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::models::CursorSession; use std::cmp::Ordering; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 9a84614229..1e4e4779fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -1,7 +1,5 @@ //! Maximum automatic-start recovery and evidence boundary. -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::models::CursorSession; use std::cmp::Ordering; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 9d439d3bb5..30858a454f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,14 +1,8 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_intrinsics; -use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; -use crate::bindings::_helpers::model_config::typed_config::{ - config_node_of, config_text_is, config_text_object, config_value, model_values, -}; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::{ @@ -33,6 +27,14 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; +use sqlbuild_python_front::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const APPEND_STRATEGY: &str = "append"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index 0bcc7bfa7c..8429bcf5ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use crate::bindings::_helpers::planner_resolution::values::{ cursor_wrap, filtering_of, method, string, }; -use crate::bindings::_helpers::sqltext::code_scanner::SqlCodeScanner; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -16,6 +15,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::sqltext::models::SqlCodeScanner; use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans::reference_spans}; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs index 8baddcc5c4..d2d5e16d63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs @@ -1,6 +1,5 @@ //! Boundary calls shared by reference and source resolution. -use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; @@ -8,6 +7,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; use sqlbuild_sqltext::resolution::models::CursorFilter; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 409da03d00..2e69862e57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -4,7 +4,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; @@ -44,6 +43,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; #[pyfunction] #[pyo3(signature = (**kwargs))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs index 945a0bbe8b..669fa0088e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs @@ -7,7 +7,6 @@ use crate::bindings::_helpers::planner_scenario::{ sql::{required_target, resolve_sql, source_label}, values::{error, string}, }; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_reader_ctes::scenario_reader_ctes; @@ -19,6 +18,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -28,7 +28,7 @@ pub(super) fn extract_fixtures<'py>( scenario: &Bound<'py, PyAny>, prefix: &str, ) -> PyResult> { - crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes( + sqlbuild_python_front::sql_test_glue::main::prefixed_ctes::prefixed_ctes( &native_field(scenario, "authored_ctes")?, prefix, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs index c11b51ea0c..3b66e666b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs @@ -6,9 +6,9 @@ use sqlbuild_analysis::compiler::models::{ScenarioHelperScope, ScenarioScopeInpu use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; use crate::bindings::_helpers::planner_scenario::{models::SqlContext, values::error}; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const FIXTURE_LOCATIONS: [(&str, &str); 4] = [ ("__source__", "source_fixture_locations"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index 79ed69af3f..b10b6a00fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::planner_scenario::{ values::{error, string}, }; use crate::bindings::_helpers::references::source_relations::render_source_relation; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; use sqlbuild_analysis::planner_scenario::{ @@ -18,6 +17,7 @@ use sqlbuild_analysis::planner_scenario::{ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(super) fn resolve_sql( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs index 5701b57bd2..4d9bde263f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -1,6 +1,5 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::bindings::_helpers::contracts::spec_resolution::loader_parts; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_scope::values::{ @@ -15,6 +14,7 @@ use sqlbuild_analysis::planner_scope::main::{ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::loader_parts::loader_parts; use std::collections::HashSet; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs index 9aafa335d4..136cac00ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -13,12 +13,13 @@ use sqlbuild_analysis::planner_selection::models::{ }; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_selection::carrier::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const TAGS_KEY: &str = "tags"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index 63317f06f4..7a8c3a69ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -13,12 +13,12 @@ use sqlbuild_analysis::planner_selection::models::{ }; use crate::bindings::_helpers::compiled_project::job_catalog::job_names; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::models::NativeModelFile; /// A bare selector's kind. const NAME_SELECTOR_KIND: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 7267639344..3e331b8bbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -5,7 +5,6 @@ use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; use std::collections::HashSet; -use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::auditing::render::render_audit_sql; use crate::bindings::_helpers::compiled_views::audit_reads; use crate::bindings::_helpers::compiled_views::project_reads; @@ -23,6 +22,7 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 74dcd375f8..457d7b7b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -9,10 +9,6 @@ use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, Cursor use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::cursor::scalars::parse_value; -use crate::bindings::_helpers::model_config::typed_config::{ - config_cursor_bound, config_text_object, config_value, model_values, -}; use crate::bindings::_helpers::planner_identity::metadata::or_empty; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_strategy::constants::{ @@ -31,6 +27,11 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; +use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// Whether a model is incremental with a cursor column. fn incremental_cursor_flag( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index 0274eb77fe..2d15fabc57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -6,12 +6,13 @@ use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const CONTRACT_KEY: &str = "contract"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 4b782d2bcc..30baee3ec5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -12,7 +12,6 @@ use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_text_is, model_values}; use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, @@ -21,6 +20,8 @@ use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; const MICROBATCH_MODE: &str = "microbatch"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs index 3072dad55b..239ab716a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -18,7 +18,6 @@ use sqlbuild_analysis::planner_strategy::types::Timeline; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; -use crate::bindings::_helpers::cursor::durations::shifted_moment; use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; use crate::bindings::_helpers::planner_strategy::values::text; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -27,6 +26,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::durations::shifted_moment::shifted_moment; const INTEGER_CURSOR: &str = "integer"; const TIMESTAMP_CURSOR: &str = "timestamp"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 07244e1a41..20efc7a4c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -18,11 +18,6 @@ use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; -use crate::bindings::_helpers::model_config::typed_config::{ - ConfigValues, config_cursor_bound, config_flag, config_integer_object, config_text_object, - config_text_tuple, config_value, model_values, -}; use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, TABLE_RECREATING_ACTIONS, @@ -48,6 +43,15 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; +use sqlbuild_python_front::model_config::main::config::config_flag::config_flag; +use sqlbuild_python_front::model_config::main::config::config_integer_object::config_integer_object; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::config_text_tuple::config_text_tuple; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_front::model_config::types::ConfigValues; /// Model config keys copied to the plan entry as optional strings, with their entry fields. const STRING_FIELDS: [(&str, &str); 15] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 1bfcc0eed0..22e3989765 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -1,13 +1,12 @@ //! Conversions between the planner's Python models and the native strategy rows. -use crate::bindings::_helpers::model_config::typed_config::{ - config_value as typed_config_value, model_values, -}; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_front::model_config::main::config::config_value::config_value as typed_config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_analysis::planner_changes::models::{ SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 0f49154e68..c51fb8fe18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -5,12 +5,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::cursor::intervals::partition; -use crate::bindings::_helpers::cursor::scalars; -use crate::bindings::_helpers::cursor::scalars::compare_values; -use crate::bindings::_helpers::cursor::scalars::render_value; -use crate::bindings::_helpers::cursor::scalars::to_inclusive; -use crate::bindings::_helpers::model_config::typed_config::{config_cursor_bound, model_values}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, @@ -49,6 +43,13 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_front::cursor::main::intervals::partition::partition; +use sqlbuild_python_front::cursor::main::scalars::compare_values::compare_values; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; +use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -126,7 +127,7 @@ fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &str, ) -> PyResult> { - scalars::parse(raw, &text(py, cursor_type)) + sqlbuild_python_front::cursor::main::scalars::parse::parse(raw, &text(py, cursor_type)) } fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { @@ -337,7 +338,7 @@ fn floor_lifts(info: &CursorModel<'_>, observed: &Bound<'_, PyAny>) -> PyResult< let (Some(floor), Some(cursor_type)) = (&info.floor, &info.cursor_type) else { return Ok(false); }; - match scalars::try_parse(&text(py, floor), &text(py, cursor_type))? { + match try_parse(&text(py, floor), &text(py, cursor_type))? { Some(floor) => Ok(compare_values(&floor, observed)? > 0), None => Ok(false), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index f71982ea05..68d93023e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; @@ -16,6 +15,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::collections::BTreeSet; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 817166d5f3..02cf6f7df8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -8,7 +8,6 @@ use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::object; @@ -42,6 +41,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index 9325a8c3da..b93aaa112b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -2,14 +2,11 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compiled_views::audit_reads; use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ optional_text, planner_error, replace, text, @@ -26,6 +23,10 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::authored_values::main::render_project_var_text::render_project_var_text; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 49f022336a..96362f5a61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -27,6 +26,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_front::discovery::main::project::yaml_entry_line::yaml_entry_line; use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index 6164c5eca1..b1adf98e3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; @@ -14,6 +13,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; use std::collections::HashMap; const CONTRACT_KEY: &str = "contract"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index a920f64c24..e72461f44f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -17,9 +17,7 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_model_config::templates::models::Scalar; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; -use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, }; @@ -34,6 +32,8 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::authored_values::main::project_var_text::project_var_text; +use sqlbuild_python_front::model_config::main::config::record_reads::record_reads; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index c41ed31e7d..64cb409ce9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -6,10 +6,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index b137473697..95592855ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -3,13 +3,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; -use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::project::discover_connection_configuration::discover_connection_configuration; +use sqlbuild_python_front::discovery::models::SecretPolicy; /// The project directory, defaulting to the working directory, with its inputs and adapter. pub(crate) struct ProjectAdapter<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs index b1ef52d28f..f7960e287c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs @@ -8,13 +8,13 @@ use pyo3::types::{ use serde_json::{Map, Number, Value}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::discovery::snapshot::escaped; use crate::bindings::_helpers::recall::constants::{ EVENT_OPTIONAL_TEXT_FIELDS, EVENT_TEXT_FIELDS, INVOCATION_METADATA_SCHEMA_VERSION, TIMESTAMP_PARTS, UTC_ZONE_NAME, }; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::snapshots::escaped::escaped; /// The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. pub(crate) fn escaped_text(text: &Bound<'_, PyString>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs index b703553e5f..322bb6c179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs @@ -2,9 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::discovery::imported_providers::install_recall_writer; use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; use crate::bindings::_helpers::recall::writer::RecallWriter; +use sqlbuild_python_front::discovery::main::python_files::install_recall_writer::install_recall_writer; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 3dc6eb9540..ea894c5e18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -74,9 +74,8 @@ use crate::bindings::_helpers::record_behaviours::values::{ }; use crate::bindings::_helpers::{ analysis, analysis_session, build_progress, compile_command, compiled_views, compute_logs, - dbt_events, dbt_interop, discovery, event_exporting, execution_history, graph, model_config, - planner_entries, planner_migrations, planner_phases, project_assembly, result_output, rules, - scopes, + dbt_events, dbt_interop, event_exporting, execution_history, graph, planner_entries, + planner_migrations, planner_phases, project_assembly, result_output, rules, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; @@ -365,11 +364,11 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ dbt_events::getters::GETTERS, dbt_interop::getters::GETTERS, sqlbuild_python_base::diagnostics::constants::GETTERS, - discovery::getters::GETTERS, + sqlbuild_python_front::discovery::constants::GETTERS, event_exporting::getters::GETTERS, execution_history::getters::GETTERS, graph::getters::GETTERS, - model_config::getters::GETTERS, + sqlbuild_python_front::model_config::constants::GETTERS, planner_entries::getters::GETTERS, planner_migrations::getters::GETTERS, planner_phases::getters::GETTERS, @@ -377,7 +376,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ project_assembly::getters::GETTERS, result_output::getters::GETTERS, rules::getters::GETTERS, - scopes::getters::GETTERS, + sqlbuild_python_front::scopes::constants::GETTERS, ]; /// Every record behaviour, installed into `records::behaviours` at registration. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs index a2ea181cf2..a3297ad4d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs @@ -4,13 +4,13 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cursor::scalars::parse_value; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; /// One cursor value conversion. type Conversion = for<'py> fn(&Bound<'py, PyAny>) -> PyResult>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs index 3dd2e61847..02a11c2eed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs @@ -4,11 +4,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::cursor::durations::{ - parse_cursor_duration, shift_by_cursor_duration, -}; -use crate::bindings::_helpers::cursor::intervals::interval_validation; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::main::durations::parse_cursor_duration::parse_cursor_duration; +use sqlbuild_python_front::cursor::main::durations::shift_by_cursor_duration::shift_by_cursor_duration; +use sqlbuild_python_front::cursor::main::intervals::interval_validation::interval_validation; /// `(field, unit suffix)` of each duration part, in declaration order. const DURATION_UNITS: [(&str, &str); 6] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs index 31bce33d1d..b485186ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs @@ -10,9 +10,10 @@ use sqlbuild_sqltext::sql_scan::main::reads_differently_from_generic::reads_diff use sqlbuild_sqltext::sql_scan::main::unsupported_line_comment_prefixes::unsupported_line_comment_prefixes; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::sqltext::lexical_syntax::{record_syntax, sorted_texts}; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::sqltext::main::record_syntax::record_syntax; +use sqlbuild_python_front::sqltext::main::sorted_texts::sorted_texts; /// `SqlLexicalSyntax.line_comment_prefixes` position. const LINE_COMMENT_PREFIXES_FIELD: usize = 5; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index 7c55bc795e..f93c691b4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -6,11 +6,10 @@ use pyo3::types::{PyDict, PyList, PyTuple, PyTupleMethods}; use crate::bindings::_helpers::graph::project_graph::{ResourceRow, project_graph}; use crate::bindings::_helpers::record_behaviours::compiled::compile_input_error; -use crate::bindings::_helpers::render::declaration_values::{ - constant_declaration_sql, render_enum_member_value, -}; use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::render::main::declarations::constant_declaration_sql::constant_declaration_sql; +use sqlbuild_python_front::render::main::declarations::render_enum_member_value::render_enum_member_value; /// The parameter name through which a macro receives its context. const MACRO_CONTEXT_PARAMETER_NAME: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs index a31f6f8907..86338b5ad5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs @@ -10,7 +10,6 @@ use sqlbuild_core::migration_values::main::model_migration_blocks_build::model_m use sqlbuild_core::migration_values::main::storage_transition::storage_transition; use sqlbuild_core::migration_values::models::{DecisionTraits, ModelMigrationFacts}; -use crate::bindings::_helpers::cursor::scalars::parse_value; use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; const DECREASING_DIRECTIONS: [&str; 2] = ["decrease", "mixed"]; const STEP_ACTIONS: [&str; 2] = ["archive_and_view", "view_only"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs index 218f3a4910..f0e246c0e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -1,7 +1,3 @@ -pub(crate) mod contract_compare; -pub(crate) mod contract_edits; pub(crate) mod offline_compile; -pub(crate) mod python_strings; pub(crate) mod refactor_command; -pub(crate) mod refactor_facts; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 0fa0c28a5e..f0c038c686 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -38,9 +38,6 @@ use sqlbuild_refactor::refactoring::types::Originals; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; use crate::bindings::_helpers::refactor::offline_compile::{OfflineCompile, compile_for_refactor}; -use crate::bindings::_helpers::refactor::refactor_facts::{ - declaration_placement, model_facts, refactor_facts, unit_test_rebindings, -}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; @@ -49,6 +46,10 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::refactor::main::declaration_placement::declaration_placement; +use sqlbuild_python_front::refactor::main::model_facts::model_facts; +use sqlbuild_python_front::refactor::main::refactor_facts::refactor_facts; +use sqlbuild_python_front::refactor::main::unit_test_rebindings::unit_test_rebindings; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs index 95663e5197..2d8ce156fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs @@ -2,13 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::refactor::{ - contract_compare, contract_edits, python_strings, refactor_command, -}; +use crate::bindings::_helpers::refactor::refactor_command; +use sqlbuild_python_front::refactor::main::register_contract_compare::register_contract_compare; +use sqlbuild_python_front::refactor::main::register_contract_edits::register_contract_edits; +use sqlbuild_python_front::refactor::main::register_python_strings::register_python_strings; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor_command::register(module)?; - contract_edits::register(module)?; - contract_compare::register(module)?; - python_strings::register(module) + register_contract_edits(module)?; + register_contract_compare(module)?; + register_python_strings(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 169a7f9d43..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,20 +1 @@ -pub(crate) mod authored_sql; -pub(crate) mod declaration_expansion; -pub(crate) mod declaration_indexes; -pub(crate) mod declaration_values; -pub(crate) mod macro_arguments; -pub(crate) mod macro_call_classes; -pub(crate) mod macro_call_memo; -pub(crate) mod macro_call_rows; -pub(crate) mod macro_calls; -pub(crate) mod macro_evaluation; -pub(crate) mod macro_expansion; -pub(crate) mod macro_expansion_facts; -pub(crate) mod macro_expansion_result; -pub(crate) mod macro_generated_references; -pub(crate) mod macro_modules; pub(crate) mod registration; -pub(crate) mod sql_macro_host; -pub(crate) mod sql_macro_inventory; -pub(crate) mod sql_macro_mocks; -pub(crate) mod sql_macro_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index 4362a03894..6820ff458a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -2,20 +2,25 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::render::{ - authored_sql, declaration_indexes, declaration_values, macro_arguments, macro_calls, - macro_expansion, macro_expansion_result, macro_modules, sql_macro_inventory, -}; +use sqlbuild_python_front::render::main::calls::register_macro_arguments::register_macro_arguments; +use sqlbuild_python_front::render::main::calls::register_macro_calls::register_macro_calls; +use sqlbuild_python_front::render::main::declarations::register_authored_sql::register_authored_sql; +use sqlbuild_python_front::render::main::declarations::register_declaration_indexes::register_declaration_indexes; +use sqlbuild_python_front::render::main::declarations::register_declaration_values::register_declaration_values; +use sqlbuild_python_front::render::main::expansion::register_macro_expansion::register_macro_expansion; +use sqlbuild_python_front::render::main::expansion::register_macro_expansion_result::register_macro_expansion_result; +use sqlbuild_python_front::render::main::expansion::register_macro_modules::register_macro_modules; +use sqlbuild_python_front::render::main::sql_macros::register_sql_macro_inventory::register_sql_macro_inventory; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - authored_sql::register(module)?; - declaration_values::register(module)?; - declaration_indexes::register(module)?; - macro_arguments::register(module)?; - macro_calls::register(module)?; - macro_expansion::register(module)?; - macro_expansion_result::register(module)?; - macro_modules::register(module)?; - sql_macro_inventory::register(module)?; + register_authored_sql(module)?; + register_declaration_values(module)?; + register_declaration_indexes(module)?; + register_macro_arguments(module)?; + register_macro_calls(module)?; + register_macro_expansion(module)?; + register_macro_expansion_result(module)?; + register_macro_modules(module)?; + register_sql_macro_inventory(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs index 3d0780ca6c..3b13e982ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs @@ -12,8 +12,8 @@ use sqlbuild_host::types::ResultEnum; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::assets::{or_none, without_none}; use crate::bindings::_helpers::result_output::projection::{Projection, ResultIdentity, rounded}; -use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; const DECIMAL_KIND: &str = "decimal"; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs index 2b2705b7da..3689487c6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs @@ -11,10 +11,10 @@ use sqlbuild_core::integration_results::main::normalize_integration_record::norm use sqlbuild_core::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::result_output::main::json_value::json_value; use sqlbuild_python_base::result_output::main::validation_error::validation_error; +use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; /// One decoded part of a record: its fields as attributes, and as a dictionary. fn field(fields: &Py, py: Python<'_>, name: &str) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs index bac0a07885..ecce1d6d96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs @@ -14,7 +14,6 @@ use sqlbuild_analysis::review_command::models::{IncrementalStart, ReviewGraph, V use crate::bindings::_helpers::build_command::command::run_build_command; use crate::bindings::_helpers::clone::clone_command::run_clone_command; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::review_command::planning::ReviewSettings; use crate::bindings::_helpers::review_command::support::{quietly, review_error}; @@ -24,6 +23,7 @@ use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; const FAILED_STATUS: &str = "failed"; const AUDIT_KIND: &str = "audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index e7dc5eeba3..0a225d96f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -27,12 +27,6 @@ use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::contracts::spec_resolution::{ - resolve_effective_adapter_name, resolve_target_name, -}; -use crate::bindings::_helpers::discovery::project_config::{ - configured_git_base_ref, configured_review, -}; use crate::bindings::_helpers::execution_history::conversion::optional_text; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; @@ -43,6 +37,10 @@ use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; +use sqlbuild_python_front::discovery::main::project::configured_review::configured_review; const BASE_REF_ALIAS: &str = "@base"; const UDF_KIND: &str = "udf"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs index 1f21885182..99aba3c21c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs @@ -8,7 +8,6 @@ use sqlbuild_analysis::review_command::main::review_schema_name::review_schema_n use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::janitor::janitor_keys::utc_datetime; use crate::bindings::_helpers::review_command::planning::{ReviewProject, ReviewSettings}; use crate::bindings::_helpers::review_command::support::review_error; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; /// Where the review builds and where its ownership marker goes. pub(crate) struct ReviewSchema { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs index 0101fba536..ce808b309e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs @@ -11,8 +11,8 @@ use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::models::{ParsedThresholds, ThresholdBound}; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::model_config::header_metadata::native_column_record; use crate::bindings::_helpers::rules::custom_hosts::rules_error; +use sqlbuild_python_front::model_config::main::headers::native_column_record::native_column_record; /// One native header column with the text of the path its location names. #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index c3d355114f..2b8ca2857c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::compiled_views::relation_keys::model_key_row; use crate::bindings::_helpers::compiled_views::relation_keys::{ RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, }; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; @@ -40,6 +39,7 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interruptibly; use sqlbuild_python_base::native_threads::models::Joined; use sqlbuild_python_base::records::main::fields::field::field as record_field; +use sqlbuild_python_front::render::models::SqlExpansionInputs; /// The lint thresholds used when the project leaves them unset. const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 8948387504..bdbb81f607 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -20,8 +20,6 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compiled_views::relation_keys::{ RelationKeys, project_relation_keys, }; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::fix_facts::compile_facts; @@ -43,6 +41,8 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::render::models::SqlExpansionInputs; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs index bd1e587a19..67026835f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs @@ -8,13 +8,13 @@ use sqlbuild_rules::lint_text::main::format_fixture_blocks::format_fixture_block use sqlbuild_rules::lint_text::models::FixtureFormatBlock; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::discovery::sql_test_records::parse_sql_test_file; use crate::bindings::_helpers::rules::constants::MODEL_TEST_MODE; use crate::bindings::_helpers::rules::fixture_schemas; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::records::parse_sql_test_file::parse_sql_test_file; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; #[pyfunction] #[pyo3(signature = (*, files, project_dir, discovered_inputs, sql_lexical_syntax))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs index e591c396c5..e398733481 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs @@ -9,10 +9,10 @@ use sqlbuild_rules::lint_text::types::FixtureFormatColumn; use std::collections::HashMap; use std::path::Path; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; -use crate::bindings::_helpers::model_config::header_columns::schema_columns; -use crate::bindings::_helpers::model_config::model_config_builder::raised_config_error; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::main::headers::schema_columns::schema_columns; +use sqlbuild_python_front::model_config::main::model_nodes::raised_config_error::raised_config_error; pub(crate) fn columns(columns: &Bound<'_, PyAny>) -> PyResult> { columns diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index f04f75f1a6..6c625c4144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -11,8 +11,6 @@ use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ ExpansionSources, ExpansionVars, present, sql_expansion_inputs, }; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -20,6 +18,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::render::models::SqlExpansionInputs; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 85491051e2..794d7a499b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -10,12 +10,12 @@ use crate::bindings::_helpers::compiled_views::model_input_view::{ ModelInputRecord, NativeModelInput, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::render::types::SpanRow; const FILE_PATH_ATTRIBUTE: &str = "file_path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 697ca29ddc..2f9ddf87c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -44,9 +44,6 @@ use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::project_tree::main::project_real_paths::project_real_paths; use sqlclean::sql_lint::main::fill_finding_path::fill_finding_path; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; -use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::dsl_headers::header_violations; use crate::bindings::_helpers::rules::native_header_parser::{ FUNCTION_KIND, HeaderFailure, NativeHeaderParser, external_identifiers, @@ -60,6 +57,10 @@ use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::render::main::declarations::expand_with_inputs::expand_with_inputs; +use sqlbuild_python_front::render::models::AuthoredExpansion; +use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_front::render::types::SpanRow; pub(crate) const NATIVE_ENGINE: &str = "native"; const HEADER_ENGINE: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index bd07d782b9..c22c853333 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -5,7 +5,6 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyTuple}; use sqlbuild_rules::lint_text::models::{ExpansionSpan, LintSqlBody}; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::lint_pipeline::{ BodyRole, SingleBody, SingleBodyRequest, prepare_single_body, }; @@ -14,6 +13,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::render::models::SqlExpansionInputs; /// `ExpansionSpan` objects of one pass. fn span_objects<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs index c90ecf9996..2568abb08e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs @@ -12,10 +12,10 @@ use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; use sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure; use sqlbuild_sqltext::header_strings::models::QuoteFailure; -use crate::bindings::_helpers::sqltext::model_headers::header_failure; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::sqltext::main::header_failure::header_failure; /// The path header rules report header syntax errors at. const SENTINEL_PATH: &str = ""; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 314da78b83..cce37d1f3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -37,7 +37,6 @@ use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::rule_run::bulk_cache_file; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; -use crate::bindings::_helpers::scopes::scope_reports::metadata_projection_value; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::value_error::value_error; @@ -48,6 +47,7 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::scopes::main::reports::metadata_projection_value::metadata_projection_value; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 1c11cf3692..673c166fd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -9,7 +9,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; -use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; @@ -18,6 +17,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ lint_sources, }; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::render::models::SqlExpansionInputs; /// The selected paths, or `None` for every file. pub(crate) fn selected_paths(selected: &Bound<'_, PyAny>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index d5b9c70d43..c0f7a4e6fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -649,7 +649,7 @@ fn path_rows(outcome: LintOutcome) -> Vec<(PathBuf, LintViolationRow)> { /// Build lint expansion with the compiled target and vars, reusing the compiler's scope only for the same discovery. fn lint_expansion_context( run: &SqlRun<'_, '_>, -) -> PyResult { +) -> PyResult { let py: Python<'_> = run.project.py(); let reuse: Option<&ExpansionReuse> = match run .early_lint diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 243048f163..11983dd763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -26,10 +26,7 @@ use crate::bindings::_helpers::build_progress::progress_text::write_execution_he use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_scenario_config; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -43,6 +40,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; +use sqlbuild_python_front::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 54a4e75a47..1422bd6b50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -24,7 +24,6 @@ use crate::bindings::_helpers::change_selection::base_snapshot::{ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; @@ -45,6 +44,7 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; const LOCAL_ADAPTER: &str = "duckdb"; const LOCAL_STATUSES: [&str; 4] = ["PASS", "FAIL", "ERROR", "SKIP"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs deleted file mode 100644 index b1de60ca4b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Declaration resolution contexts built natively from a Python scope lookup's positions. - -use std::collections::HashMap; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, intern, pyclass, pymethods}; - -use crate::bindings::_helpers::scopes::declaration_resolution::ResolutionInputs; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_scopes::scope_index::main::classify_resource::classify_resource; -use sqlbuild_scopes::scope_index::models::{ - ConsumerResource, GrantEntry, VisibilityReason, VisibilityTable, VisibleEntry, -}; - -const ENUM_KIND: u8 = 0; -const CONSTANT_KIND: u8 = 1; -const MACRO_KIND: u8 = 2; - -pub(crate) type TableRow = ( - Vec, - HashMap>, - HashMap>, -); -type VisibleRow<'py> = (Bound<'py, PyAny>, VisibleEntry, Option>); - -/// One declaration position: identity, first record, name, kinds, runtime value and key. -#[derive(FromPyObject)] -pub(crate) struct PositionRow(Py, Py, Py, u8, Py, u8, u32); - -/// The Python classes and enum members the contexts are built from. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ContextClasses { - visibility_record: Py, - pub(crate) resolution_context: Py, - reasons: Py, -} - -/// Positions, visibility table and classes of one scope lookup. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct NativeDeclarationContexts { - positions: Vec, - table: VisibilityTable, - pub(crate) classes: ContextClasses, - lookup: LookupMappings, - pub(crate) resolution: ResolutionInputs, -} - -/// The Python lookup mappings a matching resource's private positions and grants come from. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct LookupMappings { - identity_keys: Py, - private_positions: Py, - grants_by_resource: Py, -} - -/// Dictionaries one context is assembled into, in Python's insertion order. -struct ContextParts<'py> { - enums: Bound<'py, PyDict>, - constants: Bound<'py, PyDict>, - macros: Bound<'py, PyDict>, - macro_records: Bound<'py, PyDict>, - inaccessible: [Bound<'py, PyDict>; 3], - visibility: [Bound<'py, PyDict>; 3], -} - -#[pymethods] -impl NativeDeclarationContexts { - #[new] - pub(crate) fn new( - positions: Vec, - table: TableRow, - classes: ContextClasses, - lookups: (LookupMappings, ResolutionInputs), - ) -> Self { - let (lookup, resolution) = lookups; - let (global, local, inherited) = table; - let identity_keys: Vec = positions.iter().map(|row| row.6).collect(); - Self { - table: VisibilityTable { - identity_keys, - global, - local, - inherited, - }, - positions, - classes, - lookup, - resolution, - } - } - - /// `resolve_declaration_context`: the context and whether it was projected, not cached. - #[pyo3(signature = (resolver, file_path, resource=None))] - fn resolve<'py>( - &self, - resolver: Bound<'py, PyAny>, - file_path: Bound<'py, PyAny>, - resource: Option>, - ) -> PyResult<(Bound<'py, PyAny>, bool)> { - compiler_guard(|| { - let path: String = file_path.str()?.to_string(); - self.resolved_context(&resolver, &path, resource.as_ref()) - }) - } -} - -impl NativeDeclarationContexts { - pub(crate) fn classified_context<'py>( - &self, - py: Python<'py>, - matches: Vec>, - consumer: Bound<'py, PyAny>, - ) -> PyResult> { - let mut visible: Vec> = Vec::new(); - let mut inaccessible: Vec = Vec::new(); - for record in matches { - let identity: Bound<'py, PyAny> = native_field(&record, "identity")?; - let path: String = native_field(&record, "path")?.extract()?; - let private: Vec = self - .lookup - .private_positions - .bind(py) - .call_method1(intern!(py, "get"), (&identity, PyTuple::empty(py)))? - .extract()?; - if private - .iter() - .any(|position| *position >= self.positions.len()) - { - return Err(compiler_error( - "private declaration outside the scope lookup", - )); - } - let (grants, throughs) = self.grants(py, &identity)?; - let classified = classify_resource( - &self.table, - &ConsumerResource { - private, - path, - grants, - }, - ) - .map_err(|deferral| compiler_error(deferral.reason))?; - for entry in classified.visible { - let through: Option> = - entry.through.map(|slot| throughs[slot].clone()); - visible.push((identity.clone(), entry, through)); - } - inaccessible.extend(classified.inaccessible); - } - self.assemble(py, &visible, &inaccessible, consumer) - } - - /// The resource's grants of indexed declarations, with each grant's `through` value. - fn grants<'py>( - &self, - py: Python<'py>, - identity: &Bound<'py, PyAny>, - ) -> PyResult<(Vec, Vec>)> { - let records: Bound<'py, PyAny> = self - .lookup - .grants_by_resource - .bind(py) - .call_method1(intern!(py, "get"), (identity, PyTuple::empty(py)))?; - let identity_keys: &Bound<'py, PyDict> = self.lookup.identity_keys.bind(py); - let mut grants: Vec = Vec::new(); - let mut throughs: Vec> = Vec::new(); - for grant in records.try_iter()? { - let grant: Bound<'py, PyAny> = grant?; - let Some(identity_key) = - identity_keys.get_item(native_field(&grant, "declaration")?)? - else { - continue; - }; - let kind: String = native_field(&grant, "kind")? - .getattr(intern!(py, "value"))? - .extract()?; - grants.push(GrantEntry { - identity_key: identity_key.extract()?, - reason: if kind == VisibilityReason::TestedMacro.as_str() { - VisibilityReason::TestedMacro - } else { - VisibilityReason::ExpectedModel - }, - through: throughs.len(), - }); - throughs.push(native_field(&grant, "through")?); - } - Ok((grants, throughs)) - } - - fn assemble<'py>( - &self, - py: Python<'py>, - visible: &[VisibleRow<'py>], - inaccessible: &[usize], - consumer: Bound<'py, PyAny>, - ) -> PyResult> { - let parts: ContextParts<'py> = ContextParts { - enums: PyDict::new(py), - constants: PyDict::new(py), - macros: PyDict::new(py), - macro_records: PyDict::new(py), - inaccessible: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], - visibility: [PyDict::new(py), PyDict::new(py), PyDict::new(py)], - }; - let mut records_by_identity: HashMap>> = HashMap::new(); - for (resource, entry, through) in visible { - let record: Bound<'py, PyAny> = self.visibility_record(py, resource, entry, through)?; - records_by_identity - .entry(self.positions[entry.position].6) - .or_default() - .push(record); - } - for (_, entry, _) in visible { - self.add_visible_value(py, &parts, entry.position)?; - } - for position in inaccessible { - let row: &PositionRow = &self.positions[*position]; - if let Some(target) = parts.inaccessible.get(usize::from(row.3)) { - target.set_item(row.2.bind(py), row.1.bind(py))?; - } - } - for (_, entry, _) in visible { - let row: &PositionRow = &self.positions[entry.position]; - if let Some(target) = parts.visibility.get(usize::from(row.3)) { - let records: &[Bound<'py, PyAny>] = - records_by_identity.get(&row.6).map_or(&[], Vec::as_slice); - target.set_item(row.2.bind(py), PyTuple::new(py, records)?)?; - } - } - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("enums", &parts.enums)?; - fields.set_item("constants", &parts.constants)?; - fields.set_item("inaccessible_enums", &parts.inaccessible[0])?; - fields.set_item("inaccessible_constants", &parts.inaccessible[1])?; - fields.set_item("enum_visibility", &parts.visibility[0])?; - fields.set_item("constant_visibility", &parts.visibility[1])?; - fields.set_item("macros", &parts.macros)?; - fields.set_item("macro_records", &parts.macro_records)?; - fields.set_item("macro_visibility", &parts.visibility[2])?; - fields.set_item("inaccessible_macros", &parts.inaccessible[2])?; - fields.set_item("consumer", consumer)?; - self.classes - .resolution_context - .bind(py) - .call((), Some(&fields)) - } - - fn add_visible_value<'py>( - &self, - py: Python<'py>, - parts: &ContextParts<'py>, - position: usize, - ) -> PyResult<()> { - let row: &PositionRow = &self.positions[position]; - let name: &Bound<'py, PyAny> = row.2.bind(py); - let value: &Bound<'py, PyAny> = row.4.bind(py); - match row.5 { - ENUM_KIND => parts.enums.set_item(name, value), - CONSTANT_KIND => parts.constants.set_item(name, value), - MACRO_KIND => { - parts.macros.set_item(name, value)?; - parts.macro_records.set_item(name, row.1.bind(py)) - } - _ => Ok(()), - } - } - - fn visibility_record<'py>( - &self, - py: Python<'py>, - resource: &Bound<'py, PyAny>, - entry: &VisibleEntry, - through: &Option>, - ) -> PyResult> { - let none: Bound<'py, PyAny> = py.None().into_bound(py); - let reason: Bound<'py, PyAny> = self - .classes - .reasons - .bind(py) - .get_item(entry.reason.as_str())? - .unwrap_or_else(|| none.clone()); - self.visibility_record_for( - resource, - self.positions[entry.position].0.bind(py), - &reason, - through.as_ref().unwrap_or(&none), - ) - } - - /// A `VisibilityRecord` of `resource`, `declaration`, `reason` and `through`. - pub(crate) fn visibility_record_for<'py>( - &self, - resource: &Bound<'py, PyAny>, - declaration: &Bound<'py, PyAny>, - reason: &Bound<'py, PyAny>, - through: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = resource.py(); - let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); - let record: Bound<'py, PyAny> = class.call1((resource, declaration, reason, through))?; - Ok(record) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 5c1b242f57..3ab6a93648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -1,20 +1,3 @@ -pub(crate) mod compile_usages; -pub(crate) mod declaration_contexts; -pub(crate) mod declaration_resolution; -pub(crate) mod getters; -pub(crate) mod placement; pub(crate) mod registration; -pub(crate) mod relationship_names; -pub(crate) mod relationship_scan; -pub(crate) mod scope_cache; pub(crate) mod scope_command; -pub(crate) mod scope_errors; -pub(crate) mod scope_index; pub(crate) mod scope_loading; -pub(crate) mod scope_lookup; -pub(crate) mod scope_objects; -pub(crate) mod scope_report_objects; -pub(crate) mod scope_reports; -pub(crate) mod scope_tolerant; -pub(crate) mod scope_validation; -pub(crate) mod scope_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs index 59556826ae..745c924fe0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -2,22 +2,28 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::scopes::{ - compile_usages, declaration_contexts, relationship_names, scope_cache, scope_command, - scope_index, scope_loading, scope_lookup, scope_reports, scope_validation, scope_visibility, -}; +use crate::bindings::_helpers::scopes::{scope_command, scope_loading}; +use sqlbuild_python_front::scopes::main::checks::register_scope_validation::register_scope_validation; +use sqlbuild_python_front::scopes::main::checks::register_scope_visibility::register_scope_visibility; +use sqlbuild_python_front::scopes::main::declarations::register_declaration_contexts::register_declaration_contexts; +use sqlbuild_python_front::scopes::main::index::register_scope_cache::register_scope_cache; +use sqlbuild_python_front::scopes::main::index::register_scope_index::register_scope_index; +use sqlbuild_python_front::scopes::main::index::register_scope_lookup::register_scope_lookup; +use sqlbuild_python_front::scopes::main::relationships::register_compile_usages::register_compile_usages; +use sqlbuild_python_front::scopes::main::relationships::register_relationship_names::register_relationship_names; +use sqlbuild_python_front::scopes::main::reports::register_scope_reports::register_scope_reports; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_index::register(module)?; - relationship_names::register(module)?; - declaration_contexts::register(module)?; - compile_usages::register(module)?; - scope_lookup::register(module)?; - scope_cache::register(module)?; + register_scope_index(module)?; + register_relationship_names(module)?; + register_declaration_contexts(module)?; + register_compile_usages(module)?; + register_scope_lookup(module)?; + register_scope_cache(module)?; scope_loading::register(module)?; - scope_reports::register(module)?; + register_scope_reports(module)?; scope_command::register(module)?; - scope_validation::register(module)?; - scope_visibility::register(module)?; + register_scope_validation(module)?; + register_scope_visibility(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 52e8f26d74..69f9af83f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -14,13 +14,6 @@ use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; use crate::bindings::_helpers::scopes::scope_loading::load_or_build_scope_index; -use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; -use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; -use crate::bindings::_helpers::scopes::scope_report_objects::read_result; -use crate::bindings::_helpers::scopes::scope_reports::{ - ExplainArguments, ListArguments, ReportArguments, scope_browse, scope_explain, scope_list, - scope_move_preview, scope_report, serialize_scope_report, -}; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -29,6 +22,18 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::scopes::main::index::build_scope_lookup::build_scope_lookup; +use sqlbuild_python_front::scopes::main::reports::read_result::read_result; +use sqlbuild_python_front::scopes::main::reports::scope_browse::scope_browse; +use sqlbuild_python_front::scopes::main::reports::scope_explain::scope_explain; +use sqlbuild_python_front::scopes::main::reports::scope_list::scope_list; +use sqlbuild_python_front::scopes::main::reports::scope_move_preview::scope_move_preview; +use sqlbuild_python_front::scopes::main::reports::scope_report::scope_report; +use sqlbuild_python_front::scopes::main::reports::serialize_scope_report::serialize_scope_report; +use sqlbuild_python_front::scopes::models::ExplainArguments; +use sqlbuild_python_front::scopes::models::ListArguments; +use sqlbuild_python_front::scopes::models::ReportArguments; +use sqlbuild_python_front::scopes::models::ScopeClasses; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index bbf1e815ee..8f9d283296 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -14,16 +14,8 @@ use sqlbuild_host::entries::models::{AdapterHost, UserCodeHost}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::project::compile_adapter_context; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapshot; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; -use crate::bindings::_helpers::scopes::scope_cache::{ - project_fingerprint, read_scope_index_cache, write_scope_index_cache, -}; -use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; -use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; -use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -31,6 +23,14 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::snapshots::discover_scope_snapshot::discover_scope_snapshot; +use sqlbuild_python_front::scopes::main::checks::scope_index_with_compile_usages::scope_index_with_compile_usages; +use sqlbuild_python_front::scopes::main::checks::scope_tolerant_index::scope_tolerant_index; +use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_native_scope_index; +use sqlbuild_python_front::scopes::main::index::project_fingerprint::project_fingerprint; +use sqlbuild_python_front::scopes::main::index::read_scope_index_cache::read_scope_index_cache; +use sqlbuild_python_front::scopes::main::index::write_scope_index_cache::write_scope_index_cache; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 9062640ff8..74fc45deaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -18,7 +18,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; @@ -39,6 +38,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index fc4d1568a6..cbbae49101 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -15,9 +15,6 @@ use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_views::model_reads::binding_validated; -use crate::bindings::_helpers::model_config::config_nodes::PyConfigNode; -use crate::bindings::_helpers::model_config::header_audits::AuditInstance; -use crate::bindings::_helpers::model_config::header_metadata::HeaderValue; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::inputs::{ column_names, resource_sql_diagnostics, sql_test_columns, @@ -32,6 +29,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::model_config::models::AuditInstance; +use sqlbuild_python_front::model_config::types::HeaderValue; +use sqlbuild_python_front::model_config::types::PyConfigNode; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index da8ce29f46..0cc5d322ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::analysis::schema_request::{ SchemaRequestParts, schema_validation_request, }; use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; -use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; use sqlbuild_sqltext::resolution::main::unquoted_column_name::unquoted_column_name; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index cd434692d9..a721a2c9df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -36,7 +36,6 @@ use crate::bindings::_helpers::change_selection::base_snapshot::{ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; -use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; @@ -70,6 +69,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/timeouts.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/timeouts.rs index 99f7b4d756..ac94e0c23f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/timeouts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/timeouts.rs @@ -9,10 +9,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use sqlbuild_model_config::execution_limits::models::ExecutionLimit; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use sqlbuild_python_base::records::main::construct::new_error::new_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; const INVALID_TIMEOUT_CODE: &str = "C418"; const LOCAL_FILE: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index cee12a05bc..477491e529 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::path_posix_text::path_posix_text; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const MACRO_TOKEN: char = '@'; const INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index 8af42fa725..b9ab6050b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -8,13 +8,7 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; -use crate::bindings::_helpers::attachments::cursor_intrinsics::{ - compile_input_error, resolve_cursor_intrinsics, -}; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::cursor::conversion::CursorSession; -use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; -use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::sql_test_glue::constants::{ CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, @@ -26,6 +20,13 @@ use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_v use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::compile_input_error::compile_input_error; +use sqlbuild_python_front::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; +use sqlbuild_python_front::cursor::main::scalars::parse_value::parse_value; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// One model's cursor name, type and timestamp grain. struct ModelCursor<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 2ff1cb2cb8..ac3d60ae0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod assembly; -pub(crate) mod authored_ctes; pub(crate) mod compiled_tests; pub(crate) mod constants; mod cursor_window; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 6e8c36474c..01f3019e20 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -14,9 +14,9 @@ use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expecte use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; -use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::sql_test_glue::main::prefixed_ctes::prefixed_ctes; fn prefixed_names<'py>(test: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { let py = test.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index 965cac65cf..4e5953ee77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -4,13 +4,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_front::sql_test_glue::main::prefixed_ctes::prefixed_ctes; pub(super) fn validated_test<'py>( test: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index 587de8a5e9..45a0cfcced 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -13,10 +13,10 @@ use sqlbuild_analysis::compiler::models::{ }; use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 9b9f6f8a0d..8c7176fe72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests}; +use crate::bindings::_helpers::sql_test_glue::compiled_tests; +use sqlbuild_python_front::sql_test_glue::main::register_authored_ctes::register_authored_ctes; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -22,6 +23,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::plan_driver::sql_test_plan_error_messages, module )?)?; - authored_ctes::register(module)?; + register_authored_ctes(module)?; compiled_tests::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs deleted file mode 100644 index 0e2b6ebdff..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_sqltext::code_scanner::main::code_scanner::code_scanner; -use sqlbuild_sqltext::code_scanner::main::scanner_matching_paren::scanner_matching_paren; -use sqlbuild_sqltext::code_scanner::main::scanner_non_code_end::scanner_non_code_end; -use sqlbuild_sqltext::code_scanner::models::CodeScanner; -use sqlbuild_sqltext::sql_scan::models::Unclosed; - -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; - -/// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct SqlCodeScanner { - scanner: CodeScanner, - context: String, -} - -#[pymethods] -impl SqlCodeScanner { - #[new] - #[pyo3(signature = (sql, syntax, context))] - pub(crate) fn new(sql: &str, syntax: Option, context: String) -> Self { - Self { - scanner: code_scanner(sql, syntax.map(Into::into)), - context, - } - } - - /// The end of the comment or quoted text at `index`, or `None` when code starts there. - pub(crate) fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { - scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) - } - - /// The index of the parenthesis closing the one at `open`. - pub(crate) fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { - scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) - } -} - -impl SqlCodeScanner { - /// `CompileInputError(" contains an unclosed ...")`. - fn error(&self, py: Python<'_>, unclosed: Unclosed) -> PyErr { - let construct: &str = match unclosed { - Unclosed::BlockComment => "block comment", - Unclosed::Quote => "quoted string", - Unclosed::Parenthesis => "parenthesis", - }; - let message: String = format!("{} contains an unclosed {construct}", self.context); - let raised = || -> PyResult { - let error: Bound<'_, PyAny> = - error_class(py, ErrorId::CompileInput)?.call1((message,))?; - Ok(PyErr::from_value(error)) - }; - raised().unwrap_or_else(|error| error) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 670af7fcaa..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,10 +1 @@ -pub(crate) mod authored_values; -pub(crate) mod code_scanner; -pub(crate) mod ddl_effects; -pub(crate) mod declaration_references; -pub(crate) mod lexical_syntax; -pub(crate) mod model_headers; -pub(crate) mod placeholders; pub(crate) mod registration; -pub(crate) mod sql_references; -pub(crate) mod static_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 655c754951..c9a5f48cf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -2,15 +2,17 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::sqltext::{ - code_scanner, ddl_effects, model_headers, placeholders, sql_references, -}; +use sqlbuild_python_front::sqltext::main::register_code_scanner::register_code_scanner; +use sqlbuild_python_front::sqltext::main::register_ddl_effects::register_ddl_effects; +use sqlbuild_python_front::sqltext::main::register_model_headers::register_model_headers; +use sqlbuild_python_front::sqltext::main::register_placeholders::register_placeholders; +use sqlbuild_python_front::sqltext::main::register_sql_references::register_sql_references; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_headers::register(module)?; - code_scanner::register(module)?; - sql_references::register(module)?; - placeholders::register(module)?; - ddl_effects::register(module)?; + register_model_headers(module)?; + register_code_scanner(module)?; + register_sql_references(module)?; + register_placeholders(module)?; + register_ddl_effects(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index 59acf36d54..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -1,11 +1 @@ -pub(crate) mod constants; -pub(crate) mod events; -pub(crate) mod freshness; -pub(crate) mod migration_io; -pub(crate) mod migration_objects; -pub(crate) mod migration_projections; -pub(crate) mod migration_rows; -pub(crate) mod models; pub(crate) mod registration; -pub(crate) mod state_tables; -pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index cb78b58903..1137b4acb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -2,16 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::state_rows::{ - events, freshness, migration_io, migration_projections, migration_rows, state_tables, -}; +use sqlbuild_python_front::state_rows::main::event_rows::register_events::register_events; +use sqlbuild_python_front::state_rows::main::event_rows::register_freshness::register_freshness; +use sqlbuild_python_front::state_rows::main::migrations::register_migration_io::register_migration_io; +use sqlbuild_python_front::state_rows::main::migrations::register_migration_projections::register_migration_projections; +use sqlbuild_python_front::state_rows::main::migrations::register_migration_rows::register_migration_rows; +use sqlbuild_python_front::state_rows::main::tables::register_state_tables::register_state_tables; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - events::register(module)?; - freshness::register(module)?; - migration_rows::register(module)?; - migration_io::register(module)?; - migration_projections::register(module)?; - state_tables::register(module)?; + register_events(module)?; + register_freshness(module)?; + register_migration_rows(module)?; + register_migration_io(module)?; + register_migration_projections(module)?; + register_state_tables(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index ce1d809570..b48ee3ef23 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -98,67 +98,6 @@ impl From for validation::PositionInput { } } -/// The directory listings one discovery pass shares across its native calls. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct NativeProjectTree { - pub(crate) inner: sqlbuild_discovery::tree::models::ProjectTree, -} - -/// One discovery pass of a project: the memo its native calls share, active inside its scope. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct NativeDirectorySnapshot { - pub(crate) project_dir: pyo3::Py, - pub(crate) memo: pyo3::Py, -} - -/// The context manager that makes one snapshot active for a project until it exits. -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct NativeDirectorySnapshotScope { - pub(crate) project_dir: pyo3::Py, - /// The context-variable token of the snapshot this scope activated, if it activated one. - pub(crate) token: Option>, -} - -/// One native YAML file discovery request; `kind` is `source` or `schema`. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct YamlDiscoveryRequest { - pub(crate) project_dir: String, - pub(crate) display_prefix: String, - pub(crate) kind: String, -} - -/// One in-memory SQL model parse request; `file_path` is the path messages name. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct ModelTextRequest { - pub(crate) file_path: String, - pub(crate) supported_keys: Vec, - pub(crate) removed_keys: Vec, - pub(crate) extract_implicit_alias_columns: bool, - pub(crate) extract_output_column_locations: bool, -} - -/// The declaration collections one Python discovery pass read natively, kept for later stages. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct NativeDiscoverySession { - pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, -} - -/// One in-memory declaration file parse request; `kind` names the collection it belongs to. -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct DeclarationTextRequest { - pub(crate) kind: String, - pub(crate) function_keys: Vec, - pub(crate) audit_keys: Vec, - pub(crate) hook_keys: Vec, -} - /// A record of another crate exchanged with Python as its positional tuple. #[derive(Clone, Debug)] pub(crate) struct Positional(pub(crate) T); diff --git a/fensu.toml b/fensu.toml index 9ab61266bc..0899b845ad 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1445,7 +1445,7 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgre reason = "PostgreSQL and SQLite execution history each read a page through their own connection, locking and cursor keys; filters and record conversion come from conversion.rs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs", "crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." [[dupes.allowlist]] @@ -1492,10 +1492,6 @@ reason = "Metadata sync keeps its pending page for retry and work polling drops paths = ["crates/sqlbuild-cli/src/workers/_helpers/child.rs"] reason = "terminate and kill are the two escalation steps for a worker child; each maps every child kind to its own signal." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] -reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." - [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py" @@ -1680,8 +1676,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 197891 } -reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Set after #1514 moved the shared runtime into sqlbuild-python-base, plus 4 lines for the janitor preserve check from #1512." +thresholds = { max_crate_code_lines = 169791 } +reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] @@ -1704,13 +1700,13 @@ allowed = [ allowed_paths = [ "crates/sqlbuild-core/src/constants.rs=Names the observation state table.", "crates/sqlbuild-orchestration=Mirrors observations into the control store and serves them for display.", - "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants=Exports the table name to Python adapters, which create it.", "crates/sqlbuild-python/src/bindings/_helpers/janitor=Prunes observation history.", - "crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs=Declares the observation state bindings module.", - "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs=Renders observation state SQL for adapters.", + "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs=Renders observation state SQL for adapters.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs=Reads observations for the control-store mirror.", - "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", + "crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs=Declares the observation state module.", + "crates/sqlbuild-python-front/src/native_host/main/append_observations.rs=Writes observations after a build.", + "crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs=Registers the observation state bindings.", ] [targets.rust.rule_options.XSB079] @@ -1718,7 +1714,6 @@ allowed_paths = [ "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", - "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs=Defines the synced-copy adapter.", "crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs=Registers the synced copy's refusal error with Python.", @@ -1732,6 +1727,7 @@ allowed_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=Preview branch: sources outside the copy are not probed.", + "crates/sqlbuild-python-front/src/native_host/models.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", ] [targets.rust.rule_options.XSB077] @@ -1759,6 +1755,42 @@ path = "crates/sqlbuild-python-base/src/native_host/models.rs" symbols = ["AuditProjection", "FamilyProjection"] reason = "The header projections implement Debug manually because their cached public tuples sit in a PyOnceLock, which does not implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-front/src/cursor/models.rs" +symbols = ["CursorSession"] +reason = "CursorSession implements Debug manually because it holds the interpreter token, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-front/src/discovery/models.rs" +symbols = ["NativeModelFile"] +reason = "NativeModelFile implements Debug manually because its cached Python texts sit in PyOnceLocks, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-front/src/model_config/models.rs" +symbols = ["PythonHost", "HeaderColumn", "HeaderColumns"] +reason = "PythonHost implements Debug manually because it holds the interpreter token, which does not implement Debug, and the header columns because their cached projections sit in a PyOnceLock, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-front/src/render/models.rs" +symbols = ["SqlMacroSource"] +reason = "SqlMacroSource implements Debug manually because it borrows the macro scopes as a trait object, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-front/src/scopes/models.rs" +symbols = ["ScopeClasses"] +reason = "ScopeClasses implements Debug manually because it holds the interpreter token, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-front/src/render/types.rs" +symbols = ["StateTuple"] +reason = "Python passes the expansion state as one five-item tuple; StateInput reads it in a single tuple extraction and names each position, as the positional record it replaces did." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -1919,6 +1951,7 @@ crate_names = [ "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python-base", + "sqlbuild-python-front", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -1936,7 +1969,7 @@ paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/cla reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs"] +paths = ["crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs"] reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." [[dupes.allowlist]] @@ -1989,6 +2022,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/records/main/registry/native_attr.rs", "crates/sqlbuild-python-base/src/warehouse_access", + "crates/sqlbuild-python-front/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2029,6 +2063,98 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registration.rs=26", "crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs=2", "crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs=3", + "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=1", + "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=3", + "crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs=1", + "crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs=3", + "crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs=3", + "crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs=10", + "crates/sqlbuild-python-front/src/authored_values/models.rs=2", + "crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs=4", + "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs=2", + "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=2", + "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=9", + "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=2", + "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", + "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=7", + "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs=13", + "crates/sqlbuild-python-front/src/cursor/_helpers/values.rs=4", + "crates/sqlbuild-python-front/src/cursor/models.rs=5", + "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs=9", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs=5", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs=16", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs=3", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs=3", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs=14", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=9", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs=15", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=8", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs=5", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=14", + "crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs=2", + "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=26", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs=5", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs=4", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs=2", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs=4", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs=3", + "crates/sqlbuild-python-front/src/model_config/models.rs=2", + "crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs=6", + "crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs=1", + "crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs=11", + "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs=4", + "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs=1", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs=2", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs=24", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs=1", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs=12", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs=14", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs=6", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs=4", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs=9", + "crates/sqlbuild-python-front/src/render/models.rs=10", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs=5", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs=5", + "crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs=15", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs=24", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs=8", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs=18", + "crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs=4", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs=3", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs=23", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs=2", + "crates/sqlbuild-python-front/src/scopes/models.rs=13", + "crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs=2", + "crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs=3", + "crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs=1", + "crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs=1", + "crates/sqlbuild-python-front/src/sqltext/models.rs=1", + "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs=1", + "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs=4", + "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs=4", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs=7", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=1", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=5", + "crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", @@ -2037,16 +2163,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", @@ -2083,7 +2203,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", @@ -2107,7 +2226,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs=3", @@ -2143,8 +2261,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", @@ -2159,12 +2275,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=7", @@ -2206,30 +2316,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", @@ -2310,27 +2396,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", @@ -2458,27 +2528,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", @@ -2530,21 +2585,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=32", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs=18", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", @@ -2555,7 +2597,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", @@ -2565,18 +2606,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", "crates/sqlbuild-python/src/bindings/models.rs=0", ] @@ -2589,6 +2618,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/warehouse_access", + "crates/sqlbuild-python-front/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2609,13 +2639,51 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs=1", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/error_classes.rs=1", "crates/sqlbuild-python-base/src/records/models.rs=3", + "crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs=3", + "crates/sqlbuild-python-front/src/authored_values/models.rs=6", + "crates/sqlbuild-python-front/src/compile_driver/models.rs=3", + "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=1", + "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", + "crates/sqlbuild-python-front/src/cursor/models.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs=3", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs=12", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", + "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=13", + "crates/sqlbuild-python-front/src/discovery/models.rs=7", + "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=9", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs=1", + "crates/sqlbuild-python-front/src/model_config/models.rs=23", + "crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs=2", + "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs=1", + "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs=4", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs=10", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs=2", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs=2", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs=19", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_facts.rs=2", + "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs=6", + "crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs=1", + "crates/sqlbuild-python-front/src/render/models.rs=39", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs=1", + "crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs=12", + "crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs=13", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs=5", + "crates/sqlbuild-python-front/src/scopes/models.rs=11", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", + "crates/sqlbuild-python-front/src/state_rows/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=6", @@ -2648,7 +2716,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs=2", @@ -2671,13 +2738,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", @@ -2690,17 +2754,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", @@ -2742,12 +2795,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_classes.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", @@ -2787,23 +2835,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=33", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=4", @@ -2828,11 +2863,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=3", @@ -2844,9 +2874,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs=3", - "crates/sqlbuild-python/src/bindings/models.rs=4", + "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB075] @@ -2869,11 +2897,11 @@ module_layers = [ "analysis_session/query_columns=project", "analysis_session/session=project", "analysis_session/stage_functions=project", - "attachments=front", + "attachments=registry", "audit_command=commands", "auditing=compile", "auditing/render=project", - "authored_values=front", + "authored_values=registry", "build_command=commands", "build_progress=output", "cache=registry", @@ -2882,14 +2910,11 @@ module_layers = [ "cli=commands", "clone=commands", "command_runtime=commands", - "command_runtime/concurrency=front", "compile_attachment=compile", "compile_attachment/cursor_roles=project", - "compile_attachment/effective_config=front", "compile_command=compile", "compile_delta=compile", "compile_driver=compile", - "compile_driver/config_validation=front", "compile_driver/plan_pipeline=commands", "compile_driver/plan_result=commands", "compile_driver/static_context=commands", @@ -2900,14 +2925,12 @@ module_layers = [ "contract_command=commands", "consumption=plan", "contracts=project", - "contracts/effective_config=front", - "contracts/spec_resolution=front", "cost_command=commands", "cost_report=output", "cost_telemetry=output", "cost_telemetry/context=front", "cost_telemetry/values=front", - "cursor=front", + "cursor=registry", "dag_artifact=compile", "dagster_invocation=commands", "dbt_command=commands", @@ -2916,7 +2939,7 @@ module_layers = [ "debug_command=commands", "diagnostics=registry", "diff_report=commands", - "discovery=front", + "discovery=registry", "event_exporting=output", "execution_history=front", "exported_constants=registry", @@ -2937,15 +2960,12 @@ module_layers = [ "lineage/lineage_facts=project", "load=commands", "location_command=commands", - "macro_bridge=front", + "macro_bridge=registry", "manifest=compile", "microbatches=plan", - "model_config=front", + "model_config=registry", "native_host=front", - "native_host/audit_results_state=front", - "native_host/consumption_state=front", "native_host/model_lookup=project", - "native_host/observations_state=front", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", "native_host/resource_costs_state=front", @@ -2995,7 +3015,7 @@ module_layers = [ "refactor/refactor_command=commands", "references=project", "relation_naming=project", - "render=front", + "render=registry", "result_output=output", "review_command=commands", "rules=compile", @@ -3008,7 +3028,6 @@ module_layers = [ "semantic_shapes=project", "sql_test_command=commands", "sql_test_glue=compile", - "sql_test_glue/authored_ctes=front", "sql_test_glue/constants=infra", "sql_test_glue/cursor_window=project", "sql_test_glue/fixture_validation=project", @@ -3022,8 +3041,8 @@ module_layers = [ "sql_test_glue/plan_rows=project", "sql_test_glue/plan_values=project", "sql_test_glue/planning=project", - "sqltext=front", - "state_rows=front", + "sqltext=registry", + "state_rows=registry", "type_system=project", "workspace_commands=commands", ] @@ -3066,11 +3085,11 @@ paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "c reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs", "crates/sqlbuild-analysis/src/manifest/_helpers/values.rs", "crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs", "crates/sqlbuild-config/src/project/_helpers/fields.rs", "crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs", "crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs"] +paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs", "crates/sqlbuild-analysis/src/manifest/_helpers/values.rs", "crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs", "crates/sqlbuild-config/src/project/_helpers/fields.rs", "crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs", "crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs"] reason = "Each fills its own ordered key/value list in its own crate; a shared ordered-map helper across these crates is a separate change." [[dupes.allowlist]] @@ -3098,7 +3117,7 @@ paths = ["crates/sqlbuild-analysis/src/semantic_validation/models.rs", "crates/s reason = "Hand-written Debug implementations that each choose which of their own fields to show, so secrets, caches and large catalogs stay out of logs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs"] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs"] reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." [[dupes.allowlist]] @@ -3145,6 +3164,38 @@ reason = "Each public entry forwards to its own helper with the same arguments." paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs"] reason = "Each public entry forwards to its own helper with the same arguments." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs", "crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs", "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs", "crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs", "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] +reason = "Each reads its own Python record's fields into its own native record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs", "crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/models.rs", "crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs"] +reason = "Each host keeps its own first Python error and returns its own failure type; only the string read is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/authored_values/models.rs", "crates/sqlbuild-python-front/src/cursor/models.rs"] +reason = "SqlValueClasses::load and CursorSession::new each load the record classes their own conversions need; CursorSession's to_utc and utc_offset each turn the zone call's result into their own value." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index f4a5d3bcbf..658b6ada96 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -286,6 +286,7 @@ NATIVE_BINDINGS_ROOTS: tuple[str, ...] = ( "crates/sqlbuild-python/src/", "crates/sqlbuild-python-base/src/", + "crates/sqlbuild-python-front/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index ebe33e1b7d..2c5ea422f0 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -4,6 +4,7 @@ SCANNED_SOURCE_ROOTS: tuple[str, ...] = ( "crates/sqlbuild-python-base/src", + "crates/sqlbuild-python-front/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 316c2ae063..4c175d640f 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -181,10 +181,10 @@ crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs Warehous crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs CliUserError.code crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs CliUserError.help crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs CliUserError.message -crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs EventExporterCommandScope.cached_extensions -crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs SourceWriteStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs ScopeKind.value +crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs EventExporterCommandScope.cached_extensions +crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs SourceWriteStrategy.value +crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs DeclarationKind.value +crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs ScopeKind.value crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs EventExporterCommandScope.cached_extensions crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs TransientStatusReporter.complete crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs SourceFreshnessAgeStatus.value @@ -250,9 +250,9 @@ crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs LoadCallbackRe crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs LoadProgressReporter.on_progress crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs LoadProgressReporter.on_start crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs CollectionRendering.value -crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs SqlValueKind.value +crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs CollectionRendering.value +crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs DeclarationKind.value +crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs DeclarationKind.value crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs ScopeKind.value crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs SourceWriteStrategy.value @@ -380,13 +380,13 @@ crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs Ma crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.relative_position crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs ColumnMigrationDecision.blocks_build crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs SqlValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs CollectionRendering.value -crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs VisibilityReason.value -crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs NativeMacroExpansion.render_relation -crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs ResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs ResourceKind.value +crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs DeclarationKind.value +crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs CollectionRendering.value +crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs VisibilityReason.value +crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs DeclarationKind.value +crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs NativeMacroExpansion.render_relation +crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs ResourceKind.value +crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs ResourceKind.value crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs PlanReason.value crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs RuleSubject.value @@ -399,22 +399,22 @@ crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs Sc crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs PlanningProgressReporter.on_progress crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs ScenarioProgressRelay.on_scenario_complete crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs ScenarioProgressRelay.on_scenario_start -crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs ScopeKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs ScopeDiagnosticCode.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs CollectionRendering.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs ScopeKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs SqlValueKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs DeclarationKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs ScopeKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs ScopeDiagnosticCode.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs CollectionRendering.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs ScopeKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs NativeScopeIndex.grant_relationships -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs GrantKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs OwnershipRootKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs ResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs ScopeDiagnosticCode.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs ScopeDiagnosticSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs ScopeKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs UsageKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs ScopeDiagnosticCode.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs DeclarationKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs GrantKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs OwnershipRootKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ResourceKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDiagnosticCode.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDiagnosticSeverity.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs UsageKind.value +crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs ScopeDiagnosticCode.value crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.configure_resources crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.consume_resource_terminal @@ -431,11 +431,11 @@ crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs Curs crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs CompiledResourceType.value crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs WarningSeverity.value crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs CompiledResourceType.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs OldNameViewEventType.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs ColumnMigrationDecision.records_event -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs OldNameViewEventType.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs ColumnMigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs MigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs MigrationDiscovery.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs OldNameViewDropReason.value -crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs OldNameViewEventType.value +crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs OldNameViewEventType.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs ColumnMigrationDecision.records_event +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs OldNameViewEventType.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs ColumnMigrationDecision.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs MigrationDecision.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs MigrationDiscovery.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs OldNameViewDropReason.value +crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs OldNameViewEventType.value From 372f7b22d125ac1d5fd9cea1ce5c25b4db884f30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:27:36 +0000 Subject: [PATCH 02/10] fix: keep the observation write and registration entries under the observation boundary The observation boundary matched `observations_state` as a whole word, so the `append_observations` and `register_observations_state` entries the front crate exports, and the files that call them, fell outside it after the move. The pattern now names both entries, and their callers are listed again. --- fensu.toml | 3 +++ scripts/fensu_policy/constants.py | 3 ++- .../fensu_policy/test_observation_boundary.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/fensu.toml b/fensu.toml index 0899b845ad..c15a410de7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1707,6 +1707,9 @@ allowed_paths = [ "crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs=Declares the observation state module.", "crates/sqlbuild-python-front/src/native_host/main/append_observations.rs=Writes observations after a build.", "crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs=Registers the observation state bindings.", + "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", + "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", + "crates/sqlbuild-python-front/src/native_host/main/mod.rs=Declares the observation write and registration entries.", ] [targets.rust.rule_options.XSB079] diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 658b6ada96..77f5de4f70 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -488,7 +488,8 @@ ), ) OBSERVATION_STATE_PATTERN: str = ( - r"\b(?:observations_state|OBSERVATIONS_TABLE_NAME|WarehouseObservationRecord" + r"\b(?:observations_state|register_observations_state|append_observations" + r"|OBSERVATIONS_TABLE_NAME|WarehouseObservationRecord" r"|WarehouseHistoryKind::Observations|_sqlbuild_observations|sqb_warehouse_observations)\b" ) SYNCED_COPY_ALLOWED_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( diff --git a/tests/integration/scripts/fensu_policy/test_observation_boundary.py b/tests/integration/scripts/fensu_policy/test_observation_boundary.py index 70bdb1ad5d..0b153492fd 100644 --- a/tests/integration/scripts/fensu_policy/test_observation_boundary.py +++ b/tests/integration/scripts/fensu_policy/test_observation_boundary.py @@ -39,6 +39,21 @@ expected_returncode=1, expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:1:0",), ), + ObservationBoundaryTestCase( + description="planner calling the observation write and registration entry points", + rule="XSB078", + allowed=("crates/sqlbuild-python/src/observations",), + planner=( + "pub fn plan() {}\n" + "pub fn record(rows: &[Row]) { append_observations(rows); }\n" + "pub fn install(module: &Module) { register_observations_state(module); }\n" + ), + expected_returncode=1, + expected_faults=( + " --> crates/sqlbuild-python/src/planner.rs:2:0", + " --> crates/sqlbuild-python/src/planner.rs:3:0", + ), + ), ObservationBoundaryTestCase( description="allowed path that no longer names observation state", rule="XSB078", From f80196faae2811b57118ddb04d7d7bbefd436af1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:13:57 +0000 Subject: [PATCH 03/10] wip: move the project layer into sqlbuild-python-project Incomplete: both crates compile with no warnings; fensu layout findings (helper containers, models roles, tuple arity, Debug derives) and the pipeline's budget, allowlist and readme steps are still open. Not tested. --- Cargo.lock | 34 +- Cargo.toml | 5 +- crates/sqlbuild-python-project/Cargo.toml | 37 + .../src/analysis/_helpers/binding_catalog.rs | 152 ++ .../src/analysis/_helpers}/catalog.rs | 35 +- .../src/analysis/_helpers}/conversions.rs | 0 .../src/analysis/_helpers/mod.rs | 9 + .../src/analysis/_helpers/native_classes.rs} | 63 +- .../src/analysis/_helpers}/normalization.rs | 6 +- .../_helpers}/normalization_results.rs | 0 .../src/analysis/_helpers}/positions.rs | 3 +- .../src/analysis/_helpers}/relation_names.rs | 4 +- .../src/analysis/_helpers}/schema_request.rs | 13 +- .../src/analysis/constants.rs | 11 + .../main/extract_sql_relation_names.rs | 16 + .../src/analysis/main/ignores_quoted_case.rs | 15 + .../src/analysis/main/mod.rs | 7 + .../analysis/main/normalize_analysis_sql.rs | 14 + .../analysis/main/register_binding_catalog.rs | 12 + .../analysis/main/register_normalization.rs | 12 + .../main/schema_validation_request.rs | 17 + .../src/analysis/main/schema_validations.rs | 17 + .../src/analysis/mod.rs | 7 + .../src/analysis/models.rs} | 284 +-- .../src/analysis/types.rs | 4 + .../_helpers}/fixture_columns.rs | 6 +- .../src/analysis_session/_helpers/mod.rs | 6 + .../_helpers}/nullability_rules.rs | 0 .../_helpers}/profile_inputs.rs | 10 +- .../_helpers}/query_columns.rs | 11 +- .../src/analysis_session/_helpers}/session.rs | 310 +-- .../_helpers}/stage_functions.rs | 33 +- .../src/analysis_session/constants.rs | 3 + .../main/case_sensitive_shapes.rs | 15 + .../main/dialect_or_generic.rs | 12 + .../main/expression_source_shapes.rs | 16 + .../infer_native_expression_source_shapes.rs | 15 + .../main/infer_native_query_columns.rs | 15 + .../src/analysis_session/main/mod.rs | 18 + .../main/nullability_rules.rs | 14 + .../analysis_session/main/profile_dialect.rs | 12 + .../main/prove_dynamic_column_contracts.rs | 16 + .../main/query_column_rows.rs | 17 + .../main/query_columns_error.rs | 11 + .../main/register_fixture_columns.rs | 12 + .../main/register_query_columns.rs | 12 + .../analysis_session/main/register_session.rs | 12 + .../main/register_stage_functions.rs | 12 + .../src/analysis_session/main/return_types.rs | 13 + .../src/analysis_session/main/shape_rows.rs | 13 + .../main/start_model_analysis_session.rs | 23 + .../src/analysis_session/main/text_pairs.rs | 13 + .../src/analysis_session/mod.rs | 7 + .../src/analysis_session/models.rs | 425 ++++ .../src/analysis_session/types.rs | 4 + .../src/auditing/_helpers/mod.rs | 1 + .../src/auditing/_helpers}/render.rs | 5 +- .../src/auditing/main/mod.rs | 2 + .../src/auditing/main/register_render.rs | 12 + .../src/auditing/main/render_audit_sql.rs | 12 + .../src/auditing/mod.rs | 4 + .../_helpers}/cursor_roles.rs | 9 +- .../src/compile_attachment/_helpers/mod.rs | 1 + .../src/compile_attachment/main/mod.rs | 2 + .../main/resolve_cursor_input_roles.rs | 12 + .../resolve_cursor_input_roles_function.rs | 11 + .../src/compile_attachment/mod.rs | 4 + .../_helpers/attachment_inputs.rs | 77 + .../compiled_project/_helpers}/job_catalog.rs | 6 +- .../src/compiled_project/_helpers/mod.rs | 14 + .../src/compiled_project/_helpers/project.rs | 175 ++ .../src/compiled_project/_helpers}/rows.rs | 13 +- .../_helpers}/schema_facts.rs | 0 .../src/compiled_project/_helpers/snapshot.rs | 48 + .../_helpers}/snapshot_boundary.rs | 12 +- .../_helpers}/snapshot_declarations.rs | 8 +- .../_helpers/snapshot_execution.rs | 32 + .../_helpers}/snapshot_models.rs | 14 +- .../_helpers}/snapshot_resources.rs | 21 +- .../_helpers}/snapshot_roles.rs | 2 +- .../_helpers/snapshot_worker.rs | 24 + .../src/compiled_project/_helpers}/values.rs | 0 .../src}/compiled_project/constants.rs | 14 +- .../main/attachment_inputs/install.rs | 10 + .../main/attachment_inputs/mod.rs | 1 + .../main/job_catalog/graph_with_jobs.rs | 18 + .../main/job_catalog/job_names.rs | 11 + .../compiled_project/main/job_catalog/mod.rs | 5 + .../main/job_catalog/needed_jobs.rs | 19 + .../main/job_catalog/triggered_jobs.rs | 18 + .../main/job_catalog/with_job_scope.rs | 13 + .../src/compiled_project/main/mod.rs | 12 + .../main/project/config_values.rs | 12 + .../main/project/declarations.rs | 12 + .../main/project/materialization.rs | 12 + .../src/compiled_project/main/project/mod.rs | 6 + .../main/project/model_facts.rs | 12 + .../main/project/register_project.rs | 12 + .../main/project/text_value.rs | 12 + .../src/compiled_project/main/rows/mod.rs | 1 + .../main/rows/reference_row_tuple.rs | 11 + .../main/schema_facts/entry_schema_fact.rs | 16 + .../main/schema_facts/family_fact.rs | 11 + .../compiled_project/main/schema_facts/mod.rs | 4 + .../main/schema_facts/python_column_fact.rs | 16 + .../main/schema_facts/python_column_facts.rs | 16 + .../src/compiled_project/main/snapshot/mod.rs | 1 + .../main/snapshot/register_snapshot.rs | 12 + .../main/snapshot_boundary/mod.rs | 1 + .../main/snapshot_boundary/relative_path.rs | 12 + .../compile_declarations.rs | 28 + .../main/snapshot_declarations/mod.rs | 2 + .../python_declarations.rs | 19 + .../main/snapshot_models/mod.rs | 1 + .../main/snapshot_models/model_nodes.rs | 22 + .../main/snapshot_resources/mod.rs | 1 + .../main/snapshot_resources/resource_nodes.rs | 20 + .../snapshot_roles/compile_project_roles.rs | 19 + .../main/snapshot_roles/mod.rs | 1 + .../src/compiled_project/main/values/mod.rs | 1 + .../main/values/plain_value.rs | 13 + .../src/compiled_project/mod.rs | 7 + .../src/compiled_project/models.rs | 2003 +++++++++++++++ .../src/compiled_project/types.rs | 36 + .../compiled_views/_helpers}/audit_reads.rs | 2 +- .../compiled_views/_helpers}/audit_views.rs | 100 +- .../_helpers}/compiled_model_analysis.rs | 13 +- .../_helpers/compiled_model_view.rs | 262 ++ .../_helpers}/function_reads.rs | 2 +- .../_helpers}/function_views.rs | 152 +- .../src/compiled_views/_helpers}/getters.rs | 104 +- .../compiled_views/_helpers/inputs_view.rs | 184 ++ .../src/compiled_views/_helpers/mod.rs | 20 + .../_helpers}/model_input_view.rs | 289 +-- .../compiled_views/_helpers}/model_reads.rs | 16 +- .../compiled_views/_helpers}/project_host.rs | 41 +- .../compiled_views/_helpers}/project_reads.rs | 2 +- .../compiled_views/_helpers/project_view.rs | 270 ++ .../src/compiled_views/_helpers}/records.rs | 24 +- .../compiled_views/_helpers}/relation_keys.rs | 39 +- .../_helpers}/resource_reads.rs | 15 +- .../compiled_views/_helpers}/seed_reads.rs | 2 +- .../src/compiled_views/_helpers/seed_views.rs | 128 + .../compiled_views/_helpers}/source_reads.rs | 2 +- .../compiled_views/_helpers/source_views.rs | 163 ++ .../src/compiled_views/constants.rs | 80 + .../main/audit_reads/always_run.rs | 12 + .../main/audit_reads/attached_column_name.rs | 12 + .../main/audit_reads/attached_target_kind.rs | 12 + .../main/audit_reads/attached_target_name.rs | 12 + .../main/audit_reads/definition_name.rs | 12 + .../main/audit_reads/description.rs | 12 + .../main/audit_reads/evaluation_mode.rs | 12 + .../main/audit_reads/evidence_limit.rs | 12 + .../main/audit_reads/evidence_sql.rs | 12 + .../compiled_views/main/audit_reads/key.rs | 12 + .../main/audit_reads/measurement_contract.rs | 12 + .../compiled_views/main/audit_reads/meta.rs | 12 + .../main/audit_reads/minimum_samples.rs | 12 + .../compiled_views/main/audit_reads/mod.rs | 20 + .../compiled_views/main/audit_reads/name.rs | 12 + .../main/audit_reads/references.rs | 12 + .../main/audit_reads/run_scope.rs | 12 + .../main/audit_reads/scope_deps.rs | 12 + .../main/audit_reads/severity.rs | 12 + .../main/audit_reads/sql_body.rs | 12 + .../main/audit_reads/thresholds.rs | 12 + .../main/audit_views/audit_record.rs | 11 + .../main/audit_views/audit_reference_rows.rs | 13 + .../compiled_views/main/audit_views/mod.rs | 2 + .../main/function_reads/arguments.rs | 12 + .../main/function_reads/body_sql.rs | 12 + .../main/function_reads/deps.rs | 12 + .../main/function_reads/destination.rs | 12 + .../main/function_reads/entry_point.rs | 12 + .../function_reads/fingerprint_destination.rs | 12 + .../compiled_views/main/function_reads/key.rs | 12 + .../main/function_reads/language.rs | 12 + .../compiled_views/main/function_reads/mod.rs | 15 + .../main/function_reads/name.rs | 12 + .../main/function_reads/packages.rs | 12 + .../main/function_reads/relative_path.rs | 12 + .../main/function_reads/return_columns.rs | 12 + .../main/function_reads/returns.rs | 12 + .../main/function_reads/runtime_version.rs | 12 + .../main/function_reads/source_file_path.rs | 12 + .../main/function_views/function_record.rs | 11 + .../compiled_views/main/function_views/mod.rs | 1 + .../src/compiled_views/main/mod.rs | 14 + .../main/model_input_view/attached_columns.rs | 13 + .../main/model_input_view/attached_schema.rs | 13 + .../model_input_view/input_analysis_sql.rs | 12 + .../main/model_input_view/input_model_name.rs | 12 + .../model_input_view/input_reference_rows.rs | 13 + .../input_sql_macro_checks.rs | 13 + .../main/model_input_view/mod.rs | 6 + .../main/model_reads/binding_validated.rs | 12 + .../compiled_views/main/model_reads/config.rs | 12 + .../main/model_reads/declared_rows.rs | 13 + .../main/model_reads/dep_rows.rs | 12 + .../compiled_views/main/model_reads/deps.rs | 12 + .../main/model_reads/destination.rs | 12 + .../model_reads/dynamic_columns_declared.rs | 12 + .../main/model_reads/enum_columns.rs | 12 + .../main/model_reads/inferred_columns.rs | 12 + .../main/model_reads/inferred_names.rs | 12 + .../main/model_reads/inferred_rows.rs | 13 + .../main/model_reads/lineage_facts.rs | 13 + .../main/model_reads/lineage_inputs.rs | 13 + .../compiled_views/main/model_reads/mod.rs | 24 + .../main/model_reads/model_key.rs | 12 + .../main/model_reads/model_name.rs | 12 + .../main/model_reads/name_object.rs | 12 + .../main/model_reads/output_proven.rs | 12 + .../main/model_reads/proven_inputs.rs | 12 + .../main/model_reads/query_sql.rs | 12 + .../main/model_reads/references.rs | 12 + .../main/model_reads/relative_path.rs | 12 + .../main/model_reads/schema_entry.rs | 12 + .../main/model_reads/star_flags.rs | 12 + .../main/model_reads/unchecked_columns.rs | 12 + .../main/project_reads/audits.rs | 12 + .../main/project_reads/collection.rs | 12 + .../main/project_reads/functions.rs | 12 + .../compiled_views/main/project_reads/mod.rs | 7 + .../main/project_reads/models.rs | 12 + .../main/project_reads/seeds.rs | 12 + .../main/project_reads/sources.rs | 12 + .../main/project_reads/sql_tests.rs | 12 + .../main/records/compile_inputs_view.rs | 17 + .../main/records/compiled_audit_record.rs | 12 + .../main/records/compiled_function_record.rs | 12 + .../main/records/compiled_model_record.rs | 12 + .../main/records/compiled_project_facts.rs | 15 + .../main/records/compiled_seed_record.rs | 12 + .../main/records/compiled_source_record.rs | 12 + .../main/records/declaration_rows.rs | 16 + .../main/records/declaration_rows_function.rs | 11 + .../main/records/inputs_with.rs | 16 + .../src/compiled_views/main/records/mod.rs | 17 + .../main/records/plain_typed_value.rs | 12 + .../main/records/project_with.rs | 16 + .../main/records/record_model_analyses.rs | 16 + .../main/records/register_records.rs | 12 + .../compiled_views/main/records/replace.rs | 15 + .../main/records/with_binding_diagnostics.rs | 15 + .../main/records/with_type_recovery.rs | 17 + .../relation_keys/entry_relation_key_row.rs | 19 + .../compiled_views/main/relation_keys/mod.rs | 7 + .../main/relation_keys/model_key_row.rs | 13 + .../main/relation_keys/one_row_seed_names.rs | 12 + .../relation_keys/project_relation_keys.rs | 13 + .../relation_keys/register_relation_keys.rs | 12 + .../main/relation_keys/relation_keys.rs | 20 + .../main/relation_keys/source_key_input.rs | 12 + .../main/resource_reads/destination.rs | 12 + .../main/resource_reads/graph_row.rs | 13 + .../compiled_views/main/resource_reads/key.rs | 12 + .../compiled_views/main/resource_reads/mod.rs | 6 + .../main/resource_reads/name.rs | 12 + .../main/resource_reads/references.rs | 12 + .../main/resource_reads/schema_entry.rs | 12 + .../main/seed_reads/destination.rs | 12 + .../main/seed_reads/external.rs | 12 + .../src/compiled_views/main/seed_reads/key.rs | 12 + .../src/compiled_views/main/seed_reads/mod.rs | 6 + .../compiled_views/main/seed_reads/name.rs | 12 + .../main/seed_reads/schema_entry.rs | 12 + .../main/seed_reads/seed_file.rs | 12 + .../src/compiled_views/main/seed_views/mod.rs | 1 + .../main/seed_views/paired_seeds.rs | 18 + .../compiled_views/main/source_reads/key.rs | 12 + .../compiled_views/main/source_reads/mod.rs | 4 + .../compiled_views/main/source_reads/name.rs | 12 + .../main/source_reads/source_entry.rs | 12 + .../main/source_reads/source_file.rs | 12 + .../compiled_views/main/source_views/mod.rs | 3 + .../main/source_views/read_source_view.rs | 17 + .../main/source_views/replaced.rs | 19 + .../main/source_views/retain_sources.rs | 16 + .../src/compiled_views/mod.rs | 7 + .../src/compiled_views/models.rs | 2252 +++++++++++++++++ .../src/compiled_views/types.rs | 68 + .../_helpers}/contract_diagnostics.rs | 13 +- .../contracts/_helpers}/exception_types.rs | 0 .../src/contracts/_helpers/mod.rs | 3 + .../contracts/_helpers}/model_contracts.rs | 2 +- .../main/evaluate_model_contracts.rs | 18 + .../src/contracts/main/mod.rs | 4 + .../main/promotion_conflict_diagnostics.rs | 24 + .../main/register_contract_diagnostics.rs | 12 + .../main/register_exception_types.rs | 12 + .../src/contracts/mod.rs | 4 + .../src/fingerprints/_helpers}/latest_rows.rs | 0 .../src/fingerprints/_helpers/mod.rs | 4 + .../fingerprints/_helpers}/query_hashes.rs | 0 .../src/fingerprints/_helpers}/storage_io.rs | 10 +- .../src/fingerprints/_helpers}/storage_sql.rs | 2 +- .../fingerprints/main/compute_query_hash.rs | 15 + .../src/fingerprints/main/mod.rs | 7 + .../main/read_latest_fingerprints.rs | 15 + .../fingerprints/main/register_latest_rows.rs | 12 + .../main/register_query_hashes.rs | 12 + .../fingerprints/main/register_storage_io.rs | 12 + .../src/fingerprints/main/row_fingerprint.rs | 17 + .../main/select_latest_fingerprints.rs | 18 + .../src/fingerprints/mod.rs | 4 + .../src/graph/_helpers}/algorithms.rs | 2 +- .../src/graph/_helpers}/execution_gates.rs | 0 .../src/graph/_helpers}/getters.rs | 12 +- .../src/graph/_helpers/keys.rs | 12 + .../src/graph/_helpers/mod.rs | 9 + .../src/graph/_helpers/project_graph.rs | 76 + .../src/graph/_helpers}/python_lifecycle.rs | 0 .../src/graph/_helpers}/python_node_graph.rs | 4 +- .../src/graph/_helpers}/python_selection.rs | 110 +- .../src/graph/_helpers}/views.rs | 15 +- .../src/graph/constants.rs | 12 + .../src/graph/main/algorithms/closure.rs | 17 + .../src/graph/main/algorithms/inverted.rs | 13 + .../src/graph/main/algorithms/mod.rs | 4 + .../main/algorithms/register_algorithms.rs | 12 + .../algorithms/transitive_closure_many.rs | 18 + .../src/graph/main/execution_gates/mod.rs | 3 + .../model_python_hook_names.rs | 17 + .../register_execution_gates.rs | 12 + .../graph/main/execution_gates/sql_ref_key.rs | 16 + .../src/graph/main/keys/mod.rs | 1 + .../src/graph/main/keys/native_key.rs | 12 + .../src/graph/main/mod.rs | 8 + .../src/graph/main/project_graph/mod.rs | 2 + .../graph/main/project_graph/project_graph.rs | 12 + .../project_graph/register_project_graph.rs | 12 + .../build_python_sql_run_lifecycle.rs | 15 + .../src/graph/main/python_lifecycle/mod.rs | 2 + .../register_python_lifecycle.rs | 12 + .../src/graph/main/python_node_graph/mod.rs | 3 + .../python_node_graph/python_node_graph.rs | 14 + .../register_python_node_graph.rs | 12 + .../run_selection_from_inputs.rs | 20 + .../src/graph/main/python_selection/mod.rs | 3 + .../python_selection/python_node_names.rs | 16 + .../register_python_selection.rs | 12 + .../resolve_python_check_excludes.rs | 16 + .../graph/main/views/compiled_graph_keys.rs | 17 + .../src/graph/main/views/edge_dict.rs | 17 + .../main/views/graph_from_view_mappings.rs | 14 + .../src/graph/main/views/mod.rs | 6 + .../main/views/native_graph_from_edges.rs | 16 + .../graph/main/views/project_lineage_views.rs | 16 + .../src/graph/main/views/register_views.rs | 12 + .../sqlbuild-python-project/src/graph/mod.rs | 6 + .../src/graph/models.rs | 496 ++++ crates/sqlbuild-python-project/src/lib.rs | 25 + .../src/lineage/_helpers}/lineage_facts.rs | 0 .../src/lineage/_helpers/mod.rs | 1 + .../src/lineage/main/confidence_member.rs | 13 + .../src/lineage/main/facts_eq.rs | 19 + .../src/lineage/main/facts_getitem.rs | 19 + .../src/lineage/main/facts_iter.rs | 19 + .../src/lineage/main/facts_len.rs | 19 + .../src/lineage/main/facts_projected.rs | 13 + .../src/lineage/main/facts_resource_name.rs | 19 + .../src/lineage/main/mod.rs | 8 + .../src/lineage/main/transform_kind_member.rs | 13 + .../src/lineage/mod.rs | 4 + .../src/native_host/_helpers/mod.rs | 1 + .../_helpers}/orchestration_triggers.rs | 10 +- .../src/native_host/main/compile_calendars.rs | 22 + .../src/native_host/main/compile_sensors.rs | 22 + .../src/native_host/main/mod.rs | 2 + .../src/native_host/mod.rs | 5 + .../src/native_host/models.rs} | 41 +- .../src/planner_identity/_helpers}/graph.rs | 40 +- .../src/planner_identity/_helpers}/hashing.rs | 0 .../src/planner_identity/_helpers/mod.rs | 2 + .../baselined_function_definition_hash.rs | 23 + .../main/baselined_model_query_hash.rs | 18 + .../planner_identity/main/execution_deps.rs | 13 + .../planner_identity/main/execution_order.rs | 16 + .../main/execution_origins.rs | 15 + .../main/function_definition_hash.rs | 15 + .../src/planner_identity/main/graph_key.rs | 13 + .../src/planner_identity/main/mod.rs | 10 + .../main/model_definition_hash.rs | 16 + .../planner_identity/main/register_graph.rs | 12 + .../planner_identity/main/register_hashing.rs | 12 + .../src/planner_identity/mod.rs | 4 + .../src/planner_phases/_helpers/mod.rs | 1 + .../src/planner_phases/_helpers}/objects.rs | 11 +- .../src/planner_phases/main/by_name.rs | 15 + .../src/planner_phases/main/config_str.rs | 12 + .../src/planner_phases/main/construct.rs | 18 + .../src/planner_phases/main/frozenset.rs | 13 + .../src/planner_phases/main/member.rs | 18 + .../src/planner_phases/main/mod.rs | 17 + .../src/planner_phases/main/model_key.rs | 13 + .../src/planner_phases/main/native.rs | 18 + .../src/planner_phases/main/native_member.rs | 18 + .../src/planner_phases/main/object.rs | 19 + .../src/planner_phases/main/optional_text.rs | 12 + .../src/planner_phases/main/planner_error.rs | 15 + .../src/planner_phases/main/planner_model.rs | 18 + .../src/planner_phases/main/replace.rs | 15 + .../src/planner_phases/main/sorted_names.rs | 13 + .../src/planner_phases/main/strings.rs | 12 + .../src/planner_phases/main/text.rs | 12 + .../src/planner_phases/main/tuple.rs | 13 + .../src/planner_phases/mod.rs | 4 + .../src/planner_scope/_helpers}/loaders.rs | 34 +- .../src/planner_scope/_helpers/mod.rs | 5 + .../src/planner_scope/_helpers/retained.rs | 94 + .../src/planner_scope/_helpers}/scope.rs | 38 +- .../src/planner_scope/_helpers}/sources.rs | 33 +- .../src/planner_scope/_helpers}/values.rs | 10 +- .../main/build_intermediate_source_map.rs | 16 + .../build_intermediate_source_map_function.rs | 11 + .../planner_scope/main/build_planner_scope.rs | 17 + .../main/build_source_load_entries.rs | 19 + .../build_source_load_entries_function.rs | 11 + .../main/build_source_load_map_function.rs | 11 + .../build_upstream_intermediate_source_map.rs | 16 + .../src/planner_scope/main/edges.rs | 13 + ...d_selected_loader_dependencies_function.rs | 11 + .../src/planner_scope/main/keys.rs | 13 + .../src/planner_scope/main/mod.rs | 16 + .../src/planner_scope/main/project_graph.rs | 13 + .../src/planner_scope/main/project_scope.rs | 12 + .../src/planner_scope/main/scope_data.rs | 14 + .../src/planner_scope/main/source_load_map.rs | 18 + .../main/upstream_loader_dependency_names.rs | 16 + ...stream_loader_dependency_names_function.rs | 11 + .../src/planner_scope/mod.rs | 5 + .../src/planner_scope/models.rs | 131 + .../src/planner_selection/_helpers/carrier.rs | 28 + .../src/planner_selection/_helpers/mod.rs | 4 + .../_helpers}/model_selectors.rs | 12 +- .../planner_selection/_helpers}/selectors.rs | 15 +- .../planner_selection/_helpers}/unit_tests.rs | 5 +- .../main/expand_project_upstream_keys.rs | 16 + .../main/match_project_selector.rs | 18 + .../src/planner_selection/main/mod.rs | 15 + .../main/parse_project_selector.rs | 13 + .../src/planner_selection/main/py_strings.rs | 13 + .../main/register_model_selectors.rs | 12 + .../main/reject_unit_test_selectors.rs | 16 + .../main/resolve_graph_keys.rs | 17 + .../main/resolve_graph_selection.rs | 19 + .../main/resolve_project_selector_tokens.rs | 17 + .../main/resolve_project_selectors.rs | 17 + .../resolve_sqlbuild_model_selector_names.rs | 17 + .../main/select_project_graph.rs | 17 + .../main/split_project_unit_test_selectors.rs | 22 + .../main/split_selector_expansion.rs | 13 + .../main/split_unit_test_selectors.rs | 16 + .../src/planner_selection/mod.rs | 5 + .../src/planner_selection/models.rs} | 76 +- .../_helpers}/fixture_completion.rs | 37 +- .../_helpers}/fixture_context.rs | 59 +- .../_helpers}/fixture_metadata.rs | 14 +- .../_helpers}/fixture_preparation.rs | 9 +- .../_helpers}/fixture_reads.rs | 28 +- .../_helpers}/fixture_selection.rs | 10 +- .../planner_strategy/_helpers}/fixture_sql.rs | 2 +- .../_helpers}/fixture_types.rs | 15 +- .../_helpers}/fixture_values.rs | 0 .../src/planner_strategy/_helpers/mod.rs | 10 + .../src/planner_strategy/_helpers}/types.rs | 3 - .../src/planner_strategy/constants.rs | 49 + .../main/complete_empty_fixture_sql.rs | 17 + .../main/complete_fixtures.rs | 18 + .../main/complete_typed_null_columns.rs | 17 + .../main/count_chain_resolution.rs | 11 + .../src/planner_strategy/main/mod.rs | 8 + .../planner_strategy/main/planning_context.rs | 17 + .../main/register_fixture_completion.rs | 12 + .../main/register_fixture_context.rs | 12 + .../main/register_fixture_reads.rs | 12 + .../src/planner_strategy/mod.rs | 7 + .../src/planner_strategy/models.rs | 111 + .../src/planner_strategy/types.rs | 4 + .../_helpers}/explicit_references.rs | 24 +- .../src/references/_helpers/mod.rs | 4 + .../references/_helpers}/reference_facts.rs | 0 .../src/references/_helpers}/relations.rs | 5 +- .../references/_helpers}/source_relations.rs | 0 .../main/explicit_reference_report.rs | 18 + .../main/hard_coded_relation_remedy.rs | 16 + .../references/main/match_project_relation.rs | 17 + .../src/references/main/mod.rs | 10 + .../main/native_execution_gate_edges.rs | 16 + .../main/old_name_reference_message.rs | 14 + .../main/register_explicit_references.rs | 12 + .../src/references/main/register_relations.rs | 12 + .../main/register_source_relations.rs | 12 + .../src/references/main/render_source_read.rs | 15 + .../references/main/render_source_relation.rs | 15 + .../src/references/mod.rs | 5 + .../src/references/models.rs | 22 + .../src/relation_naming/_helpers}/fitting.rs | 0 .../src/relation_naming/_helpers/mod.rs | 1 + .../main/fitted_logical_name.rs | 16 + .../src/relation_naming/main/mod.rs | 3 + .../relation_naming/main/register_fitting.rs | 12 + .../src/relation_naming/main/shared_error.rs | 11 + .../src/relation_naming/mod.rs | 4 + .../src/semantic_shapes/_helpers}/coverage.rs | 7 +- .../semantic_shapes/_helpers}/declarations.rs | 0 .../semantic_shapes/_helpers}/delta_shapes.rs | 0 .../src/semantic_shapes/_helpers/mod.rs | 5 + .../src/semantic_shapes/_helpers}/shapes.rs | 38 +- .../_helpers}/source_bindings.rs | 30 +- .../src/semantic_shapes/constants.rs | 10 + .../semantic_shapes/main/declared_types.rs | 15 + .../semantic_shapes/main/function_names.rs | 12 + .../src/semantic_shapes/main/mod.rs | 9 + .../semantic_shapes/main/project_shapes.rs | 16 + .../semantic_shapes/main/register_coverage.rs | 12 + .../semantic_shapes/main/register_shapes.rs | 12 + .../main/register_source_bindings.rs | 12 + .../semantic_shapes/main/semantic_coverage.rs | 13 + .../main/snapshot_validity_columns.rs | 12 + .../main/source_binding_diagnostics.rs | 15 + .../src/semantic_shapes/mod.rs | 5 + .../sql_test_glue/_helpers}/cursor_window.rs | 29 +- .../_helpers}/fixture_validation.rs | 19 +- .../src/sql_test_glue/_helpers/mod.rs | 12 + .../sql_test_glue/_helpers}/plan_assembly.rs | 65 +- .../sql_test_glue/_helpers}/plan_driver.rs | 70 +- .../sql_test_glue/_helpers}/plan_entries.rs | 15 +- .../sql_test_glue/_helpers}/plan_errors.rs | 9 +- .../sql_test_glue/_helpers}/plan_fixtures.rs | 14 +- .../sql_test_glue/_helpers}/plan_rendering.rs | 37 +- .../src/sql_test_glue/_helpers}/plan_rows.rs | 4 +- .../sql_test_glue/_helpers}/plan_values.rs | 6 +- .../src/sql_test_glue/_helpers}/planning.rs | 16 +- .../src/sql_test_glue/_helpers}/types.rs | 2 +- .../src/sql_test_glue/constants.rs | 43 + .../main/assemble_sql_test_plans.rs | 18 + .../src/sql_test_glue/main/mod.rs | 8 + ...an_compiled_sql_test_artifacts_function.rs | 11 + .../main/plan_sql_tests_function.rs | 11 + .../main/plan_sql_tests_natively_function.rs | 11 + .../src/sql_test_glue/main/planned_rows.rs | 18 + .../main/sql_test_model_chain_names_by_key.rs | 16 + ..._test_model_chain_names_by_key_function.rs | 11 + .../sql_test_plan_error_messages_function.rs | 11 + .../src/sql_test_glue/mod.rs | 6 + .../src/sql_test_glue/models.rs | 29 + .../src/type_system/_helpers/mod.rs | 1 + .../src/type_system/_helpers}/nullability.rs | 0 .../src/type_system/main/mod.rs | 1 + .../type_system/main/register_nullability.rs | 12 + .../src/type_system/mod.rs | 4 + crates/sqlbuild-python/Cargo.toml | 4 +- .../_helpers/analysis/binding_locations.rs | 2 +- .../src/bindings/_helpers/analysis/getters.rs | 2 +- .../src/bindings/_helpers/analysis/mod.rs | 8 - .../src/bindings/_helpers/analysis/queries.rs | 6 +- .../analysis_session/fixture_inferences.rs | 81 - .../_helpers/analysis_session/getters.rs | 2 +- .../bindings/_helpers/analysis_session/mod.rs | 8 - .../analysis_session/model_analyses.rs | 34 +- .../_helpers/analysis_session/models.rs | 10 - .../analysis_session/outcome_objects.rs | 3 +- .../_helpers/analysis_session/registration.rs | 20 +- .../_helpers/audit_command/command.rs | 3 +- .../src/bindings/_helpers/auditing/mod.rs | 1 - .../_helpers/auditing/registration.rs | 2 +- .../_helpers/build_command/command.rs | 4 +- .../_helpers/build_command/defer_clone.rs | 7 +- .../_helpers/build_command/execution.rs | 2 +- .../_helpers/build_command/invocation.rs | 2 +- .../_helpers/build_command/lifecycle.rs | 6 +- .../build_progress/build_callbacks.rs | 2 +- .../change_selection/definition_snapshot.rs | 61 +- .../_helpers/change_selection/selection.rs | 24 +- .../bindings/_helpers/check/check_command.rs | 16 +- .../src/bindings/_helpers/check/check_core.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../bindings/_helpers/clone/clone_command.rs | 2 +- .../compile_attachment/declaration_scope.rs | 2 +- .../_helpers/compile_attachment/mod.rs | 1 - .../compile_attachment/model_inputs.rs | 4 +- .../compile_attachment/registration.rs | 8 +- .../_helpers/compile_command/report.rs | 2 +- .../_helpers/compile_command/target_writer.rs | 10 +- .../_helpers/compile_delta/report_hooks.rs | 2 +- .../_helpers/compile_driver/analysis.rs | 6 +- .../_helpers/compile_driver/assembly.rs | 6 +- .../compile_driver/assembly_analysis.rs | 21 +- .../_helpers/compile_driver/assembly_facts.rs | 9 +- .../compile_driver/assembly_models.rs | 10 +- .../compile_driver/assembly_project.rs | 2 +- .../bindings/_helpers/compile_driver/diff.rs | 9 +- .../_helpers/compile_driver/inputs.rs | 5 +- .../_helpers/compile_driver/phases.rs | 2 +- .../compile_driver/pipeline_entries.rs | 3 +- .../_helpers/compile_driver/plan_result.rs | 23 +- .../_helpers/compile_driver/project.rs | 4 +- .../compile_driver/python_plan_entries.rs | 4 +- .../compile_driver/relation_targets.rs | 2 +- .../compile_driver/selection_phases.rs | 5 +- .../_helpers/compile_driver/static_context.rs | 2 +- .../compile_driver/target_defaults.rs | 8 +- .../compiled_project/attachment_inputs.rs | 298 --- .../bindings/_helpers/compiled_project/mod.rs | 15 - .../_helpers/compiled_project/project.rs | 664 ----- .../_helpers/compiled_project/registration.rs | 8 +- .../_helpers/compiled_project/snapshot.rs | 107 - .../compiled_project/snapshot_execution.rs | 750 ------ .../compiled_project/snapshot_worker.rs | 217 -- .../compiled_views/compiled_model_view.rs | 942 ------- .../_helpers/compiled_views/inputs_view.rs | 374 --- .../bindings/_helpers/compiled_views/mod.rs | 20 - .../_helpers/compiled_views/project_view.rs | 612 ----- .../_helpers/compiled_views/registration.rs | 4 +- .../_helpers/compiled_views/seed_views.rs | 350 --- .../_helpers/compiled_views/source_views.rs | 333 --- .../bindings/_helpers/consumption/facts.rs | 77 +- .../bindings/_helpers/consumption/policy.rs | 4 +- .../_helpers/consumption/relation_reads.rs | 5 +- .../_helpers/contract_command/command.rs | 4 +- .../src/bindings/_helpers/contracts/mod.rs | 3 - .../_helpers/contracts/registration.rs | 7 +- .../_helpers/dag_artifact/artifact.rs | 2 +- .../_helpers/dag_artifact/selection.rs | 2 +- .../bindings/_helpers/dbt_interop/graph.rs | 5 +- .../bindings/_helpers/diff_report/command.rs | 2 +- .../_helpers/exported_constants/constants.rs | 3 +- .../src/bindings/_helpers/fingerprints/mod.rs | 4 - .../_helpers/fingerprints/registration.rs | 10 +- .../_helpers/format_command/selection.rs | 4 +- .../_helpers/freshness/freshness_command.rs | 9 +- .../_helpers/freshness/observation.rs | 2 +- .../bindings/_helpers/freshness/planning.rs | 2 +- .../src/bindings/_helpers/graph/keys.rs | 61 - .../src/bindings/_helpers/graph/mod.rs | 10 - .../src/bindings/_helpers/graph/node_set.rs | 25 - .../bindings/_helpers/graph/project_graph.rs | 340 --- .../bindings/_helpers/graph/registration.rs | 16 +- .../_helpers/hand_built/compiled_model.rs | 2 +- .../_helpers/input_loops/attachment.rs | 26 +- .../_helpers/input_loops/audit_loop.rs | 9 +- .../_helpers/input_loops/function_loop.rs | 12 +- .../_helpers/input_loops/loop_classes.rs | 5 +- .../_helpers/input_loops/model_loop.rs | 34 +- .../_helpers/input_loops/model_loop_header.rs | 2 +- .../_helpers/input_loops/model_loop_host.rs | 21 +- .../input_loops/model_loop_opt_out.rs | 2 +- .../_helpers/input_loops/model_loop_schema.rs | 2 +- .../_helpers/input_loops/registration.rs | 4 +- .../input_loops/resource_attachment.rs | 30 +- .../input_loops/resource_diagnostics.rs | 19 +- .../_helpers/input_loops/scenario_loop.rs | 2 +- .../_helpers/input_loops/source_loop.rs | 9 +- .../_helpers/input_loops/sql_test_loop.rs | 9 +- .../janitor/janitor_classification.rs | 2 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/janitor/janitor_facts.rs | 2 +- .../_helpers/janitor/janitor_old_names.rs | 9 +- .../_helpers/janitor/janitor_planning.rs | 2 +- .../_helpers/lineage/column_lineage_graph.rs | 3 +- .../_helpers/lineage/lineage_command.rs | 13 +- .../src/bindings/_helpers/lineage/mod.rs | 1 - .../_helpers/lineage/project_lineage.rs | 6 +- .../_helpers/lineage/project_schema.rs | 3 +- .../bindings/_helpers/load/load_command.rs | 4 +- .../_helpers/location_command/command.rs | 6 +- .../_helpers/location_command/snapshot.rs | 23 +- .../location_command/snapshot_providers.rs | 2 +- .../_helpers/location_command/worker.rs | 11 +- .../src/bindings/_helpers/manifest/builder.rs | 4 +- .../src/bindings/_helpers/native_host/mod.rs | 2 - .../native_host/orchestration_sensors.rs | 2 +- .../native_host/warehouse_metadata.rs | 4 +- .../_helpers/observations/publication.rs | 24 +- .../bindings/_helpers/plan_command/rules.rs | 4 +- .../_helpers/planner_changes/actions.rs | 2 +- .../_helpers/planner_changes/detect.rs | 62 +- .../_helpers/planner_changes/renames.rs | 23 +- .../_helpers/planner_entries/clone_plans.rs | 178 +- .../_helpers/planner_entries/command_plans.rs | 75 +- .../_helpers/planner_entries/display_plan.rs | 27 +- .../planner_entries/execution_plan.rs | 2 +- .../_helpers/planner_entries/model_entries.rs | 37 +- .../_helpers/planner_entries/models.rs | 2 +- .../_helpers/planner_entries/plan_output.rs | 89 +- .../_helpers/planner_entries/relations.rs | 33 +- .../_helpers/planner_identity/identities.rs | 53 +- .../_helpers/planner_identity/metadata.rs | 115 +- .../bindings/_helpers/planner_identity/mod.rs | 2 - .../_helpers/planner_identity/registration.rs | 10 +- .../_helpers/planner_identity/staleness.rs | 4 +- .../planner_migrations/column_candidates.rs | 12 +- .../planner_migrations/column_entries.rs | 24 +- .../planner_migrations/column_objects.rs | 11 +- .../planner_migrations/column_overlay.rs | 16 +- .../planner_migrations/column_planning.rs | 10 +- .../planner_migrations/compatibility.rs | 11 +- .../_helpers/planner_migrations/discovery.rs | 6 +- .../planner_migrations/discovery_events.rs | 9 +- .../planner_migrations/discovery_models.rs | 31 +- .../planner_migrations/discovery_requests.rs | 21 +- .../planner_migrations/fingerprint.rs | 6 +- .../_helpers/planner_migrations/matching.rs | 12 +- .../planner_migrations/model_entries.rs | 11 +- .../planner_migrations/model_handover.rs | 22 +- .../planner_migrations/model_planning.rs | 16 +- .../planner_migrations/old_name_entries.rs | 9 +- .../planner_migrations/old_name_objects.rs | 9 +- .../planner_migrations/old_name_planning.rs | 24 +- .../planner_migrations/old_name_references.rs | 8 +- .../_helpers/planner_migrations/old_names.rs | 25 +- .../planner_migrations/planning_origin.rs | 35 +- .../planner_migrations/state_inspection.rs | 2 +- .../_helpers/planner_phases/buildability.rs | 19 +- .../_helpers/planner_phases/contexts.rs | 2 +- .../_helpers/planner_phases/full_refresh.rs | 15 +- .../_helpers/planner_phases/getters.rs | 2 +- .../_helpers/planner_phases/identities.rs | 11 +- .../bindings/_helpers/planner_phases/mod.rs | 1 - .../planner_phases/output_assembly.rs | 17 +- .../_helpers/planner_phases/registration.rs | 30 +- .../_helpers/planner_phases/retention.rs | 75 +- .../_helpers/planner_phases/scopes.rs | 4 +- .../planner_phases/warehouse_state.rs | 6 +- .../planner_resolution/cursor_model.rs | 25 +- .../planner_resolution/cursor_overrides.rs | 4 +- .../planner_resolution/cursor_ranges.rs | 9 +- .../_helpers/planner_resolution/locations.rs | 8 +- .../planner_resolution/maximum_safety.rs | 2 +- .../planner_resolution/orchestration.rs | 25 +- .../_helpers/planner_resolution/sources.rs | 2 +- .../_helpers/planner_scenario/artifacts.rs | 2 +- .../bindings/_helpers/planner_scenario/cli.rs | 14 +- .../_helpers/planner_scenario/entries.rs | 22 +- .../_helpers/planner_scenario/execution.rs | 58 +- .../_helpers/planner_scenario/graph.rs | 35 +- .../_helpers/planner_scenario/relations.rs | 13 +- .../bindings/_helpers/planner_scenario/sql.rs | 4 +- .../bindings/_helpers/planner_scope/mod.rs | 5 - .../_helpers/planner_scope/registration.rs | 29 +- .../_helpers/planner_scope/retained.rs | 199 -- .../_helpers/planner_selection/mod.rs | 4 - .../planner_selection/registration.rs | 4 +- .../planner_strategy/audit_entries.rs | 105 +- .../_helpers/planner_strategy/audits.rs | 27 +- .../_helpers/planner_strategy/constants.rs | 29 - .../planner_strategy/cursor_inputs.rs | 22 +- .../_helpers/planner_strategy/lineage.rs | 18 +- .../planner_strategy/microbatch_limits.rs | 2 +- .../bindings/_helpers/planner_strategy/mod.rs | 12 - .../planner_strategy/model_entries.rs | 71 +- .../_helpers/planner_strategy/models.rs | 110 - .../_helpers/planner_strategy/registration.rs | 25 +- .../_helpers/planner_strategy/strategy.rs | 5 +- .../_helpers/planner_warehouse/cursors.rs | 53 +- .../planner_warehouse/fingerprint_cache.rs | 2 +- .../planner_warehouse/semantic_sources.rs | 14 +- .../_helpers/planner_warehouse/snapshot.rs | 93 +- .../planner_warehouse/source_columns.rs | 12 +- .../planner_warehouse/source_deferral.rs | 68 +- .../planner_warehouse/source_freshness.rs | 15 +- .../planner_warehouse/source_tables.rs | 42 +- .../_helpers/planner_warehouse/watermarks.rs | 18 +- .../project_assembly/declared_relations.rs | 13 +- .../project_assembly/native_resources.rs | 2 +- .../project_assembly/syntax_checks.rs | 2 +- .../bindings/_helpers/query_command/dag.rs | 2 +- .../src/bindings/_helpers/recall/session.rs | 2 +- .../_helpers/record_behaviours/compiled.rs | 2 +- .../_helpers/record_behaviours/constants.rs | 19 +- .../_helpers/record_behaviours/macros.rs | 3 +- .../record_behaviours/python_nodes.rs | 2 +- .../src/bindings/_helpers/references/mod.rs | 4 - .../_helpers/references/registration.rs | 6 +- .../bindings/_helpers/relation_naming/mod.rs | 1 - .../_helpers/relation_naming/registration.rs | 4 +- .../_helpers/review_command/comparison.rs | 6 +- .../_helpers/review_command/impact.rs | 12 +- .../_helpers/review_command/planning.rs | 40 +- .../src/bindings/_helpers/rules/catalogue.rs | 2 +- .../_helpers/rules/custom_host_payload.rs | 11 +- .../_helpers/rules/custom_rule_cache.rs | 2 +- .../_helpers/rules/early_builtin_rules.rs | 7 +- .../bindings/_helpers/rules/early_sql_lint.rs | 10 +- .../bindings/_helpers/rules/fact_replay.rs | 12 +- .../bindings/_helpers/rules/fix_session.rs | 5 +- .../src/bindings/_helpers/rules/lint.rs | 6 +- .../bindings/_helpers/rules/lint_inputs.rs | 7 +- .../src/bindings/_helpers/rules/lint_text.rs | 2 +- .../src/bindings/_helpers/rules/rows.rs | 10 +- .../bindings/_helpers/rules/rule_dispatch.rs | 2 +- .../src/bindings/_helpers/rules/rule_plan.rs | 2 +- .../src/bindings/_helpers/rules/sql_rules.rs | 10 +- .../_helpers/rules/sql_test_objects.rs | 4 +- .../_helpers/scenario_command/setup.rs | 2 +- .../bindings/_helpers/scopes/scope_loading.rs | 2 +- .../bindings/_helpers/seed_command/command.rs | 3 +- .../_helpers/semantic_checks/completion.rs | 9 +- .../_helpers/semantic_checks/inputs.rs | 14 +- .../_helpers/semantic_checks/metadata.rs | 6 +- .../_helpers/semantic_checks/resource_sql.rs | 6 +- .../_helpers/semantic_checks/session_facts.rs | 2 +- .../_helpers/semantic_checks/stage.rs | 6 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../_helpers/semantic_checks/type_recovery.rs | 13 +- .../_helpers/semantic_shapes/constants.rs | 4 - .../bindings/_helpers/semantic_shapes/mod.rs | 6 - .../_helpers/semantic_shapes/registration.rs | 10 +- .../_helpers/sql_test_command/python_tests.rs | 9 +- .../_helpers/sql_test_command/test_command.rs | 3 +- .../_helpers/sql_test_glue/assembly.rs | 2 +- .../_helpers/sql_test_glue/compiled_tests.rs | 6 +- .../_helpers/sql_test_glue/constants.rs | 27 - .../bindings/_helpers/sql_test_glue/mod.rs | 14 - .../bindings/_helpers/sql_test_glue/models.rs | 20 - .../_helpers/sql_test_glue/registration.rs | 21 +- .../src/bindings/_helpers/type_system/mod.rs | 1 - .../_helpers/type_system/registration.rs | 4 +- .../src/bindings/main/register.rs | 14 +- crates/sqlbuild-python/src/bindings/mod.rs | 1 - fensu.toml | 254 +- scripts/fensu_policy/constants.py | 1 + scripts/native_module_paths/constants.py | 1 + 826 files changed, 15494 insertions(+), 9022 deletions(-) create mode 100644 crates/sqlbuild-python-project/Cargo.toml create mode 100644 crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/catalog.rs (75%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/conversions.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/models.rs => sqlbuild-python-project/src/analysis/_helpers/native_classes.rs} (51%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/normalization.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/normalization_results.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/positions.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/relation_names.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-project/src/analysis/_helpers}/schema_request.rs (80%) create mode 100644 crates/sqlbuild-python-project/src/analysis/constants.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs => sqlbuild-python-project/src/analysis/models.rs} (55%) create mode 100644 crates/sqlbuild-python-project/src/analysis/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/fixture_columns.rs (95%) create mode 100644 crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/nullability_rules.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/profile_inputs.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/query_columns.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/session.rs (53%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-project/src/analysis_session/_helpers}/stage_functions.rs (91%) create mode 100644 crates/sqlbuild-python-project/src/analysis_session/constants.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/mod.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/models.rs create mode 100644 crates/sqlbuild-python-project/src/analysis_session/types.rs create mode 100644 crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/auditing => sqlbuild-python-project/src/auditing/_helpers}/render.rs (95%) create mode 100644 crates/sqlbuild-python-project/src/auditing/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/auditing/main/register_render.rs create mode 100644 crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs create mode 100644 crates/sqlbuild-python-project/src/auditing/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-project/src/compile_attachment/_helpers}/cursor_roles.rs (90%) create mode 100644 crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs create mode 100644 crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs create mode 100644 crates/sqlbuild-python-project/src/compile_attachment/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/job_catalog.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/rows.rs (56%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/schema_facts.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/snapshot_boundary.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/snapshot_declarations.rs (98%) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/snapshot_models.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/snapshot_resources.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/snapshot_roles.rs (95%) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_project => sqlbuild-python-project/src/compiled_project/_helpers}/values.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-project/src}/compiled_project/constants.rs (78%) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/models.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/audit_reads.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/audit_views.rs (77%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/compiled_model_analysis.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/function_reads.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/function_views.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/getters.rs (91%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/model_input_view.rs (55%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/model_reads.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/project_host.rs (65%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/project_reads.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/records.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/relation_keys.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/resource_reads.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/seed_reads.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compiled_views => sqlbuild-python-project/src/compiled_views/_helpers}/source_reads.rs (96%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/constants.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/models.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-project/src/contracts/_helpers}/contract_diagnostics.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-project/src/contracts/_helpers}/exception_types.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-project/src/contracts/_helpers}/model_contracts.rs (99%) create mode 100644 crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs create mode 100644 crates/sqlbuild-python-project/src/contracts/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs create mode 100644 crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs create mode 100644 crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs create mode 100644 crates/sqlbuild-python-project/src/contracts/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/fingerprints => sqlbuild-python-project/src/fingerprints/_helpers}/latest_rows.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/fingerprints => sqlbuild-python-project/src/fingerprints/_helpers}/query_hashes.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/fingerprints => sqlbuild-python-project/src/fingerprints/_helpers}/storage_io.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/fingerprints => sqlbuild-python-project/src/fingerprints/_helpers}/storage_sql.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/algorithms.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/execution_gates.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/getters.rs (75%) create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/keys.rs create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/python_lifecycle.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/python_node_graph.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/python_selection.rs (72%) rename crates/{sqlbuild-python/src/bindings/_helpers/graph => sqlbuild-python-project/src/graph/_helpers}/views.rs (93%) create mode 100644 crates/sqlbuild-python-project/src/graph/constants.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/keys/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/views/register_views.rs create mode 100644 crates/sqlbuild-python-project/src/graph/mod.rs create mode 100644 crates/sqlbuild-python-project/src/graph/models.rs create mode 100644 crates/sqlbuild-python-project/src/lib.rs rename crates/{sqlbuild-python/src/bindings/_helpers/lineage => sqlbuild-python-project/src/lineage/_helpers}/lineage_facts.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_len.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs create mode 100644 crates/sqlbuild-python-project/src/lineage/mod.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-project/src/native_host/_helpers}/orchestration_triggers.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs => sqlbuild-python-project/src/native_host/models.rs} (66%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-project/src/planner_identity/_helpers}/graph.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-project/src/planner_identity/_helpers}/hashing.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs create mode 100644 crates/sqlbuild-python-project/src/planner_identity/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-project/src/planner_phases/_helpers}/objects.rs (94%) create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/construct.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/member.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/native.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/object.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/replace.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/strings.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/text.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs create mode 100644 crates/sqlbuild-python-project/src/planner_phases/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scope => sqlbuild-python-project/src/planner_scope/_helpers}/loaders.rs (77%) create mode 100644 crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scope => sqlbuild-python-project/src/planner_scope/_helpers}/scope.rs (79%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scope => sqlbuild-python-project/src/planner_scope/_helpers}/sources.rs (78%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scope => sqlbuild-python-project/src/planner_scope/_helpers}/values.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/edges.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/keys.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_scope/models.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_selection => sqlbuild-python-project/src/planner_selection/_helpers}/model_selectors.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_selection => sqlbuild-python-project/src/planner_selection/_helpers}/selectors.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_selection => sqlbuild-python-project/src/planner_selection/_helpers}/unit_tests.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs create mode 100644 crates/sqlbuild-python-project/src/planner_selection/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs => sqlbuild-python-project/src/planner_selection/models.rs} (68%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_completion.rs (86%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_context.rs (82%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_metadata.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_preparation.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_reads.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_selection.rs (83%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_sql.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_types.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/fixture_values.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-project/src/planner_strategy/_helpers}/types.rs (68%) create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/constants.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/mod.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/models.rs create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/references => sqlbuild-python-project/src/references/_helpers}/explicit_references.rs (96%) create mode 100644 crates/sqlbuild-python-project/src/references/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/references => sqlbuild-python-project/src/references/_helpers}/reference_facts.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/references => sqlbuild-python-project/src/references/_helpers}/relations.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/references => sqlbuild-python-project/src/references/_helpers}/source_relations.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/match_project_relation.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/register_relations.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/register_source_relations.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/render_source_read.rs create mode 100644 crates/sqlbuild-python-project/src/references/main/render_source_relation.rs create mode 100644 crates/sqlbuild-python-project/src/references/mod.rs create mode 100644 crates/sqlbuild-python-project/src/references/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/relation_naming => sqlbuild-python-project/src/relation_naming/_helpers}/fitting.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs create mode 100644 crates/sqlbuild-python-project/src/relation_naming/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs create mode 100644 crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs create mode 100644 crates/sqlbuild-python-project/src/relation_naming/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_shapes => sqlbuild-python-project/src/semantic_shapes/_helpers}/coverage.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_shapes => sqlbuild-python-project/src/semantic_shapes/_helpers}/declarations.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_shapes => sqlbuild-python-project/src/semantic_shapes/_helpers}/delta_shapes.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_shapes => sqlbuild-python-project/src/semantic_shapes/_helpers}/shapes.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_shapes => sqlbuild-python-project/src/semantic_shapes/_helpers}/source_bindings.rs (95%) create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/constants.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs create mode 100644 crates/sqlbuild-python-project/src/semantic_shapes/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/cursor_window.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/fixture_validation.rs (93%) create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_assembly.rs (71%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_driver.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_entries.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_errors.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_fixtures.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_rendering.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_rows.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/plan_values.rs (79%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/planning.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-project/src/sql_test_glue/_helpers}/types.rs (70%) create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/constants.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/mod.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/models.rs create mode 100644 crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/type_system => sqlbuild-python-project/src/type_system/_helpers}/nullability.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/type_system/main/mod.rs create mode 100644 crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs create mode 100644 crates/sqlbuild-python-project/src/type_system/mod.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs diff --git a/Cargo.lock b/Cargo.lock index 5bf2f44b80..f8b74a9c4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4436,7 +4436,6 @@ name = "sqlbuild-python" version = "0.233.0" dependencies = [ "chrono", - "chrono-tz", "fensu-policy", "pyo3", "rayon", @@ -4444,7 +4443,6 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2 0.10.9", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -4461,6 +4459,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", @@ -4469,7 +4468,6 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "sqlclean", - "tokio", "uuid", ] @@ -4527,6 +4525,36 @@ dependencies = [ "uuid", ] +[[package]] +name = "sqlbuild-python-project" +version = "0.0.0" +dependencies = [ + "chrono", + "chrono-tz", + "pyo3", + "serde_json", + "sha2 0.10.9", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-orchestration", + "sqlbuild-project", + "sqlbuild-pysource", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-rules", + "sqlbuild-sqltext", + "tokio", + "uuid", +] + [[package]] name = "sqlbuild-recall" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index f6fc2c295a..0c51f64f55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/sqlbuild-cli", "crates/sqlbuild-python-base", "crates/sqlbuild-python-front", + "crates/sqlbuild-python-project", "crates/sqlbuild-python", ] resolver = "2" @@ -59,9 +60,10 @@ order = [ "sqlbuild-cli", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -183,6 +185,7 @@ sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } +sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/sqlbuild-python-project/Cargo.toml b/crates/sqlbuild-python-project/Cargo.toml new file mode 100644 index 0000000000..dc526705d0 --- /dev/null +++ b/crates/sqlbuild-python-project/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sqlbuild-python-project" +description = "The project layer of the sqlbuild._native bindings: the compiled project and its views, graph, references, lineage, fingerprints, semantic shapes, planner scope, selection and strategy, and SQL test planning." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +chrono.workspace = true +chrono-tz.workspace = true +pyo3.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-cli.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true +sqlbuild-project.workspace = true +sqlbuild-pysource.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-sqltext.workspace = true +tokio.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs new file mode 100644 index 0000000000..d9d12ad91e --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs @@ -0,0 +1,152 @@ +//! The compile-lifetime binding catalog: known relation shapes around one native project catalog. + +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::models::BindingCatalog; +use crate::analysis::models::CatalogKey; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; + +use sqlbuild_analysis::semantic_validation::types::BindingRequest; +use sqlbuild_python_base::types::IntoPositional; + +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; + +pub(crate) const QUOTED_IDENTIFIER_DELIMITER: char = '"'; +const SNOWFLAKE_DIALECT: &str = "snowflake"; +const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; +const SESSION_PARAMETERS: &str = "session_parameters"; +const GENERIC_DIALECT: &str = "generic"; + +/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. +pub(crate) fn ignores_quoted_case( + connection: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { + return Ok(false); + } + let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; + let Ok(parameters) = parameters.downcast::() else { + return Ok(false); + }; + for (key, value) in parameters.iter() { + let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; + if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { + return Ok(true); + } + } + Ok(false) +} + +/// The native dialect a request's adapter dialect validates as. +fn native_dialect(dialect: &Bound<'_, PyAny>) -> PyResult { + let name: String = if dialect.is_truthy()? { + dialect.extract()? + } else { + GENERIC_DIALECT.to_owned() + }; + Ok(NATIVE_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == name) + .map_or(name.clone(), |(_, native)| (*native).to_owned())) +} + +/// Binding diagnostics for a batch of complete-schema validation requests, in request order. +#[pyfunction] +#[pyo3(signature = (*, requests))] +pub(crate) fn schema_validations<'py>( + py: Python<'py>, + requests: &Bound<'py, PyAny>, +) -> PyResult> { + let requests: Vec> = requests.try_iter()?.collect::>()?; + let mut created: Vec<(CatalogKey, Py)> = Vec::new(); + let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); + for (index, request) in requests.iter().enumerate() { + let key: CatalogKey = CatalogKey { + dialect: native_dialect(&native_field(request, "dialect")?)?, + quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(request, "known_functions")?.extract()?, + known_types: native_field(request, "known_types")?.extract()?, + }; + let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; + let own: bool = !explicit.is_none(); + let catalog: Py = if own { + explicit.extract()? + } else if let Some((_, catalog)) = created.iter().find(|(known, _)| *known == key) { + catalog.clone_ref(py) + } else { + let catalog: Py = Py::new( + py, + BindingCatalog::create(py, key.clone(), &native_field(request, "schema")?)?, + )?; + created.push((key, catalog.clone_ref(py))); + catalog + }; + match groups.iter_mut().find(|(known, _, _)| known.is(&catalog)) { + Some((_, any_explicit, members)) => { + *any_explicit |= own; + members.push(index); + } + None => groups.push((catalog, own, vec![index])), + } + } + let empty: Bound<'py, PyAny> = new_record(py, RecordId::SqlBindingResult, (), None)?; + let mut results: Vec> = vec![empty; requests.len()]; + for (catalog, any_explicit, members) in groups { + let catalog = catalog.borrow(py); + let batch: Vec> = members + .iter() + .map(|index| requests[*index].clone()) + .collect(); + let prepared: Vec = catalog.prepared(py, &batch)?; + let native = catalog.native.borrow(py); + let inner: &validation::ProjectCatalog = &native.inner; + let dialect: &str = &catalog.dialect; + let rows: Vec> = if any_explicit { + py.compiler_detach(|| inner.binding_results(prepared)) + } else { + py.compiler_detach(|| inner.schema_validation_rows(dialect, prepared)) + } + .map_err(compiler_error)?; + if rows.len() != members.len() { + return Err(compiler_error( + "native SQL schema validation returned an invalid batch response", + )); + } + for (index, diagnostics) in members.into_iter().zip(rows) { + let built: Vec> = diagnostics + .into_iter() + .map(|row| { + new_record( + py, + RecordId::SqlBindingDiagnostic, + row.into_positional(py)?, + None, + ) + }) + .collect::>()?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; + results[index] = new_record(py, RecordId::SqlBindingResult, (), Some(&keywords))?; + } + } + PyTuple::new(py, results) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(schema_validations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs index a56da309e0..3ea5e12ed0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs @@ -1,9 +1,14 @@ //! Python methods of the compile-owned binding catalog and its compact analysis jobs. -use crate::bindings::_helpers::analysis::conversions; -use crate::bindings::_helpers::analysis::normalization_results::normalization_results; -use crate::bindings::models::Positional; -use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; +use crate::analysis::_helpers::conversions::analysis_relations; +use crate::analysis::_helpers::conversions::binding_requests; +use crate::analysis::_helpers::conversions::columns; +use crate::analysis::_helpers::conversions::relations; +use crate::analysis::_helpers::native_classes::CatalogInput; +use crate::analysis::_helpers::normalization_results::normalization_results; +use crate::analysis::models::CompactAnalysisJob; +use crate::analysis::models::Positional; +use crate::analysis::models::ProjectCatalog; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; @@ -23,8 +28,8 @@ impl ProjectCatalog { &self, py: Python<'_>, sql: &str, - #[pyo3(from_py_with = conversions::columns)] columns: validation::Columns, - #[pyo3(from_py_with = conversions::relations)] inputs: Relations, + #[pyo3(from_py_with = columns)] columns: validation::Columns, + #[pyo3(from_py_with = relations)] inputs: Relations, ) -> PyResult)>> { py.compiler_detach(|| self.inner.inferred_schema(sql, columns, inputs)) .map_err(compiler_error) @@ -37,17 +42,11 @@ impl ProjectCatalog { .map_err(PyValueError::new_err) } - fn update_relations( - &mut self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, - ) { + fn update_relations(&mut self, #[pyo3(from_py_with = relations)] relations: Relations) { self.inner.update_relations(relations); } - fn with_relations( - &self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, - ) -> Self { + fn with_relations(&self, #[pyo3(from_py_with = relations)] relations: Relations) -> Self { Self { inner: self.inner.with_relations(relations), } @@ -55,7 +54,7 @@ impl ProjectCatalog { fn update_analysis( &mut self, - #[pyo3(from_py_with = conversions::analysis_relations)] relations: HashMap< + #[pyo3(from_py_with = analysis_relations)] relations: HashMap< String, (validation::Columns, validation::Columns), >, @@ -65,14 +64,14 @@ impl ProjectCatalog { fn register_override( &mut self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, + #[pyo3(from_py_with = relations)] relations: Relations, ) -> usize { self.inner.register_override(relations) } fn validation_payload( &self, - #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + #[pyo3(from_py_with = binding_requests)] requests: Vec, ) -> PyResult { self.inner .validation_payload(requests) @@ -91,7 +90,7 @@ impl ProjectCatalog { fn binding_results( &self, py: Python<'_>, - #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + #[pyo3(from_py_with = binding_requests)] requests: Vec, ) -> PyResult>>> { let rows: Vec> = py .compiler_detach(|| self.inner.binding_results(requests)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/conversions.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/conversions.rs diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs new file mode 100644 index 0000000000..8cf928a522 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod binding_catalog; +pub(crate) mod catalog; +pub(crate) mod conversions; +pub(crate) mod native_classes; +pub(crate) mod normalization; +pub(crate) mod normalization_results; +pub(crate) mod positions; +pub(crate) mod relation_names; +pub(crate) mod schema_request; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs similarity index 51% rename from crates/sqlbuild-python/src/bindings/models.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs index b48ee3ef23..bd172affb0 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs @@ -1,14 +1,11 @@ -//! Python-facing classes and mapping requests, converted to native models at the boundary. +//! Analysis's Python classes and the mapping requests converted to native models at the boundary. -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyErr, PyResult, Python}; -use pyo3::types::PyTuple; -use pyo3::{FromPyObject, pyclass}; +use crate::analysis::models::Positional; +use pyo3::FromPyObject; use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_analysis::semantic_validation::types::Relations; -use sqlbuild_python_base::types::FromPositional; -use sqlbuild_python_base::types::IntoPositional; use std::collections::HashMap; #[derive(FromPyObject, Debug)] @@ -18,13 +15,13 @@ pub(crate) struct CatalogInput { pub(crate) quoted_ignore_case: bool, pub(crate) known_functions: Vec, pub(crate) known_types: Vec, - #[pyo3(from_py_with = crate::bindings::_helpers::analysis::conversions::relations)] + #[pyo3(from_py_with = crate::analysis::_helpers::conversions::relations)] pub(crate) relations: Relations, } #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] -pub(crate) struct NormalizationInput { +pub struct NormalizationInput { pub(crate) sql: String, pub(crate) dialect: String, pub(crate) stubs: HashMap, @@ -41,24 +38,6 @@ pub(crate) struct PositionInput { pub(crate) passes: Vec>>, } -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct ProjectCatalog { - pub(crate) inner: validation::ProjectCatalog, -} - -/// One resolved compact analysis batch that runs without borrowing its project catalog. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CompactAnalysisJob { - pub(crate) inner: validation::CompactAnalysisJob, -} - -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct BindingPositions { - pub(crate) inner: validation::BindingPositions, -} - impl From for validation::CatalogInput { fn from(input: CatalogInput) -> Self { Self { @@ -97,35 +76,3 @@ impl From for validation::PositionInput { } } } - -/// A record of another crate exchanged with Python as its positional tuple. -#[derive(Clone, Debug)] -pub(crate) struct Positional(pub(crate) T); - -impl Positional { - /// Each record of `items`, for Python. - pub(crate) fn wrap_all(items: Vec) -> Vec { - items.into_iter().map(Self).collect() - } - - /// Each record Python sent in `items`. - pub(crate) fn unwrap_all(items: Vec) -> Vec { - items.into_iter().map(|item| item.0).collect() - } -} - -impl<'py, T: FromPositional> FromPyObject<'py> for Positional { - fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { - T::from_positional(value).map(Self) - } -} - -impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { - type Target = PyTuple; - type Output = Bound<'py, PyTuple>; - type Error = PyErr; - - fn into_pyobject(self, py: Python<'py>) -> Result { - self.0.into_positional(py) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs index 94c65584f8..d856ce81fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::analysis::normalization_results::normalization_results; -use crate::bindings::models::Positional; +use crate::analysis::_helpers::normalization_results::normalization_results; +use crate::analysis::models::Positional; use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; @@ -12,7 +12,7 @@ use sqlbuild_python_base::types::CompilerDetach; #[pyfunction] pub(crate) fn normalize_analysis_sql( py: Python<'_>, - request: crate::bindings::models::NormalizationInput, + request: crate::analysis::_helpers::native_classes::NormalizationInput, ) -> PyResult { py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization_results.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/normalization_results.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs index c3c61fcfea..e92320e42d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs @@ -1,6 +1,7 @@ //! Python methods of authored binding positions. -use crate::bindings::models::{BindingPositions, PositionInput}; +use crate::analysis::_helpers::native_classes::PositionInput; +use crate::analysis::models::BindingPositions; use pyo3::exceptions::PyValueError; use pyo3::{PyResult, pymethods}; use sqlbuild_analysis::semantic_validation::models as validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs index a5aee1a374..b53de697dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs @@ -1,5 +1,6 @@ //! Relations named in SQL and temporary tables it creates, for hard-coded name checks. +use crate::analysis::types::RelationRow; use pyo3::PyErr; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; @@ -12,9 +13,6 @@ use sqlbuild_python_base::types::CompilerDetach; const RELATION_NAMES_CONTEXT: &str = "native relation name extraction"; -/// `(name, schema, database)`. -pub(crate) type RelationRow = (String, Option, Option); - /// `(relations, temporary names)` named in `sql`, or None if unparseable. pub(crate) fn extract_sql_relation_names( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs index dd7787f690..3899eab30c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs @@ -1,5 +1,6 @@ //! Build the stable native schema-validation request of one model query. +use crate::analysis::models::SchemaRequestParts; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -14,18 +15,6 @@ use sqlbuild_python_base::types::CompilerDetach; const GENERIC_DIALECT: &str = "generic"; -/// The query, relation schemas and binding context of one schema-validation request. -pub(crate) struct SchemaRequestParts<'py> { - pub(crate) query_sql: String, - pub(crate) placeholders: Bound<'py, PyAny>, - pub(crate) dialect: Bound<'py, PyAny>, - pub(crate) schema: Bound<'py, PyAny>, - pub(crate) known_functions: Bound<'py, PyAny>, - pub(crate) known_types: Bound<'py, PyAny>, - pub(crate) catalog: Bound<'py, PyAny>, - pub(crate) quoted_identifiers_ignore_case: bool, -} - /// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. pub(crate) fn schema_validation_request<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-project/src/analysis/constants.rs b/crates/sqlbuild-python-project/src/analysis/constants.rs new file mode 100644 index 0000000000..6a064e13b7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/constants.rs @@ -0,0 +1,11 @@ +//! Public constants of analysis. + +/// Dialects whose quoted identifiers bind case-sensitively unless the session says otherwise. +pub const CASE_SENSITIVE_BINDING_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; + +/// Adapter dialect names and the native dialect each validates as. +pub const NATIVE_DIALECT_ALIASES: [(&str, &str); 3] = [ + ("postgres", "postgresql"), + ("motherduck", "duckdb"), + ("sqlserver", "tsql"), +]; diff --git a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs new file mode 100644 index 0000000000..19d2f4a596 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs @@ -0,0 +1,16 @@ +//! `(relations, temporary names)` named in `sql`, or None if unparseable. + +use crate::analysis::_helpers::relation_names; +use crate::analysis::types::RelationRow; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `(relations, temporary names)` named in `sql`, or None if unparseable. +#[inline] +pub fn extract_sql_relation_names( + py: Python<'_>, + sql: &str, + dialect: Option<&str>, +) -> PyResult, Vec)>> { + relation_names::extract_sql_relation_names(py, sql, dialect) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs new file mode 100644 index 0000000000..89f1e7795e --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs @@ -0,0 +1,15 @@ +//! `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. + +use crate::analysis::_helpers::binding_catalog; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. +#[inline] +pub fn ignores_quoted_case( + connection: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + binding_catalog::ignores_quoted_case(connection, dialect) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/mod.rs b/crates/sqlbuild-python-project/src/analysis/main/mod.rs new file mode 100644 index 0000000000..bb9d5d1db2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/mod.rs @@ -0,0 +1,7 @@ +pub mod extract_sql_relation_names; +pub mod ignores_quoted_case; +pub mod normalize_analysis_sql; +pub mod register_binding_catalog; +pub mod register_normalization; +pub mod schema_validation_request; +pub mod schema_validations; diff --git a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs new file mode 100644 index 0000000000..ca75cbea4b --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs @@ -0,0 +1,14 @@ +//! Normalize analysis sql. + +use crate::analysis::_helpers::normalization; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Normalize analysis sql. +#[inline] +pub fn normalize_analysis_sql( + py: Python<'_>, + request: crate::analysis::_helpers::native_classes::NormalizationInput, +) -> PyResult { + normalization::normalize_analysis_sql(py, request) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs new file mode 100644 index 0000000000..4568154300 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs @@ -0,0 +1,12 @@ +//! Register the binding catalog functions and classes. + +use crate::analysis::_helpers::binding_catalog; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the binding catalog functions and classes. +#[inline] +pub fn register_binding_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { + binding_catalog::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs new file mode 100644 index 0000000000..c6f68e563a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs @@ -0,0 +1,12 @@ +//! Register the normalization functions and classes. + +use crate::analysis::_helpers::normalization; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the normalization functions and classes. +#[inline] +pub fn register_normalization(module: &Bound<'_, PyModule>) -> PyResult<()> { + normalization::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs new file mode 100644 index 0000000000..5c1e2e7a89 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs @@ -0,0 +1,17 @@ +//! A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. + +use crate::analysis::_helpers::schema_request; +use crate::analysis::models::SchemaRequestParts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. +#[inline] +pub fn schema_validation_request<'py>( + py: Python<'py>, + parts: SchemaRequestParts<'py>, +) -> PyResult> { + schema_request::schema_validation_request(py, parts) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs new file mode 100644 index 0000000000..f7badf212c --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs @@ -0,0 +1,17 @@ +//! Binding diagnostics for a batch of complete-schema validation requests, in request order. + +use crate::analysis::_helpers::binding_catalog; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// Binding diagnostics for a batch of complete-schema validation requests, in request order. +#[inline] +pub fn schema_validations<'py>( + py: Python<'py>, + requests: &Bound<'py, PyAny>, +) -> PyResult> { + binding_catalog::schema_validations(py, requests) +} diff --git a/crates/sqlbuild-python-project/src/analysis/mod.rs b/crates/sqlbuild-python-project/src/analysis/mod.rs new file mode 100644 index 0000000000..73aacbfb09 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/mod.rs @@ -0,0 +1,7 @@ +//! Analysis classes and requests: the binding catalog, relation names, normalization and schema requests. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/models.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs rename to crates/sqlbuild-python-project/src/analysis/models.rs index 5595f3c0b0..4930ec7c5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/models.rs @@ -1,85 +1,77 @@ -//! The compile-lifetime binding catalog: known relation shapes around one native project catalog. +//! Public records of analysis. -use std::sync::atomic::{AtomicBool, Ordering}; - -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyMapping, PyTuple}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::analysis::_helpers::binding_catalog::QUOTED_IDENTIFIER_DELIMITER; +use crate::analysis::_helpers::conversions; +use crate::analysis::_helpers::native_classes::CatalogInput; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use pyo3::FromPyObject; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::IntoPyObject; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyDictMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyMapping; +use pyo3::types::PyTuple; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; - -use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; -use sqlbuild_python_base::types::IntoPositional; - -use crate::bindings::_helpers::analysis::conversions; -use crate::bindings::models::{CatalogInput, ProjectCatalog}; +use sqlbuild_analysis::semantic_validation::types::BindingRequest; +use sqlbuild_analysis::semantic_validation::types::Relations; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; - -/// Dialects whose quoted identifiers bind case-sensitively unless the session says otherwise. -pub(crate) const CASE_SENSITIVE_BINDING_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; -/// Adapter dialect names and the native dialect each validates as. -pub(crate) const NATIVE_DIALECT_ALIASES: [(&str, &str); 3] = [ - ("postgres", "postgresql"), - ("motherduck", "duckdb"), - ("sqlserver", "tsql"), -]; -const QUOTED_IDENTIFIER_DELIMITER: char = '"'; -const SNOWFLAKE_DIALECT: &str = "snowflake"; -const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; -const SESSION_PARAMETERS: &str = "session_parameters"; -const GENERIC_DIALECT: &str = "generic"; +use sqlbuild_python_base::types::FromPositional; +use sqlbuild_python_base::types::IntoPositional; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; /// One native project catalog and the relation shapes the compile has told it about. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct BindingCatalog { +#[derive(Debug)] +pub struct BindingCatalog { #[pyo3(get)] - pub(crate) schemas: Py, + pub schemas: Py, #[pyo3(get)] - pub(crate) dialect: String, + pub dialect: String, #[pyo3(get)] - pub(crate) quoted_ignore_case: bool, + pub quoted_ignore_case: bool, #[pyo3(get)] - pub(crate) analysis_shapes: Py, + pub analysis_shapes: Py, #[pyo3(get)] - pub(crate) expression_shapes: Py, + pub expression_shapes: Py, #[pyo3(get)] - pub(crate) native: Py, + pub native: Py, /// Whether binding preparation has added relations to `schemas`. - pub(crate) extended: AtomicBool, + pub extended: AtomicBool, } /// A catalog's dialect, case mode, known functions and known types. -#[derive(Clone, PartialEq, Eq)] -pub(crate) struct CatalogKey { - pub(crate) dialect: String, - pub(crate) quoted_ignore_case: bool, - pub(crate) known_functions: Vec, - pub(crate) known_types: Vec, +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogKey { + pub dialect: String, + pub quoted_ignore_case: bool, + pub known_functions: Vec, + pub known_types: Vec, } impl BindingCatalog { /// Whether `schemas` holds exactly the relations it was created with and later given. - pub(crate) fn unextended(&self) -> bool { + pub fn unextended(&self) -> bool { !self.extended.load(Ordering::Relaxed) } /// A catalog for `key` that starts with `relations`. - pub(crate) fn create( - py: Python<'_>, - key: CatalogKey, - relations: &Bound<'_, PyAny>, - ) -> PyResult { + pub fn create(py: Python<'_>, key: CatalogKey, relations: &Bound<'_, PyAny>) -> PyResult { let CatalogKey { dialect, quoted_ignore_case, @@ -110,7 +102,7 @@ impl BindingCatalog { } /// `BindingCatalog.prepare`: add unknown closed relations (overrides while in use), then changed ones. - fn prepared( + pub(crate) fn prepared( &self, py: Python<'_>, requests: &[Bound<'_, PyAny>], @@ -171,7 +163,7 @@ impl BindingCatalog { /// `BindingCatalog(dialect=, quoted_ignore_case=, known_functions=, known_types=, relations=)`. #[new] #[pyo3(signature = (**request))] - fn new(py: Python<'_>, request: Option<&Bound<'_, PyDict>>) -> PyResult { + pub(crate) fn new(py: Python<'_>, request: Option<&Bound<'_, PyDict>>) -> PyResult { let request: Bound<'_, PyDict> = request.cloned().unwrap_or_else(|| PyDict::new(py)); let field = |name: &str| -> PyResult> { request.get_item(name)?.ok_or_else(|| { @@ -191,7 +183,7 @@ impl BindingCatalog { /// Send the native catalog only the analysis shapes that changed. #[pyo3(signature = (*, types, nullability))] - fn prepare_analysis( + pub(crate) fn prepare_analysis( &self, py: Python<'_>, types: &Bound<'_, PyAny>, @@ -221,7 +213,11 @@ impl BindingCatalog { } /// A copy whose relations also include `relations`. - fn with_relations(&self, py: Python<'_>, relations: &Bound<'_, PyAny>) -> PyResult { + pub(crate) fn with_relations( + &self, + py: Python<'_>, + relations: &Bound<'_, PyAny>, + ) -> PyResult { let schemas: Bound<'_, PyDict> = self.schemas.bind(py).copy()?; schemas.update(relations.downcast::()?)?; let native: ProjectCatalog = ProjectCatalog { @@ -244,7 +240,7 @@ impl BindingCatalog { /// The output schema with exact-case names where quoted identifiers bind case-sensitively. #[pyo3(signature = (*, sql, columns, inputs))] - fn inferred_schema<'py>( + pub(crate) fn inferred_schema<'py>( &self, py: Python<'py>, sql: &str, @@ -284,125 +280,73 @@ impl BindingCatalog { } } -/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. -pub(crate) fn ignores_quoted_case( - connection: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { - return Ok(false); - } - let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; - let Ok(parameters) = parameters.downcast::() else { - return Ok(false); - }; - for (key, value) in parameters.iter() { - let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; - if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { - return Ok(true); - } +/// The query, relation schemas and binding context of one schema-validation request. +#[derive(Debug)] +pub struct SchemaRequestParts<'py> { + pub query_sql: String, + pub placeholders: Bound<'py, PyAny>, + pub dialect: Bound<'py, PyAny>, + pub schema: Bound<'py, PyAny>, + pub known_functions: Bound<'py, PyAny>, + pub known_types: Bound<'py, PyAny>, + pub catalog: Bound<'py, PyAny>, + pub quoted_identifiers_ignore_case: bool, +} + +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct ProjectCatalog { + pub inner: validation::ProjectCatalog, +} + +/// One resolved compact analysis batch that runs without borrowing its project catalog. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct CompactAnalysisJob { + pub inner: validation::CompactAnalysisJob, +} + +impl std::fmt::Debug for CompactAnalysisJob { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CompactAnalysisJob") + .finish_non_exhaustive() } - Ok(false) } -/// The native dialect a request's adapter dialect validates as. -fn native_dialect(dialect: &Bound<'_, PyAny>) -> PyResult { - let name: String = if dialect.is_truthy()? { - dialect.extract()? - } else { - GENERIC_DIALECT.to_owned() - }; - Ok(NATIVE_DIALECT_ALIASES - .iter() - .find(|(alias, _)| *alias == name) - .map_or(name.clone(), |(_, native)| (*native).to_owned())) +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct BindingPositions { + pub inner: validation::BindingPositions, } -/// Binding diagnostics for a batch of complete-schema validation requests, in request order. -#[pyfunction] -#[pyo3(signature = (*, requests))] -pub(crate) fn schema_validations<'py>( - py: Python<'py>, - requests: &Bound<'py, PyAny>, -) -> PyResult> { - let requests: Vec> = requests.try_iter()?.collect::>()?; - let mut created: Vec<(CatalogKey, Py)> = Vec::new(); - let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); - for (index, request) in requests.iter().enumerate() { - let key: CatalogKey = CatalogKey { - dialect: native_dialect(&native_field(request, "dialect")?)?, - quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? - .extract()?, - known_functions: native_field(request, "known_functions")?.extract()?, - known_types: native_field(request, "known_types")?.extract()?, - }; - let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; - let own: bool = !explicit.is_none(); - let catalog: Py = if own { - explicit.extract()? - } else if let Some((_, catalog)) = created.iter().find(|(known, _)| *known == key) { - catalog.clone_ref(py) - } else { - let catalog: Py = Py::new( - py, - BindingCatalog::create(py, key.clone(), &native_field(request, "schema")?)?, - )?; - created.push((key, catalog.clone_ref(py))); - catalog - }; - match groups.iter_mut().find(|(known, _, _)| known.is(&catalog)) { - Some((_, any_explicit, members)) => { - *any_explicit |= own; - members.push(index); - } - None => groups.push((catalog, own, vec![index])), - } +/// A record of another crate exchanged with Python as its positional tuple. +#[derive(Clone, Debug)] +pub struct Positional(pub T); + +impl Positional { + /// Each record of `items`, for Python. + pub fn wrap_all(items: Vec) -> Vec { + items.into_iter().map(Self).collect() } - let empty: Bound<'py, PyAny> = new_record(py, RecordId::SqlBindingResult, (), None)?; - let mut results: Vec> = vec![empty; requests.len()]; - for (catalog, any_explicit, members) in groups { - let catalog = catalog.borrow(py); - let batch: Vec> = members - .iter() - .map(|index| requests[*index].clone()) - .collect(); - let prepared: Vec = catalog.prepared(py, &batch)?; - let native = catalog.native.borrow(py); - let inner: &validation::ProjectCatalog = &native.inner; - let dialect: &str = &catalog.dialect; - let rows: Vec> = if any_explicit { - py.compiler_detach(|| inner.binding_results(prepared)) - } else { - py.compiler_detach(|| inner.schema_validation_rows(dialect, prepared)) - } - .map_err(compiler_error)?; - if rows.len() != members.len() { - return Err(compiler_error( - "native SQL schema validation returned an invalid batch response", - )); - } - for (index, diagnostics) in members.into_iter().zip(rows) { - let built: Vec> = diagnostics - .into_iter() - .map(|row| { - new_record( - py, - RecordId::SqlBindingDiagnostic, - row.into_positional(py)?, - None, - ) - }) - .collect::>()?; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; - results[index] = new_record(py, RecordId::SqlBindingResult, (), Some(&keywords))?; - } + + /// Each record Python sent in `items`. + pub fn unwrap_all(items: Vec) -> Vec { + items.into_iter().map(|item| item.0).collect() } - PyTuple::new(py, results) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(schema_validations, module)?)?; - Ok(()) +impl<'py, T: FromPositional> FromPyObject<'py> for Positional { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + T::from_positional(value).map(Self) + } +} + +impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { + type Target = PyTuple; + type Output = Bound<'py, PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.into_positional(py) + } } diff --git a/crates/sqlbuild-python-project/src/analysis/types.rs b/crates/sqlbuild-python-project/src/analysis/types.rs new file mode 100644 index 0000000000..dca7b1ab92 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of analysis. + +/// `(name, schema, database)`. +pub type RelationRow = (String, Option, Option); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs index 1a6890da21..62330a8029 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs @@ -11,9 +11,9 @@ use sqlbuild_analysis::assembly::analysis_session::models::ProjectionFacts; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::analysis_session::models::FixtureInferenceCount; -use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; -use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; +use crate::analysis_session::_helpers::stage_functions::query_column_rows; +use crate::analysis_session::models::FixtureInferenceCount; +use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs new file mode 100644 index 0000000000..8094738564 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod fixture_columns; +pub(crate) mod nullability_rules; +pub(crate) mod profile_inputs; +pub(crate) mod query_columns; +pub(crate) mod session; +pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs index 95e365fe7f..11424c4f1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs @@ -1,13 +1,15 @@ //! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. +use crate::analysis_session::constants::ADAPTER_RULE_ID; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, NATIVE_DIALECT_ALIASES, -}; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::models::BindingCatalog; +use crate::analysis::models::CatalogKey; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -15,7 +17,7 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; const GENERIC_DIALECT: &str = "generic"; -pub(crate) const ADAPTER_RULE_ID: &str = "python"; + /// The nullability rules the native analysis knows, with their native rule ids. const KNOWN_RULES: [(AdapterHost, &str); 3] = [ (AdapterHost::FirstArgNullability, "first_arg"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs index f2761d28d6..903c5ad29a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs @@ -1,5 +1,6 @@ //! Python's per-query column analysis outside the session, run natively. +use crate::analysis_session::types::ColumnsRow; use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; @@ -14,10 +15,10 @@ use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::analysis_session::nullability_rules::{ - RuleFailure, nullability_callback, raised, -}; -use crate::bindings::models::ProjectCatalog; +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::nullability_rules::RuleFailure; +use crate::analysis_session::_helpers::nullability_rules::nullability_callback; +use crate::analysis_session::_helpers::nullability_rules::raised; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; @@ -52,8 +53,6 @@ positional_record! { } /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); -/// `(succeeded, [(name, type, nullability)] or None, has star)`. -pub(crate) type ColumnsRow = (bool, Option, String)>>, bool); /// Each query's columns, as Python's analysis outside the session inferred them. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs similarity index 53% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs index af5650a37b..67f3f2d22b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs @@ -1,48 +1,42 @@ //! The native model analysis session and expression-source shapes. +use crate::analysis_session::models::CacheStatsRow; +use crate::analysis_session::models::FamilyRow; +use crate::analysis_session::models::ModelRow; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::analysis_session::models::PivotRequestRow; +use crate::analysis_session::models::RequestRow; use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods}; use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; -use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; use sqlbuild_analysis::assembly::analysis_session::main::model_reference_facts::model_reference_facts; use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; -use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; -use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; -use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::session_sharing::session_sharing; use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::take_analysis_cache::take_analysis_cache; use sqlbuild_analysis::assembly::analysis_session::models::{ - AnalysisCacheStats, AnalysisSession, ColumnFact, DynamicFamily, ExpressionShape, - ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, ModelRequest, - PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, SessionOutcome, SessionRequest, - SessionStep, + AnalysisSession, ColumnFact, DynamicFamily, ExpressionShape, ExpressionShapeRequest, + LineageFacts, LineageRow, ModelOutcome, ModelRequest, PivotBatchRequest, PivotModel, + PivotOutcome, PivotTables, SessionRequest, SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; use sqlbuild_python_base::positional_record; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use sqlbuild_cache::store::models::NativeStore; use sqlbuild_core::constants::PANIC_MESSAGE; -use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; use std::path::PathBuf; -use crate::bindings::_helpers::analysis_session::nullability_rules::{ - RuleFailure, nullability_callback, raised, -}; -use crate::bindings::models::Positional; -use crate::bindings::models::ProjectCatalog; +use crate::analysis::models::Positional; +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::nullability_rules::RuleFailure; +use crate::analysis_session::_helpers::nullability_rules::nullability_callback; +use crate::analysis_session::_helpers::nullability_rules::raised; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::main::open_store::open_store; -use sqlbuild_python_base::cache::main::save_store::save_store; use sqlbuild_python_base::types::CompilerDetach; /// `(name, type, nullability)`. @@ -50,7 +44,7 @@ type ColumnRow = (String, Option, String); positional_record! { into; /// One lineage column: output column, transform and confidence codes and its sources. - struct LineageItem { + pub(crate) struct LineageItem { output_column: String, transform_code: u8, confidence_code: u8, @@ -59,84 +53,29 @@ positional_record! { } } -positional_record! { - extract; - /// One declared dynamic column family. - pub(crate) struct FamilyRow { - pub(crate) name: String, - pub(crate) pivot_column: String, - pub(crate) value_column: String, - pub(crate) aggregate: String, - pub(crate) data_type: String, - pub(crate) name_pattern: Option, - } -} - -impl From for FamilyRow { - fn from(family: sqlbuild_model_config::header_metadata::models::DynamicFamily) -> Self { - Self { - name: family.name, - pivot_column: family.pivot_column, - value_column: family.value_column, - aggregate: family.aggregate, - data_type: family.data_type, - name_pattern: family.name_pattern, - } - } -} - -/// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -pub(crate) struct ModelRow { - pub(crate) name: String, - pub(crate) query_sql: String, - pub(crate) placeholders: Pairs, - pub(crate) reference_rows: Vec<(String, String)>, - pub(crate) recover_cte_facts: bool, - pub(crate) snapshot_columns: Option<(String, String)>, - pub(crate) dynamic_families: Vec, - pub(crate) sql_macro_checks: Vec, -} - -/// The profile settings, relation shapes and models one session analyses. -pub(crate) struct RequestRow { - pub(crate) dialect: String, - pub(crate) case_sensitive_shapes: bool, - pub(crate) quoted_ignore_case: bool, - pub(crate) function_return_types: Pairs, - pub(crate) known_functions: Vec, - pub(crate) nullability_rules: Option, - pub(crate) rich_type_inference: bool, - pub(crate) column_types: Shapes, - pub(crate) column_nullability: Shapes, - pub(crate) complete_schemas: Shapes, - pub(crate) catalog_schemas: Shapes, - pub(crate) dynamic_families_by_table: Vec<(String, Vec)>, - pub(crate) models: Vec, -} - /// `(publications, failures)`. pub(crate) type StepRow = (Shapes, Vec); positional_record! { into; /// One model's analysis, lineage source and cleaned SQL. - struct OutcomeRow { - analysis_succeeded: bool, - columns: Option>, - lineage_source: &'static str, - lineage: Vec, - has_star: bool, - star_resolved: bool, - binding_diagnostics: Vec>, - binding_validated: bool, - cleaned_sql: String, + pub(crate) struct OutcomeRow { + pub(crate) analysis_succeeded: bool, + pub(crate) columns: Option>, + pub(crate) lineage_source: &'static str, + pub(crate) lineage: Vec, + pub(crate) has_star: bool, + pub(crate) star_resolved: bool, + pub(crate) binding_diagnostics: Vec>, + pub(crate) binding_validated: bool, + pub(crate) cleaned_sql: String, } } positional_record! { into; /// One dynamic pivot proof. - pub(crate) struct ProofRow { + pub struct ProofRow { output_proven: bool, fixed_columns: Vec, /// `(family, inferred type)` of each family. @@ -150,34 +89,9 @@ positional_record! { /// One model's pivot SQL and declared families. pub(crate) type PivotModelRow = (String, Vec); -positional_record! { - extract; - /// The dialect, relation facts and families by table every proof reads, and the models. - pub(crate) struct PivotRequestRow { - pub(crate) dialect: String, - pub(crate) column_types: Shapes, - pub(crate) authoritative_types: Shapes, - pub(crate) column_nullability: Shapes, - pub(crate) families_by_table: Vec<(String, Vec)>, - pub(crate) models: Vec, - } -} - /// `("absent" | "deferred" | "proof", proof)`. pub(crate) type ContractRow = (&'static str, Option); -positional_record! { - into; - /// The analysis cache's hits, misses, stored entries and why it was not read or saved. - #[derive(Clone)] - pub(crate) struct CacheStatsRow { - pub(crate) hits: usize, - pub(crate) misses: usize, - pub(crate) stored: usize, - pub(crate) failure: Option, - } -} - positional_record! { extract; /// The settings and expressions one expression-source shape request infers. @@ -192,176 +106,20 @@ positional_record! { /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. pub(crate) type FinishRow = (Shapes, Vec, Vec); -/// One compile's native model analysis; an internal failure raises `NativeCompilerError`. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct NativeModelAnalysisSession { - inner: Option, - finished: Option, - /// Each requested model's outcome and dynamic pivot proof, in request order, once finished. - outcomes: Vec<(ModelOutcome, PivotOutcome)>, - sharing: (usize, usize), - cache_path: Option, - cache_stats: Option, - rule_failure: RuleFailure, - /// The analysis dialect the session's models were marked and analysed in. - dialect: String, -} - /// What an internal native failure of the session names. const SESSION_CONTEXT: &str = "native model analysis"; const SHAPES_CONTEXT: &str = "native expression-source shapes"; /// The native store kind holding finished model analyses. const MODEL_ANALYSIS_STORE_KIND: &str = "model-analyses"; -impl NativeModelAnalysisSession { - /// The finished session later compile stages read, once `finish` has succeeded. - pub(crate) fn finished_session(&self) -> Option<&FinishedSession> { - self.finished.as_ref() - } - - /// Each requested model's finished outcome and dynamic pivot proof, in request order. - pub(crate) fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { - &self.outcomes - } - - /// The finished outcome and dynamic pivot proof of the model requested at `index`. - pub(crate) fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { - self.outcomes - .get(index) - .ok_or_else(|| internal_error("the session returned no analysis for a model")) - } - - /// The running session, or `NativeCompilerError` once it has failed or finished. - pub(crate) fn dialect(&self) -> &str { - &self.dialect - } - - fn running(&mut self) -> PyResult { - self.inner - .take() - .ok_or_else(|| internal_error("the session is not running")) - } -} - /// An internal native failure of the session as `NativeCompilerError`. -fn internal_error(reason: &str) -> PyErr { +pub(crate) fn internal_error(reason: &str) -> PyErr { if is_compiler_panic(reason) && reason != PANIC_MESSAGE { return compiler_error(reason); } compiler_error(native_failure(SESSION_CONTEXT, reason)) } -impl NativeModelAnalysisSession { - /// Analyse every model. - pub(crate) fn run(&mut self, py: Python<'_>) -> PyResult { - let mut session: AnalysisSession = self.running()?; - let result: Result<(AnalysisSession, SessionStep), String> = - py.compiler_detach(move || { - run_analysis_session(&mut session).map(|step| (session, step)) - }); - if let Some(error) = raised(&self.rule_failure) { - return Err(error); - } - let (session, step) = result.map_err(|reason| internal_error(&reason))?; - self.sharing = session_sharing(&session); - self.inner = Some(session); - Ok(step_row(step)) - } - - /// Every model's outcome; saves the cache best-effort. - pub(crate) fn finish(&mut self, py: Python<'_>) -> PyResult { - let mut session: AnalysisSession = self.running()?; - let cache: Option<(NativeStore, AnalysisCacheStats)> = take_analysis_cache(&mut session); - let (outcome, finished) = catch_compiler_panic(|| finish_analysis_session(session)) - .map_err(|reason| internal_error(&reason))?; - self.finished = Some(finished); - if let (Some((store, stats)), Some(path)) = (cache, self.cache_path.as_ref()) { - let saved: Option = save_store(py, &store, path, &[]) - .err() - .map(|error| error.to_string()); - self.cache_stats = Some(CacheStatsRow { - hits: stats.hits, - misses: stats.misses, - stored: stats.stored, - failure: saved, - }); - } - let SessionOutcome { - models, - schema_additions, - analysis_names, - dynamic_contracts, - } = outcome; - let succeeded: Vec = models - .iter() - .map(|model| model.analysis.analysis_succeeded) - .collect(); - self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); - Ok((schema_additions, analysis_names, succeeded)) - } -} - -#[pymethods] -impl NativeModelAnalysisSession { - /// Models whose output names and lineage the finished session keeps for later stages. - #[getter] - pub(crate) fn fact_models(&self) -> Vec { - self.finished - .as_ref() - .map(finished_fact_models) - .unwrap_or_default() - } - - /// Each requested model's raw binding diagnostic rows, in request order. - pub(crate) fn binding_rows(&self) -> Vec>> { - self.outcomes - .iter() - .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) - .collect() - } - - /// The model at `index`'s outcome and proof rows, for hand-built model inputs only. - fn outcome_rows(&self, index: usize) -> PyResult<(OutcomeRow, ContractRow)> { - let (outcome, contract) = self.outcome(index)?; - let cleaned: String = resolved_cleaned_sql(outcome) - .map_err(compiler_error)? - .into_owned(); - Ok(( - outcome_row(outcome.clone(), cleaned), - contract_row(contract.clone()), - )) - } - - /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. - #[getter] - pub(crate) fn cache_stats(&self) -> Option { - self.cache_stats.clone() - } - - /// Each model's dynamic pivot proof from the finished session's tables. - pub(crate) fn prove_dynamic_contracts( - &self, - py: Python<'_>, - models: Vec, - ) -> PyResult> { - let session: &FinishedSession = self - .finished - .as_ref() - .ok_or_else(|| internal_error("the session has not finished"))?; - let models: Vec = models.into_iter().map(pivot_model).collect(); - let outcomes: Vec = py - .compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) - .map_err(|reason| internal_error(&reason))?; - Ok(outcomes.into_iter().map(contract_row).collect()) - } - - /// `(shared members, shared members re-analysed alone)` as of the last step. - #[getter] - pub(crate) fn sharing(&self) -> (usize, usize) { - self.sharing - } -} - /// Start a session on `catalog`. pub(crate) fn start_model_analysis_session( catalog: PyRef<'_, ProjectCatalog>, @@ -528,7 +286,7 @@ fn model_request(row: ModelRow) -> ModelRequest { } } -fn step_row(step: SessionStep) -> StepRow { +pub(crate) fn step_row(step: SessionStep) -> StepRow { (step.publications, step.failures) } @@ -547,7 +305,9 @@ fn lineage_items(rows: Vec) -> Vec { .collect() } -fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec)> { +pub(crate) fn families_by_table( + rows: Vec<(String, Vec)>, +) -> Vec<(String, Vec)> { rows.into_iter() .map(|(name, families)| (name, families.into_iter().map(dynamic_family).collect())) .collect() @@ -572,7 +332,7 @@ fn dynamic_family(row: FamilyRow) -> DynamicFamily { } } -fn contract_row(outcome: PivotOutcome) -> ContractRow { +pub(crate) fn contract_row(outcome: PivotOutcome) -> ContractRow { match outcome { PivotOutcome::Absent => ("absent", None), PivotOutcome::Proof(proof) => ( @@ -589,7 +349,7 @@ fn contract_row(outcome: PivotOutcome) -> ContractRow { } } -fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { +pub(crate) fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { let analysis = outcome.analysis; let (source, lineage) = match analysis.lineage { LineageFacts::Native(rows) => ("native", lineage_items(rows)), @@ -640,7 +400,7 @@ pub(crate) fn prove_dynamic_column_contracts( Ok(outcomes.into_iter().map(contract_row).collect()) } -fn pivot_model(row: PivotModelRow) -> PivotModel { +pub(crate) fn pivot_model(row: PivotModelRow) -> PivotModel { let (sql, families) = row; PivotModel { sql, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs index 68bc94ec1e..8d8e4a1013 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs @@ -8,18 +8,27 @@ use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - aliased_dialect, case_sensitive_shapes, dialect_or_generic, nullability_rules, profile_catalog, - profile_dialect, return_types, shape_rows, -}; -use crate::bindings::_helpers::analysis_session::query_columns::{ - ColumnsRow, QueryRequestRow, QueryRow, infer_query_columns, -}; -use crate::bindings::_helpers::analysis_session::session::{ - ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, - PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, -}; -use crate::bindings::models::ProjectCatalog; +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::profile_inputs::aliased_dialect; +use crate::analysis_session::_helpers::profile_inputs::case_sensitive_shapes; +use crate::analysis_session::_helpers::profile_inputs::dialect_or_generic; +use crate::analysis_session::_helpers::profile_inputs::nullability_rules; +use crate::analysis_session::_helpers::profile_inputs::profile_catalog; +use crate::analysis_session::_helpers::profile_inputs::profile_dialect; +use crate::analysis_session::_helpers::profile_inputs::return_types; +use crate::analysis_session::_helpers::profile_inputs::shape_rows; +use crate::analysis_session::_helpers::query_columns::QueryRequestRow; +use crate::analysis_session::_helpers::query_columns::QueryRow; +use crate::analysis_session::_helpers::query_columns::infer_query_columns; +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::ExpressionShapesRow; +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::_helpers::session::infer_expression_source_shapes; +use crate::analysis_session::_helpers::session::prove_dynamic_column_contracts; +use crate::analysis_session::models::FamilyRow; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::analysis_session::models::PivotRequestRow; +use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPES_INFERRED; diff --git a/crates/sqlbuild-python-project/src/analysis_session/constants.rs b/crates/sqlbuild-python-project/src/analysis_session/constants.rs new file mode 100644 index 0000000000..5ba9dbf98f --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/constants.rs @@ -0,0 +1,3 @@ +//! Public constants of analysis session. + +pub const ADAPTER_RULE_ID: &str = "python"; diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs new file mode 100644 index 0000000000..ebae1e5056 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs @@ -0,0 +1,15 @@ +//! Python's `inferred_binding_shape` test for keeping authored identifier quoting. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Python's `inferred_binding_shape` test for keeping authored identifier quoting. +#[inline] +pub fn case_sensitive_shapes( + profile: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + profile_inputs::case_sensitive_shapes(profile, dialect) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs new file mode 100644 index 0000000000..867182823e --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs @@ -0,0 +1,12 @@ +//! `dialect or "generic"`. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `dialect or "generic"`. +#[inline] +pub fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { + profile_inputs::dialect_or_generic(dialect) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs new file mode 100644 index 0000000000..7c4d3fc00c --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs @@ -0,0 +1,16 @@ +//! One shape per expression, cached on `profile`'s binding catalog; a failure raises. + +use crate::analysis_session::_helpers::stage_functions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// One shape per expression, cached on `profile`'s binding catalog; a failure raises. +#[inline] +pub fn expression_source_shapes<'py>( + expressions: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + stage_functions::expression_source_shapes(expressions, profile) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs new file mode 100644 index 0000000000..c4cd1c4327 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs @@ -0,0 +1,15 @@ +//! One shape per expression, cached on the profile's binding catalog; a failure raises. + +use crate::analysis_session::_helpers::stage_functions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// One shape per expression, cached on the profile's binding catalog; a failure raises. +#[inline] +pub fn infer_native_expression_source_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + stage_functions::infer_native_expression_source_shapes(request) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs new file mode 100644 index 0000000000..67d5659bc2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs @@ -0,0 +1,15 @@ +//! Each query's columns as `NativeQueryColumns`; a native internal failure raises. + +use crate::analysis_session::_helpers::stage_functions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. +#[inline] +pub fn infer_native_query_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + stage_functions::infer_native_query_columns(request) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs new file mode 100644 index 0000000000..7d47127629 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs @@ -0,0 +1,18 @@ +pub mod case_sensitive_shapes; +pub mod dialect_or_generic; +pub mod expression_source_shapes; +pub mod infer_native_expression_source_shapes; +pub mod infer_native_query_columns; +pub mod nullability_rules; +pub mod profile_dialect; +pub mod prove_dynamic_column_contracts; +pub mod query_column_rows; +pub mod query_columns_error; +pub mod register_fixture_columns; +pub mod register_query_columns; +pub mod register_session; +pub mod register_stage_functions; +pub mod return_types; +pub mod shape_rows; +pub mod start_model_analysis_session; +pub mod text_pairs; diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs new file mode 100644 index 0000000000..2f1b088234 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs @@ -0,0 +1,14 @@ +//! The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. + +use crate::analysis_session::_helpers::profile_inputs; +use crate::analysis_session::_helpers::profile_inputs::AdapterRules; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; + +/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. +#[inline] +pub fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { + profile_inputs::nullability_rules(profile) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs new file mode 100644 index 0000000000..02ae235604 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs @@ -0,0 +1,12 @@ +//! `profile.sql_analysis_dialect or "generic"`. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `profile.sql_analysis_dialect or "generic"`. +#[inline] +pub fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { + profile_inputs::profile_dialect(profile) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs new file mode 100644 index 0000000000..a5736753f6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs @@ -0,0 +1,16 @@ +//! Each model's dynamic pivot proof outside any session. + +use crate::analysis_session::_helpers::session; +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::models::PivotRequestRow; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Each model's dynamic pivot proof outside any session. +#[inline] +pub fn prove_dynamic_column_contracts( + py: Python<'_>, + request: PivotRequestRow, +) -> PyResult> { + session::prove_dynamic_column_contracts(py, request) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs new file mode 100644 index 0000000000..92b0605f52 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs @@ -0,0 +1,17 @@ +//! Query column rows. + +use crate::analysis_session::_helpers::stage_functions; +use crate::analysis_session::types::ColumnsRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Query column rows. +#[inline] +pub fn query_column_rows( + queries: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, + (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + stage_functions::query_column_rows(queries, profile, (types, nullability)) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs new file mode 100644 index 0000000000..7f5390de5a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs @@ -0,0 +1,11 @@ +//! A normalization rejection as Python's `ValueError`; anything else as an internal failure. + +use crate::analysis_session::_helpers::query_columns; +use pyo3::PyErr; +use sqlbuild_analysis::assembly::analysis_session::errors::QueryColumnsFailure; + +/// A normalization rejection as Python's `ValueError`; anything else as an internal failure. +#[inline] +pub fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { + query_columns::query_columns_error(failure) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs new file mode 100644 index 0000000000..4c0001ab40 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs @@ -0,0 +1,12 @@ +//! Register the fixture columns functions and classes. + +use crate::analysis_session::_helpers::fixture_columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fixture columns functions and classes. +#[inline] +pub fn register_fixture_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + fixture_columns::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs new file mode 100644 index 0000000000..27e3a80089 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs @@ -0,0 +1,12 @@ +//! Register the query columns functions and classes. + +use crate::analysis_session::_helpers::query_columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the query columns functions and classes. +#[inline] +pub fn register_query_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + query_columns::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs new file mode 100644 index 0000000000..7f615b3e62 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs @@ -0,0 +1,12 @@ +//! Register the session functions and classes. + +use crate::analysis_session::_helpers::session; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the session functions and classes. +#[inline] +pub fn register_session(module: &Bound<'_, PyModule>) -> PyResult<()> { + session::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs new file mode 100644 index 0000000000..12cff3a33a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs @@ -0,0 +1,12 @@ +//! Register the stage functions functions and classes. + +use crate::analysis_session::_helpers::stage_functions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the stage functions functions and classes. +#[inline] +pub fn register_stage_functions(module: &Bound<'_, PyModule>) -> PyResult<()> { + stage_functions::register(module) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs new file mode 100644 index 0000000000..2be87656fd --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs @@ -0,0 +1,13 @@ +//! `profile.function_return_types` items in order. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; + +/// `profile.function_return_types` items in order. +#[inline] +pub fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { + profile_inputs::return_types(profile) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs new file mode 100644 index 0000000000..c337119079 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs @@ -0,0 +1,13 @@ +//! `{relation: {column: value}}` as ordered rows, each value as `str(value)`. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; + +/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. +#[inline] +pub fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + profile_inputs::shape_rows(shapes) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs new file mode 100644 index 0000000000..078f42bb9d --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs @@ -0,0 +1,23 @@ +//! Start a session on `catalog`. + +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::session; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::analysis_session::models::RequestRow; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use std::path::PathBuf; + +/// Start a session on `catalog`. +#[inline] +pub fn start_model_analysis_session( + catalog: PyRef<'_, ProjectCatalog>, + request: RequestRow, + cache: Option<(PathBuf, String)>, + adapter_rules: Option<(Py, Py)>, +) -> PyResult { + session::start_model_analysis_session(catalog, request, cache, adapter_rules) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs new file mode 100644 index 0000000000..b5a37782c9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs @@ -0,0 +1,13 @@ +//! `{column: value}` as ordered pairs, each value as `str(value)`. + +use crate::analysis_session::_helpers::profile_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; + +/// `{column: value}` as ordered pairs, each value as `str(value)`. +#[inline] +pub fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { + profile_inputs::text_pairs(shape) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/mod.rs new file mode 100644 index 0000000000..6342b807d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/mod.rs @@ -0,0 +1,7 @@ +//! The analysis session: stage functions, query columns, nullability rules, profile inputs and fixture inferences. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/analysis_session/models.rs b/crates/sqlbuild-python-project/src/analysis_session/models.rs new file mode 100644 index 0000000000..d2c5105c74 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/models.rs @@ -0,0 +1,425 @@ +//! Public records of analysis session. + +use crate::analysis::models::Positional; +use crate::analysis_session::_helpers::fixture_columns::infer_fixture_column_facts; +use crate::analysis_session::_helpers::nullability_rules::RuleFailure; +use crate::analysis_session::_helpers::nullability_rules::raised; +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::FinishRow; +use crate::analysis_session::_helpers::session::OutcomeRow; +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::_helpers::session::StepRow; +use crate::analysis_session::_helpers::session::contract_row; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::outcome_row; +use crate::analysis_session::_helpers::session::pivot_model; +use crate::analysis_session::_helpers::session::step_row; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; +use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; +use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::main::session_sharing::session_sharing; +use sqlbuild_analysis::assembly::analysis_session::main::take_analysis_cache::take_analysis_cache; +use sqlbuild_analysis::assembly::analysis_session::models::AnalysisCacheStats; +use sqlbuild_analysis::assembly::analysis_session::models::AnalysisSession; +use sqlbuild_analysis::assembly::analysis_session::models::FinishedSession; +use sqlbuild_analysis::assembly::analysis_session::models::ModelOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::PivotModel; +use sqlbuild_analysis::assembly::analysis_session::models::PivotOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::SessionOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::SessionStep; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; +use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::cache::main::save_store::save_store; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::types::CompilerDetach; +use std::path::PathBuf; +use std::sync::Mutex; + +/// Reuse inference for identical fixture SQL under an equal, catalog-free profile. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct FixtureColumnInferences { + pub(crate) by_profile: Mutex, Py)>>, +} + +#[pymethods] +impl FixtureColumnInferences { + #[new] + pub fn new() -> Self { + Self { + by_profile: Mutex::new(Vec::new()), + } + } + + /// The inference for one fixture query, computing each distinct query once. + #[pyo3(signature = (*, query_sql, inference_profile))] + pub(crate) fn infer<'py>( + &self, + query_sql: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = query_sql.py(); + let compute = || { + infer_fixture_column_facts(Some(&keywords( + py, + &[ + ("query_sql", query_sql.clone()), + ("inference_profile", inference_profile.clone()), + ], + )?)) + }; + if !native_field(inference_profile, "binding_catalog")?.is_none() { + return compute(); + } + let inferred = self.profile_results(py, inference_profile)?; + if let Some(found) = inferred.get_item(query_sql)? { + return Ok(found); + } + let computed = compute()?; + inferred.set_item(query_sql, &computed)?; + Ok(computed) + } +} + +impl FixtureColumnInferences { + pub(crate) fn profile_results<'py>( + &self, + py: Python<'py>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let known: Vec<(Py, Py)> = self + .by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .iter() + .map(|(profile, results)| (profile.clone_ref(py), results.clone_ref(py))) + .collect(); + for (profile, results) in known { + if profile.bind(py).eq(inference_profile)? { + return Ok(results.into_bound(py)); + } + } + let results = PyDict::new(py); + self.by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .push((inference_profile.clone().unbind(), results.clone().unbind())); + Ok(results) + } +} + +/// Fixture column inferences computed since the last reset. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureInferenceCount { + pub computed: u64, +} + +/// One declared dynamic column family. +#[derive(Debug)] +pub struct FamilyRow { + pub name: String, + pub pivot_column: String, + pub value_column: String, + pub aggregate: String, + pub data_type: String, + pub name_pattern: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for FamilyRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, pivot_column, value_column, aggregate, data_type, name_pattern): ( + String, + String, + String, + String, + String, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + }) + } +} + +impl From for FamilyRow { + fn from(family: sqlbuild_model_config::header_metadata::models::DynamicFamily) -> Self { + Self { + name: family.name, + pivot_column: family.pivot_column, + value_column: family.value_column, + aggregate: family.aggregate, + data_type: family.data_type, + name_pattern: family.name_pattern, + } + } +} + +/// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. +#[derive(Debug)] +pub struct ModelRow { + pub name: String, + pub query_sql: String, + pub placeholders: Pairs, + pub reference_rows: Vec<(String, String)>, + pub recover_cte_facts: bool, + pub snapshot_columns: Option<(String, String)>, + pub dynamic_families: Vec, + pub sql_macro_checks: Vec, +} + +/// The profile settings, relation shapes and models one session analyses. +#[derive(Debug)] +pub struct RequestRow { + pub dialect: String, + pub case_sensitive_shapes: bool, + pub quoted_ignore_case: bool, + pub function_return_types: Pairs, + pub known_functions: Vec, + pub nullability_rules: Option, + pub rich_type_inference: bool, + pub column_types: Shapes, + pub column_nullability: Shapes, + pub complete_schemas: Shapes, + pub catalog_schemas: Shapes, + pub dynamic_families_by_table: Vec<(String, Vec)>, + pub models: Vec, +} + +/// The dialect, relation facts and families by table every proof reads, and the models. +#[derive(Debug)] +pub struct PivotRequestRow { + pub dialect: String, + pub column_types: Shapes, + pub authoritative_types: Shapes, + pub column_nullability: Shapes, + pub families_by_table: Vec<(String, Vec)>, + pub models: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for PivotRequestRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let ( + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table, + models, + ): ( + String, + Shapes, + Shapes, + Shapes, + Vec<(String, Vec)>, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table, + models, + }) + } +} + +/// The analysis cache's hits, misses, stored entries and why it was not read or saved. +#[derive(Clone, Debug)] +pub struct CacheStatsRow { + pub hits: usize, + pub misses: usize, + pub stored: usize, + pub failure: Option, +} + +impl<'py> pyo3::IntoPyObject<'py> for CacheStatsRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject((self.hits, self.misses, self.stored, self.failure), py) + } +} + +/// One compile's native model analysis; an internal failure raises `NativeCompilerError`. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeModelAnalysisSession { + pub(crate) inner: Option, + pub(crate) finished: Option, + /// Each requested model's outcome and dynamic pivot proof, in request order, once finished. + pub(crate) outcomes: Vec<(ModelOutcome, PivotOutcome)>, + pub(crate) sharing: (usize, usize), + pub(crate) cache_path: Option, + pub(crate) cache_stats: Option, + pub(crate) rule_failure: RuleFailure, + /// The analysis dialect the session's models were marked and analysed in. + pub(crate) dialect: String, +} + +impl NativeModelAnalysisSession { + /// The finished session later compile stages read, once `finish` has succeeded. + pub fn finished_session(&self) -> Option<&FinishedSession> { + self.finished.as_ref() + } + + /// Each requested model's finished outcome and dynamic pivot proof, in request order. + pub fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { + &self.outcomes + } + + /// The finished outcome and dynamic pivot proof of the model requested at `index`. + pub fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { + self.outcomes + .get(index) + .ok_or_else(|| internal_error("the session returned no analysis for a model")) + } + + /// The running session, or `NativeCompilerError` once it has failed or finished. + pub fn dialect(&self) -> &str { + &self.dialect + } + + pub(crate) fn running(&mut self) -> PyResult { + self.inner + .take() + .ok_or_else(|| internal_error("the session is not running")) + } +} + +impl NativeModelAnalysisSession { + /// Analyse every model. + pub fn run(&mut self, py: Python<'_>) -> PyResult { + let mut session: AnalysisSession = self.running()?; + let result: Result<(AnalysisSession, SessionStep), String> = + py.compiler_detach(move || { + run_analysis_session(&mut session).map(|step| (session, step)) + }); + if let Some(error) = raised(&self.rule_failure) { + return Err(error); + } + let (session, step) = result.map_err(|reason| internal_error(&reason))?; + self.sharing = session_sharing(&session); + self.inner = Some(session); + Ok(step_row(step)) + } + + /// Every model's outcome; saves the cache best-effort. + pub fn finish(&mut self, py: Python<'_>) -> PyResult { + let mut session: AnalysisSession = self.running()?; + let cache: Option<(NativeStore, AnalysisCacheStats)> = take_analysis_cache(&mut session); + let (outcome, finished) = catch_compiler_panic(|| finish_analysis_session(session)) + .map_err(|reason| internal_error(&reason))?; + self.finished = Some(finished); + if let (Some((store, stats)), Some(path)) = (cache, self.cache_path.as_ref()) { + let saved: Option = save_store(py, &store, path, &[]) + .err() + .map(|error| error.to_string()); + self.cache_stats = Some(CacheStatsRow { + hits: stats.hits, + misses: stats.misses, + stored: stats.stored, + failure: saved, + }); + } + let SessionOutcome { + models, + schema_additions, + analysis_names, + dynamic_contracts, + } = outcome; + let succeeded: Vec = models + .iter() + .map(|model| model.analysis.analysis_succeeded) + .collect(); + self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); + Ok((schema_additions, analysis_names, succeeded)) + } +} + +#[pymethods] +impl NativeModelAnalysisSession { + /// Models whose output names and lineage the finished session keeps for later stages. + #[getter] + pub fn fact_models(&self) -> Vec { + self.finished + .as_ref() + .map(finished_fact_models) + .unwrap_or_default() + } + + /// Each requested model's raw binding diagnostic rows, in request order. + pub fn binding_rows(&self) -> Vec>> { + self.outcomes + .iter() + .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) + .collect() + } + + /// The model at `index`'s outcome and proof rows, for hand-built model inputs only. + pub(crate) fn outcome_rows(&self, index: usize) -> PyResult<(OutcomeRow, ContractRow)> { + let (outcome, contract) = self.outcome(index)?; + let cleaned: String = resolved_cleaned_sql(outcome) + .map_err(compiler_error)? + .into_owned(); + Ok(( + outcome_row(outcome.clone(), cleaned), + contract_row(contract.clone()), + )) + } + + /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. + #[getter] + pub fn cache_stats(&self) -> Option { + self.cache_stats.clone() + } + + /// Each model's dynamic pivot proof from the finished session's tables. + pub fn prove_dynamic_contracts( + &self, + py: Python<'_>, + models: Vec, + ) -> PyResult> { + let session: &FinishedSession = self + .finished + .as_ref() + .ok_or_else(|| internal_error("the session has not finished"))?; + let models: Vec = models.into_iter().map(pivot_model).collect(); + let outcomes: Vec = py + .compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) + .map_err(|reason| internal_error(&reason))?; + Ok(outcomes.into_iter().map(contract_row).collect()) + } + + /// `(shared members, shared members re-analysed alone)` as of the last step. + #[getter] + pub fn sharing(&self) -> (usize, usize) { + self.sharing + } +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/types.rs b/crates/sqlbuild-python-project/src/analysis_session/types.rs new file mode 100644 index 0000000000..2441ae1ff7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of analysis session. + +/// `(succeeded, [(name, type, nullability)] or None, has star)`. +pub type ColumnsRow = (bool, Option, String)>>, bool); diff --git a/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs b/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs new file mode 100644 index 0000000000..14f3a4214d --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod render; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs rename to crates/sqlbuild-python-project/src/auditing/_helpers/render.rs index 9bff3a370d..35a959547e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs +++ b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs @@ -5,9 +5,8 @@ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; -use crate::bindings::_helpers::references::source_relations::{ - render_source_read, render_source_relation, -}; +use crate::references::main::render_source_read::render_source_read; +use crate::references::main::render_source_relation::render_source_relation; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python-project/src/auditing/main/mod.rs b/crates/sqlbuild-python-project/src/auditing/main/mod.rs new file mode 100644 index 0000000000..fe7660516e --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_render; +pub mod render_audit_sql; diff --git a/crates/sqlbuild-python-project/src/auditing/main/register_render.rs b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs new file mode 100644 index 0000000000..4e007b92b6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs @@ -0,0 +1,12 @@ +//! Register the render functions and classes. + +use crate::auditing::_helpers::render; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the render functions and classes. +#[inline] +pub fn register_render(module: &Bound<'_, PyModule>) -> PyResult<()> { + render::register(module) +} diff --git a/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs new file mode 100644 index 0000000000..27a5ccd785 --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs @@ -0,0 +1,12 @@ +//! Render audit SQL from unresolved markers, applying relation overrides first. + +use crate::auditing::_helpers::render; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Render audit SQL from unresolved markers, applying relation overrides first. +#[inline] +pub fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { + render::render_audit_sql(request) +} diff --git a/crates/sqlbuild-python-project/src/auditing/mod.rs b/crates/sqlbuild-python-project/src/auditing/mod.rs new file mode 100644 index 0000000000..877ebc65aa --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/mod.rs @@ -0,0 +1,4 @@ +//! Rendering of audit SQL. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs rename to crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs index 46e83e5c42..b016e02993 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; +use crate::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -105,3 +105,10 @@ pub(crate) fn resolve_cursor_input_roles<'py>( ], ) } + +/// `resolve_cursor_input_roles` as a function of `module`. +pub(crate) fn resolve_cursor_input_roles_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_cursor_input_roles, module) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs new file mode 100644 index 0000000000..a3f498da74 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod cursor_roles; diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs new file mode 100644 index 0000000000..f8523830d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs @@ -0,0 +1,2 @@ +pub mod resolve_cursor_input_roles; +pub mod resolve_cursor_input_roles_function; diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs new file mode 100644 index 0000000000..4bed4dd2c0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs @@ -0,0 +1,12 @@ +//! Resolve authored cursor input fields into explicit effective roles. + +use crate::compile_attachment::_helpers::cursor_roles; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve authored cursor input fields into explicit effective roles. +#[inline] +pub fn resolve_cursor_input_roles<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + cursor_roles::resolve_cursor_input_roles(model) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs new file mode 100644 index 0000000000..7eecc98d17 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs @@ -0,0 +1,11 @@ +//! `resolve_cursor_input_roles` as a function of `module`. + +use crate::compile_attachment::_helpers::cursor_roles; + +/// `resolve_cursor_input_roles` as a function of `module`. +#[inline] +pub fn resolve_cursor_input_roles_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + cursor_roles::resolve_cursor_input_roles_function(module) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/mod.rs new file mode 100644 index 0000000000..66b3451082 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/mod.rs @@ -0,0 +1,4 @@ +//! Cursor roles a compile attaches to models. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs new file mode 100644 index 0000000000..c639b790aa --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs @@ -0,0 +1,77 @@ +//! The resource rows and settings the compile attaches into the project, and the installed runs. + +use crate::compiled_project::models::AuditFileRow; +use crate::compiled_project::models::AuditSettings; +use crate::compiled_project::models::DiagnosticSettings; +use crate::compiled_project::models::FunctionRow; +use crate::compiled_project::models::FunctionSettings; +use crate::compiled_project::models::ResourceAttachment; +use crate::compiled_project::models::ScenarioRow; +use crate::compiled_project::models::SqlTestSettings; +use crate::compiled_project::models::TestFileRow; +use crate::compiled_project::types::AttachedSqlTests; +use crate::compiled_project::types::ResourceDiagnostics; +use crate::compiled_project::types::SourceRow; +use crate::compiled_project::types::SourceSettings; +use std::sync::OnceLock; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::models::NativeSourceInput; + +const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; + +static INSTALLED: OnceLock = OnceLock::new(); + +/// Attaches the project's sources, given their rows and settings. +pub(crate) type AttachSources = fn( + &Bound<'_, NativeCompiledProject>, + Vec, + SourceSettings, +) -> PyResult>; + +/// Attaches the project's SQL, then Python, functions. +pub(crate) type AttachFunctions = fn( + &Bound<'_, NativeCompiledProject>, + (Vec, Vec), + FunctionSettings, +) -> PyResult>; + +/// Reads the project's resource diagnostics. +pub(crate) type ResourceDiagnosticsRun = for<'py> fn( + &NativeCompiledProject, + Python<'py>, + &DiagnosticSettings, +) -> PyResult>; + +/// Attaches the project's SQL test files, then its scenarios. +pub(crate) type AttachSqlTests = fn( + &Bound<'_, NativeCompiledProject>, + Vec, + Vec, + SqlTestSettings, +) -> PyResult; + +/// Attaches the project's audits; returns their views and the built-in shadow warnings. +pub(crate) type AttachAudits = + for<'py> fn( + &Bound<'py, NativeCompiledProject>, + Vec, + AuditSettings, + ) -> PyResult<(Vec, Vec>)>; + +/// Install the compile's attachment runs once, at registration. +pub(crate) fn install(attachment: ResourceAttachment) { + let _ = INSTALLED.get_or_init(|| attachment); +} + +/// The installed attachment runs. +pub(crate) fn resource_attachment() -> PyResult<&'static ResourceAttachment> { + INSTALLED + .get() + .ok_or_else(|| PyRuntimeError::new_err(ATTACHMENT_UNINSTALLED)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index f9b2367126..570483cbdb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -16,9 +16,9 @@ use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGrap use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; -use crate::bindings::_helpers::compiled_project::constants::{ - JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, -}; +use crate::compiled_project::constants::JOB_SELECTOR_KINDS; +use crate::compiled_project::constants::SELECTOR_INTERSECTION; +use crate::compiled_project::constants::SENSOR_RUN_ARGUMENT; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs new file mode 100644 index 0000000000..6a8ba26c5c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -0,0 +1,14 @@ +pub(crate) mod attachment_inputs; +pub(crate) mod job_catalog; +pub(crate) mod project; +pub(crate) mod rows; +pub(crate) mod schema_facts; +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_roles; +pub(crate) mod snapshot_worker; +pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs new file mode 100644 index 0000000000..844d904600 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs @@ -0,0 +1,175 @@ +//! The compile's natively owned project, which producers record into and consumers read. + +use crate::compiled_project::constants::CONTRACT_KEY; +use crate::compiled_project::constants::INCREMENTAL_MODE_KEY; +use crate::compiled_project::models::DeclarationPy; +use crate::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_base::positional_record; +use std::collections::BTreeMap; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::Value; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, DeclarationFact, ModelConfigFacts, ModelReferenceFact, +}; + +use crate::compiled_project::_helpers::values::object_value; +use crate::compiled_project::_helpers::values::plain_value; +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeCompiledAudit; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSource; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::models::NativeSeedInput; +use crate::compiled_views::models::NativeSourceInput; + +pub(crate) const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; +pub(crate) const ATTACHMENT_LOOP_UNSET: &str = + "no model loop was kept to attach this project's resources"; + +const MATERIALIZED_KEY: &str = "materialized"; +const TABLE_MATERIALIZATION: &str = "table"; + +/// `(name, type)`. +pub(crate) type NamedType = (String, Option); + +positional_record! { + extract; + /// A model's name, relative path, compiled and authored queries, and authored SQL file. + pub(crate) struct ModelTextsPy { + name: String, + relative_path: String, + query_sql: String, + authored_query_sql: String, + authored_sql: String, + } +} + +/// Encoded config values and the encoder's error, if any. +pub(crate) type ConfigValues = (BTreeMap, Option); + +/// One model as the compile-input stage produced it. +#[derive(FromPyObject)] +pub struct ModelInputPy<'py>( + ModelTextsPy, + (Bound<'py, PyDict>, Vec, Option), + Vec<(String, String, Option)>, + Vec, + (Vec>, Vec>), +); + +pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { + let ModelInputPy( + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, + (values, header_keys, layer_schema), + references, + enum_columns, + (enum_declarations, constant_declarations), + ) = model; + let config_values: ConfigValues = config_values(&values); + Ok(CompiledModelFacts { + name, + relative_path, + query_sql: query_sql.into(), + authored_query_sql: authored_query_sql.into(), + authored_sql: authored_sql.into(), + config: ModelConfigFacts { + values: config_values.0, + encoding_error: config_values.1, + header_keys, + layer_schema, + contract: text_value(&values, CONTRACT_KEY)?, + materialized: materialization(&values)?, + incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, + }, + references: references + .into_iter() + .map(|(kind, name, package)| ModelReferenceFact { + kind, + name, + package, + }) + .collect(), + schema: None, + enum_columns, + enum_declarations: declarations(enum_declarations)?, + constant_declarations: declarations(constant_declarations)?, + analysis: None, + typed_calls: Vec::new(), + expansion_passes: Vec::new(), + sql_analysis_skipped: false, + }) +} + +/// The config values, or the encoder's error, kept for the consumer that reads them. +pub(crate) fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { + match object_value(values) { + Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), + Ok(_) => (BTreeMap::new(), None), + Err(error) => (BTreeMap::new(), Some(error.to_string())), + } +} + +/// The `materialized` config value when it is text, `table` when it is absent. +pub(crate) fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { + if values.get_item(MATERIALIZED_KEY)?.is_none() { + return Ok(Some(TABLE_MATERIALIZATION.to_owned())); + } + text_value(values, MATERIALIZED_KEY) +} + +/// A config value when it is text. +pub(crate) fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +pub(crate) fn declarations(rows: Vec>) -> PyResult> { + rows.into_iter().map(declaration).collect() +} + +pub(crate) fn declaration(row: DeclarationPy<'_>) -> PyResult { + let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; + Ok(DeclarationFact { + name, + relative_path, + members: members + .into_iter() + .map(|(name, value)| Ok((name, plain_value(&value)?))) + .collect::>()?, + value: Some(plain_value(&value)?).filter(|value| !value.is_null()), + value_type, + render_as, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs index 5499367ebd..8de140100b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs @@ -1,18 +1,7 @@ //! Positional rows shared by the compiled views and the input loops that fill them. +use crate::compiled_project::models::ReferenceTuple; use sqlbuild_project::models::ReferenceRow; -use sqlbuild_python_base::positional_record; - -positional_record! { - into; - /// A reference as the tuple Python reads: kind, name, package and call argument count. - pub(crate) struct ReferenceTuple { - pub(crate) kind: String, - pub(crate) name: String, - package: Option, - call_argument_count: Option, - } -} /// A reference as the tuple Python reads. pub(crate) fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs new file mode 100644 index 0000000000..6d78d47b1a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs @@ -0,0 +1,48 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use crate::compiled_project::models::NativeOrchestrationSnapshot; +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use serde_json::Value; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::compiled_project::models::NativeOrchestrationWorker; + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.contains_key(&key) { + return Err(PyValueError::new_err(format!( + "duplicate snapshot node identity {key}" + ))); + } + nodes.insert(key, node); + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs index ffb48a60a0..f627d5815a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs @@ -12,11 +12,13 @@ use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::compiled_project::_helpers::snapshot::destination; +use crate::compiled_project::_helpers::snapshot::insert_node; +use crate::compiled_project::_helpers::values::plain_value; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::path_posix::path_posix; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_declarations.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_declarations.rs index f0d6e873c0..5148b77f30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_declarations.rs @@ -20,10 +20,10 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use crate::bindings::_helpers::native_host::orchestration_triggers::{ - CalendarRequest, compile_calendars, compile_sensors, -}; +use crate::graph::models::ProjectSelection; +use crate::native_host::main::compile_calendars::compile_calendars; +use crate::native_host::main::compile_sensors::compile_sensors; +use crate::native_host::models::CalendarRequest; use sqlbuild_python_base::orchestration_reads::main::imported_declarations::imported_declarations; use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; use sqlbuild_python_base::orchestration_reads::main::run_controls::run_controls; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs new file mode 100644 index 0000000000..1e3936ef2f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs @@ -0,0 +1,32 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use pyo3::exceptions::PyRuntimeError; +use sqlbuild_core::models::CompiledResourceType; +use sqlbuild_orchestration::runs::models::NodeOutcome; + +/// A finished node's outcome, rows affected, error code and error message. +pub(crate) type Completion = (NodeOutcome, Option, (Option, Option)); + +/// Failure outranks cancellation, then warnings, then success. +pub(crate) fn severity(outcome: NodeOutcome) -> u8 { + match outcome { + NodeOutcome::Skipped => 0, + NodeOutcome::Success => 1, + NodeOutcome::Warn => 2, + NodeOutcome::Cancelled => 3, + NodeOutcome::Failed => 4, + } +} + +/// Whether a snapshot node key names a scalar or table function. +pub(crate) fn is_function(key: &str) -> bool { + key.split_once(':').is_some_and(|(kind, _)| { + [CompiledResourceType::Udf, CompiledResourceType::TableFn] + .iter() + .any(|function| function.as_str() == kind) + }) +} + +pub(crate) fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs index 76bdb01204..b27f86c1c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{ - dependency_keys, destination, insert_node, -}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::compiled_project::_helpers::snapshot::dependency_keys; +use crate::compiled_project::_helpers::snapshot::destination; +use crate::compiled_project::_helpers::snapshot::insert_node; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::types::ModelInputRecord; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs index 959edffc0e..91917d39d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs @@ -7,16 +7,17 @@ use serde_json::{Value, json}; use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{ - dependency_keys, destination, insert_node, -}; -use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ - boundary_nodes, columns, metadata, relative_path, -}; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::compiled_project::_helpers::snapshot::dependency_keys; +use crate::compiled_project::_helpers::snapshot::destination; +use crate::compiled_project::_helpers::snapshot::insert_node; +use crate::compiled_project::_helpers::snapshot_boundary::boundary_nodes; +use crate::compiled_project::_helpers::snapshot_boundary::columns; +use crate::compiled_project::_helpers::snapshot_boundary::metadata; +use crate::compiled_project::_helpers::snapshot_boundary::relative_path; +use crate::compiled_project::_helpers::values::plain_value; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeProjectView; pub(crate) fn resource_nodes( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_roles.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_roles.rs index 895bfa9eb9..0ffea4986f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_roles.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; +use crate::graph::models::ProjectSelection; use sqlbuild_python_base::native_host::main::write_console::write_console; /// The project-root file that defines project roles. diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs new file mode 100644 index 0000000000..e8733cbed4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs @@ -0,0 +1,24 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use pyo3::exceptions::PyRuntimeError; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; + +#[derive(Debug)] +pub(crate) struct WorkerOwner { + pub(crate) runtime: tokio::runtime::Runtime, + pub(crate) client: WorkerClient, + pub(crate) session: WorkerSession, +} + +pub(crate) fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + pub(crate) fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs rename to crates/sqlbuild-python-project/src/compiled_project/constants.rs index a265293a21..3402663812 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -1,16 +1,26 @@ -//! Frozen execution protocol values and bounded acknowledgement retries. +//! Public constants of compiled project. pub(crate) const SKIP_ACTION: &str = "skip"; + pub(crate) const NO_CHANGE_REASON: &str = "no_change"; + pub(crate) const REPORT_ATTEMPTS: usize = 3; + /// The HTTP status a control server answers while its store is busy; reports retry it. pub(crate) const STORE_BUSY_STATUS: u16 = 503; + /// Selector kinds that can name a job: `job:` and a bare name. pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; + /// Separates the parts of an intersection selector. pub(crate) const SELECTOR_INTERSECTION: char = ','; + /// The key prefix of a SQL unit test's result, as the snapshot names it. -pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; +pub const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; /// The sensor argument naming the job it starts, as the orchestration inventory labels it. pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; + +pub const CONTRACT_KEY: &str = "contract"; + +pub const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs b/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs new file mode 100644 index 0000000000..1892d7130b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs @@ -0,0 +1,10 @@ +//! Install the compile's attachment runs once, at registration. + +use crate::compiled_project::_helpers::attachment_inputs; +use crate::compiled_project::models::ResourceAttachment; + +/// Install the compile's attachment runs once, at registration. +#[inline] +pub fn install(attachment: ResourceAttachment) { + attachment_inputs::install(attachment) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs new file mode 100644 index 0000000000..e5b1d65b77 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs @@ -0,0 +1 @@ +pub mod install; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs new file mode 100644 index 0000000000..15f2a698f6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs @@ -0,0 +1,18 @@ +//! `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. + +use crate::compiled_project::_helpers::job_catalog; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::graph::models::ProjectGraph; +use std::borrow::Cow; + +/// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. +#[inline] +pub fn graph_with_jobs<'g>( + py: Python<'_>, + graph: &'g ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult> { + job_catalog::graph_with_jobs(py, graph, selectors, also_known) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs new file mode 100644 index 0000000000..9781461c25 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs @@ -0,0 +1,11 @@ +//! The command's job and job group names, loaded on first use; empty outside a command. + +use crate::compiled_project::_helpers::job_catalog; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The command's job and job group names, loaded on first use; empty outside a command. +#[inline] +pub fn job_names(py: Python<'_>) -> PyResult> { + job_catalog::job_names(py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs new file mode 100644 index 0000000000..dd6ef73702 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs @@ -0,0 +1,5 @@ +pub mod graph_with_jobs; +pub mod job_names; +pub mod needed_jobs; +pub mod triggered_jobs; +pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs new file mode 100644 index 0000000000..ebeb109627 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs @@ -0,0 +1,19 @@ +//! The command's jobs when `selectors` name one and `graph` has none attached yet. + +use crate::compiled_project::_helpers::job_catalog; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::graph::models::JobSelection; +use sqlbuild_analysis::graph::models::ProjectGraph; +use std::collections::BTreeMap; + +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +#[inline] +pub fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + job_catalog::needed_jobs(py, graph, selectors, also_known) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs new file mode 100644 index 0000000000..39e3153282 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs @@ -0,0 +1,18 @@ +//! Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. + +use crate::compiled_project::_helpers::job_catalog; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::models::ProjectGraph; +use std::collections::BTreeSet; + +/// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. +#[inline] +pub fn triggered_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], +) -> PyResult)>> { + job_catalog::triggered_jobs(py, graph, selectors) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs new file mode 100644 index 0000000000..bc8fde79ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs @@ -0,0 +1,13 @@ +//! Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. + +use crate::compiled_project::_helpers::job_catalog; +use std::path::PathBuf; + +/// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. +#[inline] +pub fn with_job_scope( + (project_dir, selected_target): (PathBuf, Option), + run: impl FnOnce() -> T, +) -> T { + job_catalog::with_job_scope((project_dir, selected_target), run) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs new file mode 100644 index 0000000000..5a36812609 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -0,0 +1,12 @@ +pub mod attachment_inputs; +pub mod job_catalog; +pub mod project; +pub mod rows; +pub mod schema_facts; +pub mod snapshot; +pub mod snapshot_boundary; +pub mod snapshot_declarations; +pub mod snapshot_models; +pub mod snapshot_resources; +pub mod snapshot_roles; +pub mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs new file mode 100644 index 0000000000..2574a1758a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs @@ -0,0 +1,12 @@ +//! The config values, or the encoder's error, kept for the consumer that reads them. + +use crate::compiled_project::_helpers::project; +use crate::compiled_project::_helpers::project::ConfigValues; +use pyo3::prelude::Bound; +use pyo3::types::PyDict; + +/// The config values, or the encoder's error, kept for the consumer that reads them. +#[inline] +pub fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { + project::config_values(values) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs new file mode 100644 index 0000000000..dcccc7d386 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs @@ -0,0 +1,12 @@ +//! Declarations. + +use crate::compiled_project::_helpers::project; +use crate::compiled_project::models::DeclarationPy; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::DeclarationFact; + +/// Declarations. +#[inline] +pub fn declarations(rows: Vec>) -> PyResult> { + project::declarations(rows) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs new file mode 100644 index 0000000000..8b4d7256f3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs @@ -0,0 +1,12 @@ +//! The `materialized` config value when it is text, `table` when it is absent. + +use crate::compiled_project::_helpers::project; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The `materialized` config value when it is text, `table` when it is absent. +#[inline] +pub fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { + project::materialization(values) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs new file mode 100644 index 0000000000..0173ee2d17 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs @@ -0,0 +1,6 @@ +pub mod config_values; +pub mod declarations; +pub mod materialization; +pub mod model_facts; +pub mod register_project; +pub mod text_value; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs new file mode 100644 index 0000000000..8d164ea87a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs @@ -0,0 +1,12 @@ +//! Model facts. + +use crate::compiled_project::_helpers::project; +use crate::compiled_project::_helpers::project::ModelInputPy; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; + +/// Model facts. +#[inline] +pub fn model_facts(model: ModelInputPy<'_>) -> PyResult { + project::model_facts(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs new file mode 100644 index 0000000000..d3070cd2e3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs @@ -0,0 +1,12 @@ +//! Register the project functions and classes. + +use crate::compiled_project::_helpers::project; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the project functions and classes. +#[inline] +pub fn register_project(module: &Bound<'_, PyModule>) -> PyResult<()> { + project::register(module) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs new file mode 100644 index 0000000000..7f4fe85c39 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs @@ -0,0 +1,12 @@ +//! A config value when it is text. + +use crate::compiled_project::_helpers::project; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// A config value when it is text. +#[inline] +pub fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + project::text_value(values, key) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs new file mode 100644 index 0000000000..8aaca236c8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs @@ -0,0 +1 @@ +pub mod reference_row_tuple; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs b/crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs new file mode 100644 index 0000000000..43465df81d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs @@ -0,0 +1,11 @@ +//! A reference as the tuple Python reads. + +use crate::compiled_project::_helpers::rows; +use crate::compiled_project::models::ReferenceTuple; +use sqlbuild_project::models::ReferenceRow; + +/// A reference as the tuple Python reads. +#[inline] +pub fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { + rows::reference_row_tuple(row) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs new file mode 100644 index 0000000000..feeedc6cf6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs @@ -0,0 +1,16 @@ +//! The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. + +use crate::compiled_project::_helpers::schema_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; + +/// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. +#[inline] +pub fn entry_schema_fact( + entry: &Bound<'_, PyAny>, + model_path: &Bound<'_, PyAny>, +) -> PyResult> { + schema_facts::entry_schema_fact(entry, model_path) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs new file mode 100644 index 0000000000..16da422dc4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs @@ -0,0 +1,11 @@ +//! A parsed dynamic column family's fact, keyed by its casefolded name. + +use crate::compiled_project::_helpers::schema_facts; +use sqlbuild_analysis::compiled_project::models::DynamicFamilyFact; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +/// A parsed dynamic column family's fact, keyed by its casefolded name. +#[inline] +pub fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { + schema_facts::family_fact(family) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs new file mode 100644 index 0000000000..cd7255834c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs @@ -0,0 +1,4 @@ +pub mod entry_schema_fact; +pub mod family_fact; +pub mod python_column_fact; +pub mod python_column_facts; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs new file mode 100644 index 0000000000..19cdddf230 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs @@ -0,0 +1,16 @@ +//! A Python column spec's fact; its location is kept when it differs from `model_path`. + +use crate::compiled_project::_helpers::schema_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; + +/// A Python column spec's fact; its location is kept when it differs from `model_path`. +#[inline] +pub fn python_column_fact( + column: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult { + schema_facts::python_column_fact(column, model_path) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs new file mode 100644 index 0000000000..9f2a1d9d72 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs @@ -0,0 +1,16 @@ +//! The facts of every column spec in `columns`. + +use crate::compiled_project::_helpers::schema_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; + +/// The facts of every column spec in `columns`. +#[inline] +pub fn python_column_facts( + columns: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + schema_facts::python_column_facts(columns, model_path) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs new file mode 100644 index 0000000000..13bea31dc0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs @@ -0,0 +1 @@ +pub mod register_snapshot; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs new file mode 100644 index 0000000000..6889acfb6f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs @@ -0,0 +1,12 @@ +//! Register the snapshot functions and classes. + +use crate::compiled_project::_helpers::snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the snapshot functions and classes. +#[inline] +pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { + snapshot::register(module) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs new file mode 100644 index 0000000000..4205cea05f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs @@ -0,0 +1 @@ +pub mod relative_path; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs new file mode 100644 index 0000000000..48158c1483 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs @@ -0,0 +1,12 @@ +//! Relative path. + +use crate::compiled_project::_helpers::snapshot_boundary; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Relative path. +#[inline] +pub fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + snapshot_boundary::relative_path(value) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs new file mode 100644 index 0000000000..3fef29043d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs @@ -0,0 +1,28 @@ +//! Compile declarations. + +use crate::compiled_project::_helpers::snapshot_declarations; +use crate::graph::models::ProjectSelection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use std::path::Path; + +/// Compile declarations. +#[inline] +pub fn compile_declarations( + py: Python<'_>, + input: SnapshotInput, + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), + (inventory, declarations): (&OrchestrationInventory, Vec>), +) -> PyResult { + snapshot_declarations::compile_declarations( + py, + input, + (project_dir, graph, selected), + (inventory, declarations), + ) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs new file mode 100644 index 0000000000..f30195d675 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs @@ -0,0 +1,2 @@ +pub mod compile_declarations; +pub mod python_declarations; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs new file mode 100644 index 0000000000..2253891986 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs @@ -0,0 +1,19 @@ +//! The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. + +use crate::compiled_project::_helpers::snapshot_declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use std::path::Path; + +/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. +#[inline] +pub fn python_declarations<'py>( + py: Python<'py>, + project_dir: &Path, + inventory: &OrchestrationInventory, +) -> PyResult>> { + snapshot_declarations::python_declarations(py, project_dir, inventory) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs new file mode 100644 index 0000000000..8a1804d90e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs @@ -0,0 +1 @@ +pub mod model_nodes; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs new file mode 100644 index 0000000000..522e48c1d9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs @@ -0,0 +1,22 @@ +//! Each model's snapshot node, and the `columns` document of ranges kept out of node identity. + +use crate::compiled_project::_helpers::snapshot_models; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use serde_json::Map; +use serde_json::Value; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use std::collections::BTreeMap; + +/// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. +#[inline] +pub fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + nodes: BTreeMap, +) -> PyResult<(BTreeMap, Map)> { + snapshot_models::model_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs new file mode 100644 index 0000000000..2b4f92fed1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs @@ -0,0 +1 @@ +pub mod resource_nodes; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs new file mode 100644 index 0000000000..165cdc17f6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs @@ -0,0 +1,20 @@ +//! Resource nodes. + +use crate::compiled_project::_helpers::snapshot_resources; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use std::collections::BTreeMap; + +/// Resource nodes. +#[inline] +pub fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + nodes: BTreeMap, +) -> PyResult> { + snapshot_resources::resource_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs new file mode 100644 index 0000000000..f165383417 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs @@ -0,0 +1,19 @@ +//! Compile project roles. + +use crate::compiled_project::_helpers::snapshot_roles; +use crate::graph::models::ProjectSelection; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use std::path::Path; + +/// Compile project roles. +#[inline] +pub fn compile_project_roles( + py: Python<'_>, + input: SnapshotInput, + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), +) -> PyResult { + snapshot_roles::compile_project_roles(py, input, (project_dir, graph, selected)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs new file mode 100644 index 0000000000..6dbdce4266 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs @@ -0,0 +1 @@ +pub mod compile_project_roles; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs new file mode 100644 index 0000000000..a9728688b6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs @@ -0,0 +1 @@ +pub mod plain_value; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs new file mode 100644 index 0000000000..73944e7583 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs @@ -0,0 +1,13 @@ +//! orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. + +use crate::compiled_project::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use serde_json::Value; + +/// orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. +#[inline] +pub fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { + values::plain_value(value) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/mod.rs new file mode 100644 index 0000000000..67c7ce0488 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/mod.rs @@ -0,0 +1,7 @@ +//! The compiled project: its rows, schema facts, job catalog and the snapshot written and read across processes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs new file mode 100644 index 0000000000..8a360d9455 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -0,0 +1,2003 @@ +//! Public records of compiled project. + +use crate::compiled_project::_helpers::attachment_inputs::AttachAudits; +use crate::compiled_project::_helpers::attachment_inputs::AttachFunctions; +use crate::compiled_project::_helpers::attachment_inputs::AttachSources; +use crate::compiled_project::_helpers::attachment_inputs::AttachSqlTests; +use crate::compiled_project::_helpers::attachment_inputs::ResourceDiagnosticsRun; +use crate::compiled_project::_helpers::attachment_inputs::resource_attachment; +use crate::compiled_project::_helpers::project::ATTACHMENT_LOOP_UNSET; +use crate::compiled_project::_helpers::project::INPUT_CLASSES_UNSET; +use crate::compiled_project::_helpers::project::NamedType; +use crate::compiled_project::_helpers::snapshot_execution::Completion; +use crate::compiled_project::_helpers::snapshot_execution::is_function; +use crate::compiled_project::_helpers::snapshot_execution::poisoned; +use crate::compiled_project::_helpers::snapshot_execution::severity; +use crate::compiled_project::_helpers::snapshot_worker::WorkerOwner; +use crate::compiled_project::_helpers::snapshot_worker::unavailable; +use crate::compiled_project::constants::NO_CHANGE_REASON; +use crate::compiled_project::constants::REPORT_ATTEMPTS; +use crate::compiled_project::constants::SKIP_ACTION; +use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use crate::compiled_project::constants::STORE_BUSY_STATUS; +use crate::compiled_project::types::AttachedSqlTests; +use crate::compiled_project::types::LoaderRow; +use crate::compiled_project::types::NodeKeys; +use crate::compiled_project::types::ResourceDiagnostics; +use crate::compiled_project::types::SourceRow; +use crate::compiled_project::types::SourceSettings; +use crate::compiled_project::types::TestCaseRow; +use crate::compiled_views::main::audit_views::audit_record::audit_record; +use crate::compiled_views::main::function_views::function_record::function_record; +use crate::compiled_views::main::seed_views::paired_seeds::paired_seeds; +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeCompiledAudit; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSource; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::models::NativeSeedInput; +use crate::compiled_views::models::NativeSourceInput; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::models::SeedNamespace; +use crate::compiled_views::types::AttachedAudit; +use crate::compiled_views::types::AttachedFunction; +use crate::compiled_views::types::AuditRecord; +use crate::compiled_views::types::FunctionRecord; +use crate::compiled_views::types::ModelInputRecord; +use crate::compiled_views::types::SeedRecord; +use crate::compiled_views::types::SeedViews; +use crate::compiled_views::types::SourceRecord; +use pyo3::FromPyObject; +use pyo3::exceptions::PyIndexError; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use sqlbuild_analysis::compiled_project::main::project_models::project_models; +use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; +use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; +use sqlbuild_analysis::compiled_project::models::DynamicProofFact; +use sqlbuild_analysis::compiled_project::models::ModelAnalysisFact; +use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use sqlbuild_host::models::SqlTestOutcome; +use sqlbuild_host::types::ResultEnum; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::runs::models::NodeOutcome; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::runs::models::RunCommand; +use sqlbuild_orchestration::runs::models::RunExecution; +use sqlbuild_orchestration::runs::models::RunOutcome; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_project::models::CompiledProject; +use sqlbuild_project::types::InferredRow; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_project::types::SourceNamespace; +use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use sqlbuild_python_base::cache::models::SqlTestScanStore; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_front::render::models::ConstantRendering; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use uuid::Uuid; + +/// A function's paths, stem, discovered file, header values, body and SQL file contents. +#[derive(Debug)] +pub struct FunctionRow { + pub file_path: String, + pub relative_path: String, + pub name: String, + pub function_file: Py, + pub header_values: Py, + pub body: String, + pub contents: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for FunctionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, name, function_file, header_values, body, contents): ( + String, + String, + String, + Py, + Py, + String, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + name, + function_file, + header_values, + body, + contents, + }) + } +} + +/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. +#[derive(Debug)] +pub struct FunctionSettings { + pub adapter_name: String, + pub validate: bool, + pub defaults: (Option, Option), + pub target: Option<(Option, Option)>, + pub inherit: bool, + pub kind: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for FunctionSettings { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (adapter_name, validate, defaults, target, inherit, kind): ( + String, + bool, + (Option, Option), + Option<(Option, Option)>, + bool, + Py, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + adapter_name, + validate, + defaults, + target, + inherit, + kind, + }) + } +} + +/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. +#[derive(Debug)] +pub struct TestBlockRow { + pub block: Py, + pub name: Option, + pub sql_body: String, + pub mode: String, + pub parameters: Vec, + pub cases: Vec, + pub sql_body_span: Option<(usize, usize)>, +} + +impl<'py> pyo3::FromPyObject<'py> for TestBlockRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (block, name, sql_body, mode, parameters, cases, sql_body_span): ( + Py, + Option, + String, + String, + Vec, + Vec, + Option<(usize, usize)>, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + block, + name, + sql_body, + mode, + parameters, + cases, + sql_body_span, + }) + } +} + +/// A test file's paths, stem, discovered file, contents and blocks. +#[derive(Debug)] +pub struct TestFileRow { + pub file_path: String, + pub relative_path: String, + pub stem: String, + pub file: Py, + pub contents: String, + pub blocks: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for TestFileRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, stem, file, contents, blocks): ( + String, + String, + String, + Py, + String, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + stem, + file, + contents, + blocks, + }) + } +} + +/// What every SQL test and scenario of one compile resolves and records with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct SqlTestSettings { + /// `CompileSqlTestInput`, its model and direct payloads, and `CompileSqlTestCte`. + pub test_input: Py, + pub model_payload: Py, + pub direct_payload: Py, + pub test_cte: Py, + /// `CompileSqlScenarioInput` and `CompileSqlScenarioCte`. + pub scenario_input: Py, + pub scenario_cte: Py, + /// `SqlTestMode` by value. + pub modes: Py, + /// The `ResourceKind`s of tests and scenarios, and the runtime `UsageKind`. + pub test_kind: Py, + pub scenario_kind: Py, + pub runtime_usage: Py, + /// The adapter's typed scalar renderer, returning `(text, None)` or `(None, error)`. + pub render_parameter: Py, + pub adapter_name: String, + /// `SourceLocation`, for helper problems located natively. + pub source_location: Py, + /// The shared scan store with the scan algorithm and lexical rules keying it, if any. + pub scan_store: Option>, + pub scan_key: (String, String), +} + +/// A scenario file's paths, discovered file, contents, SQL body and name. +#[derive(Debug)] +pub struct ScenarioRow { + pub file_path: String, + pub relative_path: String, + pub scenario_file: Py, + pub contents: String, + pub sql_body: String, + pub name: String, +} + +impl<'py> pyo3::FromPyObject<'py> for ScenarioRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, scenario_file, contents, sql_body, name): ( + String, + String, + Py, + String, + String, + String, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + scenario_file, + contents, + sql_body, + name, + }) + } +} + +/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. +#[derive(Debug)] +pub struct AuditBlockRow { + pub block: Py, + pub name: Option, + pub sql_body: String, + pub evidence_sql: Option, + pub measurement: bool, + pub header: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for AuditBlockRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (block, name, sql_body, evidence_sql, measurement, header): ( + Py, + Option, + String, + Option, + bool, + Py, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + block, + name, + sql_body, + evidence_sql, + measurement, + header, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for AuditBlockRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.block, + self.name, + self.sql_body, + self.evidence_sql, + self.measurement, + self.header, + ), + py, + ) + } +} + +/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. +#[derive(Debug)] +pub struct AuditFileRow { + pub file_path: String, + pub relative_path: String, + pub stem: String, + pub file: Py, + pub generic: bool, + pub contents: String, + pub blocks: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for AuditFileRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, stem, file, generic, contents, blocks): ( + String, + String, + String, + Py, + bool, + String, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + stem, + file, + generic, + contents, + blocks, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for AuditFileRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.file_path, + self.relative_path, + self.stem, + self.file, + self.generic, + self.contents, + self.blocks, + ), + py, + ) + } +} + +/// What every audit in one compile resolves against, and the Python values it builds with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct AuditSettings { + pub default_severity: Option, + pub default_run_scope: Option, + /// Built-in generic definitions as audit file rows, overridden by project definitions. + pub builtins: Vec, + /// `DeclarationIdentity`, and the singular and generic audit `DeclarationKind`s. + pub declaration_identity: Py, + pub singular_kind: Py, + pub audit_kind: Py, + /// `ResourceKind` by value, for the resources singular audits read and attach to. + pub resource_kinds: Py, +} + +/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. +#[derive(Debug)] +pub struct PythonResourceRow { + pub kind: String, + pub name: String, + pub description: Option>, + pub location: Py, + pub help: String, +} + +impl<'py> pyo3::FromPyObject<'py> for PythonResourceRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (kind, name, description, location, help): ( + String, + String, + Option>, + Py, + String, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + kind, + name, + description, + location, + help, + }) + } +} + +/// A scenario's name, relative path, contents and authored header description. +#[derive(Debug)] +pub struct ScenarioDescriptionRow { + pub name: String, + pub path: String, + pub contents: String, + pub raw: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for ScenarioDescriptionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, path, contents, raw): (String, String, String, Py) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + path, + contents, + raw, + }) + } +} + +/// A SQL hook file's name, relative path, contents and description. +#[derive(Debug)] +pub struct HookDescriptionRow { + pub name: String, + pub path: String, + pub contents: String, + pub description: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for HookDescriptionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, path, contents, description): (String, String, String, Option) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + path, + contents, + description, + }) + } +} + +/// What the resource diagnostics read beyond the owner, and the Python values they build. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct DiagnosticSettings { + /// `CompilerDiagnostic`, `SourceLocation`, `Path` and `CompileInputError`. + pub compiler_diagnostic: Py, + pub compile_error: Py, + pub source_location: Py, + pub path_class: Py, + /// Every discovered model file, whose headers attach audit factories. + pub model_files: Vec>, + /// Each discovered audit factory's name, relative path and line. + pub audit_factories: Vec<(String, String, usize)>, + /// Each scenario's name, relative path, contents and authored header description. + pub scenarios: Vec, + /// Each SQL hook file's name, relative path, contents and description. + pub sql_hooks: Vec, + /// Each loader by name. + pub loaders: HashMap, + /// Python-defined resources: kind, name, description, location and help, read by Python. + pub python_resources: Vec, + /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. + pub description_limit: (usize, i64), +} + +/// The compile's runs behind the project's attachment methods. +#[derive(Clone, Copy, Debug)] +pub struct ResourceAttachment { + pub sources: AttachSources, + pub functions: AttachFunctions, + pub diagnostics: ResourceDiagnosticsRun, + pub sql_tests: AttachSqlTests, + pub audits: AttachAudits, +} + +/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. +#[derive(Debug)] +pub struct ProofPy { + pub output_proven: bool, + pub bare_dynamic_pivot: bool, + pub families: Vec, + pub failure_reason: Option, + pub named_families: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for ProofPy { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (output_proven, bare_dynamic_pivot, families, failure_reason, named_families): ( + bool, + bool, + Vec, + Option, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }) + } +} + +/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. +#[derive(Debug)] +pub struct AnalysisPy { + pub inferred: bool, + pub inferred_columns: Vec, + pub nullable_columns: Vec, + pub fast_lineage_has_star: bool, + pub unchecked: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for AnalysisPy { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked): ( + bool, + Vec, + Vec, + bool, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + }) + } +} + +/// Name, relative path, members, value, value type and rendering of one declaration. +#[derive(Debug, FromPyObject)] +pub struct DeclarationPy<'py>( + pub(crate) String, + pub(crate) String, + pub(crate) Vec<(String, Bound<'py, PyAny>)>, + pub(crate) Bound<'py, PyAny>, + pub(crate) Option, + pub(crate) Option, +); + +/// One compile's project, recorded by the stages that produce it, with its Python projections. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug, Default)] +pub struct NativeCompiledProject { + pub(crate) project: CompiledProject, + /// The Python classes model input views project their attributes with. + pub(crate) input_classes: Option>, + /// Shared immutable column projections, allocated only when Python consumers request them. + pub(crate) inferred_column_projections: Mutex>>, + /// The model loop and its constant rendering, which attach the project's other resources. + pub(crate) attachment: Option<(Py, Py)>, +} + +impl NativeCompiledProject { + /// The facts recorded so far, for native consumers such as the rules request. + pub fn facts(&self) -> &CompiledProjectFacts { + &self.project.facts + } + + /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. + pub fn with_facts(facts: CompiledProjectFacts) -> Self { + Self { + project: CompiledProject::with_facts(facts), + ..Self::default() + } + } + + /// Retain facts a producer finished natively. + pub fn retain(&mut self, facts: CompiledModelFacts) { + self.project.retain(facts); + } + + /// Retain a finished model's input record; return its index for the model's view. + pub fn retain_input(&mut self, record: ModelInputRecord) -> usize { + self.project.retain_input(record) + } + + /// Keep the classes model input views project their attributes with. + pub fn set_input_classes(&mut self, classes: Py) { + self.input_classes = Some(classes); + } + + /// Count one lineage graph a project view built. + pub fn record_graph_build(&mut self) { + self.project.graph_builds += 1; + } + + /// The kept model loop and its constant rendering. + pub fn attachment_loop(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + match &self.attachment { + Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), + None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), + } + } + + /// Keep the model loop and its constant rendering for attaching the project's other resources. + pub fn keep_attachment_loop(&mut self, model_loop: Py, host: Py) { + self.attachment = Some((model_loop, host)); + } + + /// Retain attached functions; return the index of the first. + pub fn extend_functions(&mut self, attached: Vec) -> usize { + let first: usize = self.project.functions.len(); + self.project + .functions + .extend(attached.into_iter().map(function_record)); + first + } + + /// Retain attached audits; return the index of the first. + pub fn extend_audits(&mut self, attached: Vec) -> usize { + let first: usize = self.project.audits.len(); + self.project + .audits + .extend(attached.into_iter().map(audit_record)); + first + } + + /// Every attached audit, in attachment order. + pub fn audits(&self) -> &[AuditRecord] { + self.project.audits() + } + + /// Every attached function, in attachment order. + pub fn functions(&self) -> &[FunctionRecord] { + self.project.functions() + } + + /// Every attached source, in attachment order. + pub fn sources(&self) -> &[SourceRecord] { + self.project.sources() + } + + /// Keep one attached source. + pub fn retain_source(&mut self, source: SourceRecord) { + self.project.sources.push(source); + } + + /// Every attached seed, in attachment order. + pub fn seeds(&self) -> &[SeedRecord] { + self.project.seeds() + } + + /// The schema fact recorded for an input record's model. + pub fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { + self.project.input_schema_fact(record) + } + + /// Every model's input record, in loop order. + pub fn inputs(&self) -> &[ModelInputRecord] { + &self.project.inputs + } + + /// The input record at `index`. + pub fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { + self.project + .input(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// The input record at `index`, for recording what later stages finish. + pub fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { + self.project + .inputs + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// Record that the native parser skipped the model at `index`: it is not analysed. + pub fn mark_analysis_skipped(&mut self, index: usize) -> PyResult<()> { + let record: &mut ModelInputRecord = self.input_mut(index)?; + record.sql_validation_enabled = false; + let name: String = record.name.clone(); + record_skipped_analysis(&mut self.project.facts, &name); + Ok(()) + } + + /// The classes model input views project their attributes with. + pub fn input_classes<'py>(&self, py: Python<'py>) -> PyResult> { + self.input_classes + .as_ref() + .map(|classes| classes.bind(py).clone()) + .ok_or_else(|| PyIndexError::new_err(INPUT_CLASSES_UNSET)) + } + + pub fn inferred_column<'py>( + &self, + py: Python<'py>, + classes: &CompiledViewClasses<'py>, + row: &InferredRow, + ) -> PyResult> { + let (name, column_type, nullability) = row; + if let Some(column) = self + .inferred_column_projections + .lock() + .map_err(|error| native_compiler_error(error.to_string()))? + .get(row) + { + return Ok(column.bind(py).clone()); + } + let fields = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("type", column_type.as_deref())?; + fields.set_item("nullability", classes.nullability.call1((nullability,))?)?; + let column = classes.inferred_column.call((), Some(&fields))?; + self.inferred_column_projections + .lock() + .map_err(|error| native_compiler_error(error.to_string()))? + .insert(row.clone(), column.clone().unbind()); + Ok(column) + } +} + +#[pymethods] +impl NativeCompiledProject { + #[new] + pub(crate) fn new() -> Self { + Self::default() + } + + /// Attach a recorded model's typed analysis; `False` when that model was never recorded. + pub fn record_model_analysis( + &mut self, + name: &str, + columns: AnalysisPy, + dynamic_proof: Option, + ) -> bool { + let AnalysisPy { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + } = columns; + let analysis: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof: dynamic_proof.map( + |ProofPy { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }| DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }, + ), + inferred, + nullable_columns, + fast_lineage_has_star, + unchecked_output_columns: unchecked, + }; + record_model_analysis(&mut self.project.facts, name, analysis) + } + + /// Record each assembled native model view's final typed analysis; return how many. + pub fn record_view_analyses( + slf: &Bound<'_, Self>, + views: Vec>, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let mut facts: Vec<(String, ModelAnalysisFact)> = Vec::with_capacity(views.len()); + for view in &views { + facts.push(view.analysis_fact(py)?); + } + let mut owner = slf.borrow_mut(); + let mut recorded: usize = 0; + for (name, analysis) in facts { + recorded += usize::from(record_model_analysis( + &mut owner.project.facts, + &name, + analysis, + )); + } + Ok(recorded) + } + + /// The compile inputs view of this owner's model inputs with these input fields. + pub fn inputs_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { + NativeInputsView::new(slf.unbind(), &fields) + } + + /// The compiled project view of this owner's models with these project fields. + pub fn project_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { + NativeProjectView::new(slf.unbind(), &fields) + } + + /// Pair seed declarations with their CSV files and keep them; return each seed's view. + pub fn attach_seeds( + slf: &Bound<'_, Self>, + declarations: Vec<(Py, Py)>, + seed_files: Vec>, + compile_error: &Bound<'_, PyAny>, + ) -> PyResult> { + let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; + let count: usize = seeds.len(); + slf.borrow_mut().project.seeds = seeds; + Ok((0..count) + .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) + .collect()) + } + + /// Keep the destinations native project assembly resolved, in seed order. + pub(crate) fn record_seed_namespaces( + &mut self, + namespaces: Vec, + ) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.project + .seeds + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// Attach every discovered source in order with the kept loop; return each source's view. + pub fn attach_sources( + slf: &Bound<'_, Self>, + rows: Vec, + settings: SourceSettings, + ) -> PyResult> { + (resource_attachment()?.sources)(slf, rows, settings) + } + + /// Attach every SQL, then Python, function with the kept loop; return each one's view. + pub fn attach_functions( + slf: &Bound<'_, Self>, + rows: (Vec, Vec), + settings: FunctionSettings, + ) -> PyResult> { + (resource_attachment()?.functions)(slf, rows, settings) + } + + /// The orphaned audit factory warnings, missing descriptions and cursor-input errors. + pub(crate) fn resource_diagnostics<'py>( + &self, + py: Python<'py>, + settings: DiagnosticSettings, + ) -> PyResult> { + (resource_attachment()?.diagnostics)(self, py, &settings) + } + + /// Attach every SQL test case, then every scenario, with the kept loop. + pub fn attach_sql_tests( + slf: &Bound<'_, Self>, + files: Vec, + scenarios: Vec, + settings: SqlTestSettings, + ) -> PyResult { + (resource_attachment()?.sql_tests)(slf, files, scenarios, settings) + } + + /// Attach singular, then resource-attached, audits; return views and override warnings. + pub fn attach_audits<'py>( + slf: &Bound<'py, Self>, + files: Vec, + settings: AuditSettings, + ) -> PyResult<(Vec, Vec>)> { + (resource_attachment()?.audits)(slf, files, settings) + } + /// Keep the scope dependency keys native project assembly resolved, in audit order. + pub(crate) fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.project + .audits + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_scope_deps(keys); + } + Ok(()) + } + + /// The assembled audit at `index`. + pub(crate) fn compiled_audit( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.audits.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) + } + + /// Keep the dependency keys native project assembly resolved, in function order. + pub(crate) fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.project + .functions + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_deps(keys); + } + Ok(()) + } + + /// The assembled function at `index`. + pub(crate) fn compiled_function( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.functions.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) + } + + /// Keep the namespaces native project assembly resolved, in source order. + pub(crate) fn record_source_namespaces( + &mut self, + namespaces: Vec, + ) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.project + .sources + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// The assembled source at `index`. + pub(crate) fn compiled_source( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.sources.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) + } + + /// The assembled seed at `index`. + pub(crate) fn compiled_seed( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.seeds.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) + } + + /// How many lineage graphs this owner's project views built. + pub(crate) fn graph_builds(&self) -> usize { + self.project.graph_builds + } + + /// Whether both projects retained the same facts. + pub(crate) fn __eq__(&self, other: &Self) -> bool { + self.project.facts == other.project.facts + } + + /// The recorded models' names, in production order. + pub(crate) fn model_names(&self) -> Vec { + project_models(&self.project.facts) + .iter() + .map(|model| model.name.clone()) + .collect() + } +} + +/// A reference as the tuple Python reads: kind, name, package and call argument count. +#[derive(Debug)] +pub struct ReferenceTuple { + pub kind: String, + pub name: String, + pub package: Option, + pub call_argument_count: Option, +} + +impl<'py> pyo3::IntoPyObject<'py> for ReferenceTuple { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + (self.kind, self.name, self.package, self.call_argument_count), + py, + ) + } +} + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationSnapshot { + pub input: SnapshotInput, +} + +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + pub(crate) fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + pub(crate) fn publish( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + py.detach(|| { + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + pub(crate) fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } +} + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub struct WorkerBuild { + pub worker: Py, + pub(crate) execution: RunExecution, + pub(crate) completed: Mutex>, + pub(crate) checks: Mutex>>, + /// The snapshot's audits and SQL tests, each with the node keys it checks. + pub(crate) attached: BTreeMap>, + /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. + pub(crate) blocked: BTreeMap>, + /// How many of each function's planned tests have still to report; a function is built before its tests run. + pub(crate) untested: Mutex>, + /// A built function's result, held until its tests report so a failing one can fail it instead. + pub(crate) held: Mutex>, + /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. + pub(crate) upstream: BTreeMap>, + /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. + pub(crate) causes: Mutex>, + /// The worst attached outcome of each node, for commands that only run tests or audits. + pub(crate) checked: Mutex>, + /// Whether the command returned, having run every check it planned, rather than being refused or raising. + pub(crate) returned: AtomicBool, + /// Why the worker refused the run before executing anything, reported as the run's status reason. + pub(crate) refusal: Mutex>, +} + +impl WorkerBuild { + pub fn new( + worker: &Bound<'_, NativeOrchestrationWorker>, + (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), + ) -> PyResult> { + let state = Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + checks: Mutex::default(), + attached, + blocked, + untested: Mutex::default(), + held: Mutex::default(), + upstream, + causes: Mutex::default(), + checked: Mutex::default(), + returned: AtomicBool::new(false), + refusal: Mutex::default(), + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; + } + Ok(state) + } + + pub fn command(&self) -> RunCommand { + self.execution.claim.request.command + } + + /// The server run this build executes. + pub fn run_id(&self) -> uuid::Uuid { + self.execution.claim.request.run_id + } + + /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. + pub(crate) fn checks_only(&self) -> bool { + matches!(self.command(), RunCommand::Test | RunCommand::Audit) + } + + /// Record that the command returned instead of being refused or raising. + pub fn command_returned(&self) { + self.returned.store(true, Ordering::SeqCst); + } + + pub fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) + } + + /// Record why the run is refused; the terminal report carries it as the run's status reason. + pub fn refuse(&self, reason: &str) -> PyResult<()> { + *self.refusal.lock().map_err(|_| poisoned())? = Some(reason.to_owned()); + Ok(()) + } + + pub fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject writers outside the frozen write set before any executor connection. + pub fn enforce_writers(&self, py: Python<'_>, keys: &[String]) -> Result<(), pyo3::PyErr> { + self.check_cancelled(py)?; + match keys + .iter() + .find(|key| !self.execution.claim.request.write_set.contains(*key)) + { + Some(key) => Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))), + None => Ok(()), + } + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub fn enforce_plan_writers(&self, pipeline: &Bound<'_, PyAny>) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + self.count_planned_tests(&plan)?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, (None, None)))?; + } + } + } + Ok(()) + } + + /// Count each function's tests this build plans, so a built function waits only for tests that will report. + pub(crate) fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + for entry in field(plan, "test_entries")?.try_iter()? { + let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); + for node in self.blocked.get(&test).into_iter().flatten() { + if is_function(node) { + *untested.entry(node.clone()).or_default() += 1; + } + } + } + Ok(()) + } + + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. + pub fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { + for key in keys { + self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; + } + Ok(()) + } + + /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. + pub fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { + for (key, why) in nodes { + self.complete( + py, + key, + ( + NodeOutcome::Skipped, + None, + (None, Some(format!("held back: {why}"))), + ), + )?; + } + Ok(()) + } + + /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. + pub(crate) fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, (error_code, error_message)): ( + NodeOutcome, + Option, + (Option, Option), + ), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + let skipped_because_of = match outcome { + NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { + Some(cause) => Some(cause.clone()), + None if !self.checks_only() => self.failed_upstream(key)?, + None => None, + }, + _ => None, + }; + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: skipped_because_of.clone(), + error_message: error_message.as_deref().map(redact_error_text), + }, + )?; + let cause = match outcome { + NodeOutcome::Failed => Some(key.to_owned()), + _ => skipped_because_of, + }; + if let Some(cause) = cause { + self.causes + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), cause); + } + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + self.start_key(py, &format!("{kind}:{name}")) + } + + /// Report a selected node as started unless its outcome is final, such as a planner skip. + pub fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + if self.execution.claim.request.selection.contains(key) + && !self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { + node_key: key.into(), + }, + )?; + } + Ok(()) + } + + /// Report a selected node's outcome when the command reports no executor result for it. + pub fn finish_key(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + self.complete(py, key, (outcome, None, (None, None))) + } + + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. + pub fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } + if self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.into(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), None); + if self.checks_only() { + for node in nodes { + self.start_key(py, node)?; + } + } + Ok(()) + } + + /// A test or audit finished; a check the snapshot does not record is not reported. + pub fn finish_attached(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if outcome != NodeOutcome::Skipped { + self.start_attached(py, key)?; + } + let outcome = if outcome == NodeOutcome::Failed + && self.worker.bind(py).get().refresh_cancellation(py)? + { + NodeOutcome::Cancelled + } else { + outcome + }; + if self.execution.claim.request.selection.contains(key) { + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; + } + if outcome == NodeOutcome::Skipped { + return Ok(()); + } + let mut checked = self.checked.lock().map_err(|_| poisoned())?; + for node in nodes { + let worst = checked.get(node).map_or(outcome, |previous| { + if severity(outcome) > severity(*previous) { + outcome + } else { + *previous + } + }); + checked.insert(node.clone(), worst); + } + Ok(()) + } + + /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. + pub(crate) fn failed_upstream(&self, key: &str) -> PyResult> { + let causes = self.causes.lock().map_err(|_| poisoned())?; + let mut seen: BTreeSet<&str> = BTreeSet::from([key]); + let mut pending: VecDeque<&str> = VecDeque::from([key]); + while let Some(node) = pending.pop_front() { + for upstream in self.upstream.get(node).into_iter().flatten() { + if let Some(cause) = causes.get(upstream) { + return Ok(Some(cause.clone())); + } + if seen.insert(upstream) { + pending.push_back(upstream); + } + } + } + Ok(None) + } + + /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. + pub(crate) fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py = result.py(); + let outcome: String = field(result, "outcome")?.str()?.extract()?; + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; + let name: String = field(result, "test_name")?.extract()?; + let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if !passed && !cancelled { + for model in self.blocked.get(&key).into_iter().flatten() { + self.complete( + py, + model, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + } + self.release_tested(py, &key) + } + + /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. + pub(crate) fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + let mut released: Vec<(String, Completion)> = Vec::new(); + { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + let mut held = self.held.lock().map_err(|_| poisoned())?; + for node in self.blocked.get(test).into_iter().flatten() { + let Some(remaining) = untested.get_mut(node) else { + continue; + }; + *remaining = remaining.saturating_sub(1); + if *remaining == 0 + && let Some(completion) = held.remove(node) + { + released.push((node.clone(), completion)); + } + } + } + for (node, completion) in released { + self.complete(py, &node, completion)?; + } + Ok(()) + } + + /// Publish a node's result, or hold a built function's while its tests have still to report. + pub(crate) fn complete_or_hold( + &self, + py: Python<'_>, + key: &str, + completion: Completion, + ) -> PyResult<()> { + let waiting = !self.checks_only() + && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) + && self + .untested + .lock() + .map_err(|_| poisoned())? + .get(key) + .is_some_and(|remaining| *remaining > 0); + if waiting { + self.held + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), completion); + return Ok(()); + } + self.complete(py, key, completion) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if key.starts_with(SQL_TEST_KEY_PREFIX) { + return self.finished_test(result); + } + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; + } + Ok(()) + } + + pub fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let key = format!("check:{name}"); + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.clone(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key, None); + Ok(()) + } + + /// A completion without our actual-execution start callback is a blocked check, not a failure. + pub fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = field(result, "node_name")?.extract()?; + let key = format!("check:{name}"); + let started = self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(&key); + let outcome = if !started { + NodeOutcome::Skipped + } else if field(result, "passed")?.is_truthy()? { + NodeOutcome::Success + } else if field(result, "warned")?.is_truthy()? { + NodeOutcome::Warn + } else if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + }; + self.finish_check(result.py(), &key, outcome) + } + + pub(crate) fn finish_check( + &self, + py: Python<'_>, + key: &str, + outcome: NodeOutcome, + ) -> PyResult<()> { + self.worker.bind(py).get().change( + py, + &format!("check-finished/{key}"), + RunChange::CheckFinished { + node_key: key.into(), + outcome, + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), Some(outcome)); + Ok(()) + } + + /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. + pub fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + let checks_only = self.checks_only(); + if checks_only && !cancelled { + let checked = self.checked.lock().map_err(|_| poisoned())?.clone(); + let returned = self.returned.load(Ordering::SeqCst); + for key in &self.execution.claim.request.selection { + let seen = checked.get(key).copied(); + let outcome = if returned { + Some(seen.unwrap_or(NodeOutcome::Success)) + } else { + seen.filter(|outcome| { + matches!(outcome, NodeOutcome::Failed | NodeOutcome::Cancelled) + }) + }; + if let Some(outcome) = outcome { + self.complete(py, key, (outcome, None, (None, None)))?; + } + } + } + let held: Vec<(String, Completion)> = + std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) + .into_iter() + .collect(); + for (key, completion) in held { + self.complete(py, &key, completion)?; + } + let returned = self.returned.load(Ordering::SeqCst); + let build = self.command().is_build(); + let source_needs_no_run = |key: &str| build && key.starts_with("source:"); + for key in &self.execution.claim.request.selection { + let outcome = if cancelled { + NodeOutcome::Cancelled + } else if returned && source_needs_no_run(key) { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(py, key, (outcome, None, (None, None)))?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let outcomes: Vec = completed.values().copied().collect(); + let good = completed.iter().any(|(key, value)| { + matches!(value, NodeOutcome::Success | NodeOutcome::Warn) + && !source_needs_no_run(key) + }); + let bad = outcomes.iter().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if outcomes.contains(&NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + let unfinished: Vec = self + .checks + .lock() + .map_err(|_| poisoned())? + .iter() + .filter(|(_, outcome)| outcome.is_none()) + .map(|(key, _)| key.clone()) + .collect(); + for key in unfinished { + self.finish_check( + py, + &key, + if outcome == RunOutcome::Cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + )?; + } + let reason: String = self + .refusal + .lock() + .map_err(|_| poisoned())? + .clone() + .unwrap_or_else(|| "worker executor completed and closed its connections".into()); + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason, + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationWorker { + pub(crate) owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. + pub(crate) fn with_owner( + &self, + py: Python<'_>, + work: impl FnOnce(&WorkerOwner) -> PyResult + Send, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + work(owner.as_ref().ok_or_else(unavailable)?) + }) + } + + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + self.with_owner(py, |owner| Ok(owner.session.execution.clone())) + } + + /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. + pub fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + self.with_owner(py, |owner| { + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(error) + if attempts < REPORT_ATTEMPTS + && match &error { + WorkerError::Transport(_) => true, + WorkerError::Rejected(status) => { + status.as_u16() == STORE_BUSY_STATUS + } + _ => false, + } => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + pub(crate) fn execution_json(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + pub fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + pub(crate) fn report( + &self, + py: Python<'_>, + event_id: &str, + change_json: &str, + ) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + self.with_owner(py, |owner| { + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + pub(crate) fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/types.rs b/crates/sqlbuild-python-project/src/compiled_project/types.rs new file mode 100644 index 0000000000..d57b5bdd68 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/types.rs @@ -0,0 +1,36 @@ +//! Public traits and type aliases of compiled project. + +use crate::compiled_project::models::WorkerBuild; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use std::collections::BTreeMap; +use std::sync::Arc; + +/// One discovered source: its file's path, the discovered file and the entry. +pub type SourceRow = (String, Py, Py); + +/// `(default contract, whether expressions are syntax-checked, source kind)`. +pub type SourceSettings = (Option, bool, Py); + +/// A test case's discovered case, name and parameter values in authored order. +pub type TestCaseRow = (Py, String, Vec<(String, Py)>); + +/// The attached SQL test inputs, then scenario inputs. +pub type AttachedSqlTests = (Vec>, Vec>); + +/// A loader's description, relative path and whether that path is a YAML file. +pub type LoaderRow = (Option, String, bool); + +/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. +pub type ResourceDiagnostics<'py> = ( + Vec>, + Vec>, + Vec>, +); + +/// Node keys by snapshot node key, such as the models each test checks. +pub type NodeKeys = BTreeMap>; + +/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. +pub type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs index 38f2b1c0e3..774e19300e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; +use crate::compiled_views::models::NativeCompiledAudit; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs index 5b08d91959..5937a9cb69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs @@ -1,20 +1,22 @@ //! Audits attached by the native loop, held by the compiled project, and their views. +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeCompiledAudit; +use crate::compiled_views::types::AttachedAudit; +use crate::compiled_views::types::AuditRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::{FromPyObject, pymethods}; -use sqlbuild_project::models as project_models; use sqlbuild_project::types::ObjectKeyRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, reference_tuple, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::compiled_model_view::object_key; +use crate::compiled_views::_helpers::model_input_view::InputViewClasses; +use crate::compiled_views::_helpers::model_input_view::reference_tuple; +use crate::compiled_views::_helpers::model_input_view::view_classes; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; @@ -37,27 +39,8 @@ pub(crate) type AuditAssemblyRow = ( Option<(String, String)>, ); -/// What the audit loop attached for one singular audit block or attached generic audit. -pub(crate) type AttachedAudit = project_models::AttachedAudit; - -/// One attached audit held by the compiled project. -pub(crate) type AuditRecord = project_models::AuditRecord; - -/// An attached audit's user values. -pub(crate) struct AuditHandles { - /// The discovered (or built-in) audit file and block the SQL comes from. - pub(crate) audit_file: Py, - pub(crate) audit_block: Py, - /// The authored measurement policy specs, carrying user values. - pub(crate) thresholds: Py, - pub(crate) minimum_samples: Py, - pub(crate) evidence_limit: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - /// An audit record's Python projections, each built once on first access. -pub(crate) struct AuditProjections { +pub struct AuditProjections { pub(crate) references: PyOnceLock>, pub(crate) key: PyOnceLock>, } @@ -208,7 +191,7 @@ macro_rules! audit_getters { } #[getter] - pub(crate) fn attached_target_name(&self, py: Python<'_>) -> PyResult> { + pub fn attached_target_name(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, _| { Ok(audit.attached.target.as_ref().map(|target| target.name.clone())) }) @@ -267,19 +250,6 @@ macro_rules! audit_getters { }; } -/// One attached audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeAuditInput { - owner: Py, - index: usize, -} - -impl NativeAuditInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - audit_getters!(NativeAuditInput, { #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { @@ -322,19 +292,6 @@ audit_getters!(NativeAuditInput, { } }); -/// One assembled audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledAudit { - owner: Py, - index: usize, -} - -impl NativeCompiledAudit { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - audit_getters!(NativeCompiledAudit, { /// The audit's planner-facing `CompiledObjectKey`. #[getter] @@ -404,36 +361,3 @@ pub(crate) fn audit_reference_rows( Ok(Some(record.attached.references.clone())) }) } - -impl NativeCompiledAudit { - /// The `CompiledAudit` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.scope_deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.definition_name(py)?.into_bound_py_any(py)?, - self.audit_file(py)?.into_bound(py), - self.audit_block(py)?.into_bound(py), - self.sql_body(py)?.into_bound_py_any(py)?, - self.evaluation_mode(py)?.into_bound(py), - self.measurement_contract(py)?.into_bound(py), - self.thresholds(py)?.into_bound(py), - self.minimum_samples(py)?.into_bound(py), - self.measure_sql(py)?.into_bound_py_any(py)?, - self.evidence_sql(py)?.into_bound_py_any(py)?, - self.evidence_limit(py)?.into_bound(py), - self.references(py)?.into_bound(py).into_any(), - self.attached_target_kind(py)?, - self.attached_target_name(py)?.into_bound_py_any(py)?, - self.attached_column_name(py)?.into_bound_py_any(py)?, - self.severity(py)?, - self.run_scope(py)?.into_bound_py_any(py)?, - pyo3::types::PyBool::new(py, self.always_run(py)?) - .to_owned() - .into_any(), - self.description(py)?.into_bound_py_any(py)?, - self.meta(py)?.into_bound(py), - ]) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs index ca2d959b72..b3da6dd247 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs @@ -26,11 +26,12 @@ use sqlbuild_analysis::sql_macros::models::SqlMacroFinding; use sqlbuild_project::models as project_models; use sqlbuild_project::types::InferredRow; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledViewClasses; -use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::types::ModelInputRecord; +use crate::lineage::main::confidence_member::confidence_member; +use crate::lineage::main::transform_kind_member::transform_kind_member; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; @@ -62,7 +63,7 @@ pub(crate) type SessionArgument = (Py, usize); pub(crate) type AnalysisRecord = project_models::AnalysisRecord; /// Python projections of the analysis, each built once on first access. -pub(crate) struct AnalysisProjections { +pub struct AnalysisProjections { lineage: PyOnceLock>, binding: PyOnceLock>, proof: PyOnceLock>, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs new file mode 100644 index 0000000000..1697599d39 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs @@ -0,0 +1,262 @@ +//! A compiled model, read from the native compiled project that owns it. + +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::types::LineageInput; +use sqlbuild_project::models as project_models; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::types::InferredRow; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use std::sync::Arc; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; + +use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisRecord; +use crate::compiled_views::_helpers::compiled_model_analysis::ProofArgument; +use crate::compiled_views::_helpers::compiled_model_analysis::SessionArgument; +use crate::compiled_views::_helpers::compiled_model_analysis::analysis_record; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::types::ModelInputRecord; +use crate::graph::models::ResourceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::models::HeaderColumns; + +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; +const ALIAS_KEY: &str = "alias"; +const VALUES_ATTRIBUTE: &str = "values"; +const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; +const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +const TAGS_KEY: &str = "tags"; +pub(crate) const NULLABLE: &str = "nullable"; + +/// A schema's entry audits, column specs and each audited column's index and name. +pub(crate) type AuditedSchema = (Arc, Arc, Vec<(usize, String)>); + +/// `(database, schema, name)` of a relation. +pub(crate) type DestinationRow = (Option, Option, String); + +/// What assembly and the semantic checks finished for one compiled model. +pub(crate) type CompiledRecord = project_models::CompiledRecord; + +/// A compiled model's Python attribute projections, each built once on first access. +pub struct CompiledProjections { + pub(crate) key: PyOnceLock>, + pub(crate) deps: PyOnceLock>, +} + +impl Default for CompiledProjections { + fn default() -> Self { + Self { + key: PyOnceLock::new(), + deps: PyOnceLock::new(), + } + } +} + +/// What later stages changed for one view of a model; the owner's record stays as assembled. +#[derive(Debug, Default)] +pub(crate) struct ModelOverlay { + pub(crate) binding_diagnostics: Option>, + /// Output columns whose types semantic recovery forgot, left unchecked by the contract. + pub(crate) poisoned: Vec, + pub(crate) destination: Option, +} + +impl ModelOverlay { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + binding_diagnostics: self + .binding_diagnostics + .as_ref() + .map(|diagnostics| diagnostics.clone_ref(py)), + poisoned: self.poisoned.clone(), + destination: self.destination.clone(), + } + } +} + +/// One view's projections of attributes its overlay can change, each built once. +pub(crate) struct ViewProjections { + pub(crate) destination: PyOnceLock>, + pub(crate) inferred_columns: PyOnceLock>, +} + +impl std::fmt::Debug for ViewProjections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ViewProjections") + .finish_non_exhaustive() + } +} + +impl Default for ViewProjections { + fn default() -> Self { + Self { + destination: PyOnceLock::new(), + inferred_columns: PyOnceLock::new(), + } + } +} + +/// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. +#[derive(FromPyObject)] +pub(crate) struct CompiledArguments( + Vec<(String, String)>, + Option, + Option>, + Option, + Vec, +); + +/// The compiled record of one model, with its configured destination. +pub(crate) fn compiled_record( + py: Python<'_>, + input: &ModelInputRecord, + name: &str, + arguments: CompiledArguments, +) -> PyResult { + let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; + let analysis: AnalysisRecord = + analysis_record(py, input, session, (fallback, standalone_proof))?; + let config: &Bound<'_, PyAny> = input.host.config.bind(py); + let values: Bound<'_, PyDict> = native_field(config, VALUES_ATTRIBUTE)?.downcast_into()?; + Ok(CompiledRecord { + deps, + destination: DestinationFact { + database: text_value(&values, DATABASE_KEY)?, + schema: text_value(&values, SCHEMA_KEY)?, + name: text_value(&values, ALIAS_KEY)?.unwrap_or_else(|| name.to_owned()), + qualified_name: None, + logical_schema: native_field(config, LOGICAL_SCHEMA_ATTRIBUTE)?.extract()?, + logical_database: native_field(config, LOGICAL_DATABASE_ATTRIBUTE)?.extract()?, + }, + analysis, + macro_deps, + host: CompiledProjections::default(), + }) +} + +fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => { + Ok(Some(value.extract()?)) + } + _ => Ok(None), + } +} + +/// The destination with adapter defaults filled in; `render` renders its qualified name. +pub(crate) fn defaulted_destination( + current: &DestinationFact, + defaults: (Option, Option), + render: &Bound<'_, PyAny>, +) -> PyResult { + let (default_schema, default_database) = defaults; + let schema: Option = current.schema.clone().or(default_schema); + let database: Option = current.database.clone().or(default_database); + let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); + keywords.set_item(DATABASE_KEY, &database)?; + keywords.set_item(SCHEMA_KEY, &schema)?; + keywords.set_item("name", ¤t.name)?; + let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; + Ok(DestinationFact { + database, + schema, + name: current.name.clone(), + qualified_name, + logical_schema: current.logical_schema.clone(), + logical_database: current.logical_database.clone(), + }) +} + +/// A lineage row's output column with its `(resource name, column)` sources. +pub(crate) fn lineage_input(row: &LineageRow) -> LineageInput { + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (_, resource, column) in &row.sources { + sources.push((resource.clone(), column.clone())); + } + (row.output_column.clone(), sources) +} + +/// Each family's name casefolded as Python's, with its inferred type. +pub(crate) fn casefolded_families( + families: &[(String, Option)], +) -> Vec<(String, Option)> { + families + .iter() + .map(|(name, kind)| (str::to_lowercase(name), kind.clone())) + .collect() +} + +/// The compiled record of an input, which assembly records before any view is handed out. +pub(crate) fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { + record + .compiled + .as_ref() + .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) +} + +/// The model's project graph row: its key, dependencies, tags and folder. +pub(crate) fn model_graph_row( + model: &NativeCompiledModel, + py: Python<'_>, +) -> PyResult { + model.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; + let mut tags: Vec = Vec::new(); + if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? + && (value.is_instance_of::() + || value.is_instance_of::()) + { + for tag in value.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + } + let folder: String = record.host.model_file.get().folder(py)?; + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), + tags, + folder: Some(folder), + }) + }) +} + +pub(crate) fn model_name(record: &ModelInputRecord) -> String { + record.name.clone() +} + +/// The `CompiledRelationLocation` of a destination. +pub(crate) fn relation_location( + classes: &CompiledViewClasses<'_>, + destination: &DestinationFact, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); + fields.set_item(DATABASE_KEY, &destination.database)?; + fields.set_item(SCHEMA_KEY, &destination.schema)?; + fields.set_item("name", &destination.name)?; + fields.set_item("qualified_name", &destination.qualified_name)?; + fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; + fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; + Ok(classes.relation_location.call((), Some(&fields))?.unbind()) +} + +pub(crate) fn object_key( + classes: &CompiledViewClasses<'_>, + kind: &str, + name: &str, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); + fields.set_item("resource_type", kind)?; + fields.set_item("name", name)?; + Ok(classes.object_key.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs index a560c7998d..65e385eee0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledFunction; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs index bdeb2eb1cf..7d27b19075 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs @@ -1,24 +1,25 @@ //! Functions attached by the native loop, held by the compiled project, and their views. +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::types::AttachedFunction; +use crate::compiled_views::types::FunctionRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::{FromPyObject, pymethods}; use sqlbuild_python_base::positional_record; -use sqlbuild_project::models as project_models; use sqlbuild_project::types::{NamedTypeRow, ObjectKeyRow}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - defaulted_destination, object_key, relation_location, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, reference_tuple, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::compiled_model_view::defaulted_destination; +use crate::compiled_views::_helpers::compiled_model_view::object_key; +use crate::compiled_views::_helpers::compiled_model_view::relation_location; +use crate::compiled_views::_helpers::model_input_view::InputViewClasses; +use crate::compiled_views::_helpers::model_input_view::reference_tuple; +use crate::compiled_views::_helpers::model_input_view::view_classes; use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -27,7 +28,7 @@ const SQL_LANGUAGE: &str = "sql"; const PYTHON_LANGUAGE: &str = "python"; const FUNCTION_FILE_ATTRIBUTE: &str = "relative_path"; const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const NOT_ASSEMBLED: &str = "the function was not assembled"; +pub(crate) const NOT_ASSEMBLED: &str = "the function was not assembled"; const FUNCTION_CLASSES: &str = "functions"; /// The Python classes a function view projects its argument and return column records with. @@ -50,22 +51,8 @@ positional_record! { } } -/// What the function loop attached for one SQL or Python function. -pub(crate) type AttachedFunction = project_models::AttachedFunction; - -/// One attached function held by the compiled project. -pub(crate) type FunctionRecord = project_models::FunctionRecord; - -/// An attached function's user values. -pub(crate) struct FunctionHandles { - /// The discovered function file. - pub(crate) function_file: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - /// A function record's Python projections, each built once on first access. -pub(crate) struct FunctionProjections { +pub struct FunctionProjections { arguments: PyOnceLock>, return_columns: PyOnceLock>, pub(crate) references: PyOnceLock>, @@ -108,7 +95,7 @@ fn named_types( } /// The function's `FunctionLanguage` member. -fn language<'py>( +pub(crate) fn language<'py>( function: &FunctionRecord, classes: &FunctionClasses<'py>, ) -> PyResult> { @@ -120,7 +107,7 @@ fn language<'py>( classes.function_language.call1((value,)) } -fn read_function<'py, T>( +pub(crate) fn read_function<'py, T>( owner: &Py, index: usize, py: Python<'py>, @@ -269,19 +256,6 @@ macro_rules! function_getters { }; } -/// One attached function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeFunctionInput { - owner: Py, - index: usize, -} - -impl NativeFunctionInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - function_getters!(NativeFunctionInput, { #[getter] pub(crate) fn function_file(&self, py: Python<'_>) -> PyResult> { @@ -387,72 +361,6 @@ function_getters!(NativeFunctionInput, { } }); -/// One assembled function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledFunction { - owner: Py, - index: usize, - /// The destination and fingerprint destination with target defaults applied. - defaulted: Option<(DestinationFact, DestinationFact)>, - destination: PyOnceLock>, - fingerprint_destination: PyOnceLock>, -} - -impl NativeCompiledFunction { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self::defaulted(owner, index, None) - } - - /// The function's project graph row: its key, the resources it reads and its tags. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_function(&self.owner, self.index, py, |function, _| { - let deps: &[ObjectKeyRow] = function - .deps - .as_deref() - .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(ResourceRow { - key: ( - function.attached.node_type().to_owned(), - function.attached.name.clone(), - ), - deps: deps.to_vec(), - tags: function.attached.tags.clone(), - folder: None, - }) - }) - } - - fn defaulted( - owner: Py, - index: usize, - defaulted: Option<(DestinationFact, DestinationFact)>, - ) -> Self { - Self { - owner, - index, - defaulted, - destination: PyOnceLock::new(), - fingerprint_destination: PyOnceLock::new(), - } - } - - /// The destination and fingerprint destination, with target defaults once applied. - pub(crate) fn current_destinations( - &self, - py: Python<'_>, - ) -> PyResult<(DestinationFact, DestinationFact)> { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_function(&self.owner, self.index, py, |function, _| { - Ok(( - function.attached.destination.clone(), - function.attached.fingerprint_destination.clone(), - )) - }) - } -} - function_getters!(NativeCompiledFunction, { /// The function's planner-facing `CompiledObjectKey`. #[getter] @@ -535,7 +443,7 @@ function_getters!(NativeCompiledFunction, { } /// This function with target defaults; a non-inheriting Python function is name-qualified. - pub(crate) fn with_target_defaults( + pub fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), @@ -562,29 +470,3 @@ function_getters!(NativeCompiledFunction, { )) } }); - -impl NativeCompiledFunction { - /// The `CompiledFunction` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.relative_path(py)?.into_bound(py), - self.arguments(py)?.into_bound(py).into_any(), - self.returns(py)?.into_bound_py_any(py)?, - self.body_sql(py)?.into_bound_py_any(py)?, - self.destination(py)?.into_bound(py), - self.fingerprint_destination(py)?.into_bound(py), - self.return_columns(py)?.into_bound(py).into_any(), - self.references(py)?.into_bound(py).into_any(), - self.language(py)?, - self.source_file_path(py)?.into_bound(py), - self.runtime_version(py)?.into_bound_py_any(py)?, - self.entry_point(py)?.into_bound_py_any(py)?, - self.packages(py)?.into_any(), - self.tags(py)?.into_any(), - self.description(py)?.into_bound_py_any(py)?, - ]) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs index 6da8e57153..0ab5be42f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs @@ -3,76 +3,20 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::audit_views::{ - NativeAuditInput, NativeCompiledAudit, -}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::{ - NativeCompiledFunction, NativeFunctionInput, -}; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, NativeSeedInput}; -use crate::bindings::_helpers::compiled_views::source_views::{ - NativeCompiledSource, NativeSourceInput, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeCompiledAudit; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSource; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::models::NativeSeedInput; +use crate::compiled_views::models::NativeSourceInput; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_native_audit_input, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_audit, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_model, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_function, - }, - GetterEntry { - class: class_of::, - read: read_native_function_input, - }, - GetterEntry { - class: class_of::, - read: read_native_inputs_view, - }, - GetterEntry { - class: class_of::, - read: read_native_model_input, - }, - GetterEntry { - class: class_of::, - read: read_native_project_view, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_seed, - }, - GetterEntry { - class: class_of::, - read: read_native_seed_input, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_source, - }, - GetterEntry { - class: class_of::, - read: read_native_source_input, - }, -]; - -fn read_native_audit_input<'py>( +pub(crate) fn read_native_audit_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -150,7 +94,7 @@ fn read_native_audit_input<'py>( }) } -fn read_native_compiled_audit<'py>( +pub(crate) fn read_native_compiled_audit<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -229,7 +173,7 @@ fn read_native_compiled_audit<'py>( }) } -fn read_native_compiled_model<'py>( +pub(crate) fn read_native_compiled_model<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -344,7 +288,7 @@ fn read_native_compiled_model<'py>( }) } -fn read_native_compiled_function<'py>( +pub(crate) fn read_native_compiled_function<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -406,7 +350,7 @@ fn read_native_compiled_function<'py>( }) } -fn read_native_function_input<'py>( +pub(crate) fn read_native_function_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -487,7 +431,7 @@ fn read_native_function_input<'py>( }) } -fn read_native_inputs_view<'py>( +pub(crate) fn read_native_inputs_view<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -533,7 +477,7 @@ fn read_native_inputs_view<'py>( }) } -fn read_native_model_input<'py>( +pub(crate) fn read_native_model_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -607,7 +551,7 @@ fn read_native_model_input<'py>( }) } -fn read_native_project_view<'py>( +pub(crate) fn read_native_project_view<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -669,7 +613,7 @@ fn read_native_project_view<'py>( }) } -fn read_native_compiled_seed<'py>( +pub(crate) fn read_native_compiled_seed<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -701,7 +645,7 @@ fn read_native_compiled_seed<'py>( }) } -fn read_native_seed_input<'py>( +pub(crate) fn read_native_seed_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -726,7 +670,7 @@ fn read_native_seed_input<'py>( }) } -fn read_native_compiled_source<'py>( +pub(crate) fn read_native_compiled_source<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -751,7 +695,7 @@ fn read_native_compiled_source<'py>( }) } -fn read_native_source_input<'py>( +pub(crate) fn read_native_source_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs new file mode 100644 index 0000000000..7f80eaccfc --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs @@ -0,0 +1,184 @@ +//! A compile's attached inputs, read from the native compiled project that owns its models. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +const MODEL_INPUTS_FIELD: &str = "model_inputs"; +const RUN_ID_FIELD: &str = "run_id"; +const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; + +/// Compile inputs held as Python objects: user code, discovery or an integration owns them. +#[derive(Debug)] +pub(crate) struct BoundaryInputs { + /// The discovered project config. + pub(crate) project_config: Py, + /// The discovered local config. + pub(crate) local_config: Py, + /// The discovered project inputs. + pub(crate) discovered_inputs: Py, + /// The adapter's lexical syntax. + pub(crate) sql_lexical_syntax: Py, + /// The selected target config, or `None`. + pub(crate) effective_target: Py, + /// The compile cache directory path. + pub(crate) compile_cache_dir: Py, + /// The resolved connection, adapter integration values. + pub(crate) effective_connection: Py, + /// Project variables, user values. + pub(crate) effective_vars: Py, + /// The macro context user macro bodies run with. + pub(crate) macro_context: Py, + /// Loaded macro modules, user code. + pub(crate) loaded_macros: Py, + /// Public enums, user declaration values. + pub(crate) public_enums: Py, + /// Public constants, user declaration values. + pub(crate) public_constants: Py, + /// The external reference resolver integration, or `None`. + pub(crate) external_sql_reference_resolver: Py, +} + +/// Compiler-derived inputs still held as Python objects until they are owner facts. +#[derive(Debug)] +pub(crate) struct CompilerHandles { + /// Effective settings. + pub(crate) effective_settings: Py, + /// Attached seed inputs. + pub(crate) seed_inputs: Py, + /// Attached source inputs. + pub(crate) source_inputs: Py, + /// Attached SQL function inputs. + pub(crate) sql_function_inputs: Py, + /// Attached SQL test inputs. + pub(crate) test_inputs: Py, + /// Attached SQL scenario inputs. + pub(crate) scenario_inputs: Py, + /// Attached audit inputs. + pub(crate) audit_inputs: Py, + /// Compiler diagnostics. + pub(crate) diagnostics: Py, + /// The declaration scope index. + pub(crate) scope_index: Py, + /// The declaration scope build. + pub(crate) declaration_scope: Py, +} + +/// The values of one compile inputs view. +#[derive(Debug)] +pub(crate) struct InputsFields { + pub(crate) run_id: String, + pub(crate) effective_target_name: Option, + pub(crate) no_sql_validation: bool, + pub(crate) model_inputs: Py, + pub(crate) boundary: BoundaryInputs, + pub(crate) compiler: CompilerHandles, +} + +impl InputsFields { + /// The fields named in `fields`, each required. + pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + no_sql_validation: required(fields, NO_SQL_VALIDATION_FIELD)?.extract()?, + model_inputs: required(fields, MODEL_INPUTS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + project_config: required(fields, "project_config")?.unbind(), + local_config: required(fields, "local_config")?.unbind(), + discovered_inputs: required(fields, "discovered_inputs")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + effective_target: required(fields, "effective_target")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + macro_context: required(fields, "macro_context")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + }, + compiler: CompilerHandles { + effective_settings: required(fields, "effective_settings")?.unbind(), + seed_inputs: required(fields, "seed_inputs")?.unbind(), + source_inputs: required(fields, "source_inputs")?.unbind(), + sql_function_inputs: required(fields, "sql_function_inputs")?.unbind(), + test_inputs: required(fields, "test_inputs")?.unbind(), + scenario_inputs: required(fields, "scenario_inputs")?.unbind(), + audit_inputs: required(fields, "audit_inputs")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + declaration_scope: required(fields, "declaration_scope")?.unbind(), + }, + }) + } + + /// Every field by name. + pub(crate) fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item(NO_SQL_VALIDATION_FIELD, self.no_sql_validation)?; + fields.set_item(MODEL_INPUTS_FIELD, self.model_inputs.bind(py))?; + fields.set_item("project_config", self.boundary.project_config.bind(py))?; + fields.set_item("local_config", self.boundary.local_config.bind(py))?; + fields.set_item( + "discovered_inputs", + self.boundary.discovered_inputs.bind(py), + )?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item("effective_target", self.boundary.effective_target.bind(py))?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("macro_context", self.boundary.macro_context.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item( + "effective_settings", + self.compiler.effective_settings.bind(py), + )?; + fields.set_item("seed_inputs", self.compiler.seed_inputs.bind(py))?; + fields.set_item("source_inputs", self.compiler.source_inputs.bind(py))?; + fields.set_item( + "sql_function_inputs", + self.compiler.sql_function_inputs.bind(py), + )?; + fields.set_item("test_inputs", self.compiler.test_inputs.bind(py))?; + fields.set_item("scenario_inputs", self.compiler.scenario_inputs.bind(py))?; + fields.set_item("audit_inputs", self.compiler.audit_inputs.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + fields.set_item( + "declaration_scope", + self.compiler.declaration_scope.bind(py), + )?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing compile input field {name}"))) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs new file mode 100644 index 0000000000..b0e7c674c7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs @@ -0,0 +1,20 @@ +pub(crate) mod audit_reads; +pub(crate) mod audit_views; +pub(crate) mod compiled_model_analysis; +pub(crate) mod compiled_model_view; +pub(crate) mod function_reads; +pub(crate) mod function_views; +pub(crate) mod getters; +pub(crate) mod inputs_view; +pub(crate) mod model_input_view; +pub(crate) mod model_reads; +pub(crate) mod project_host; +pub(crate) mod project_reads; +pub(crate) mod project_view; +pub(crate) mod records; +pub(crate) mod relation_keys; +pub(crate) mod resource_reads; +pub(crate) mod seed_reads; +pub(crate) mod seed_views; +pub(crate) mod source_reads; +pub(crate) mod source_views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs index df851fe4ee..8b1739a8e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs @@ -1,34 +1,29 @@ //! A model's compile input, read from the native compiled project that owns it. -use sqlbuild_project::models as project_models; +use crate::compiled_views::models::AttachedSchema; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::types::ModelInputRecord; use sqlbuild_project::models::ExpansionRecord; use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; +use pyo3::{FromPyObject, IntoPyObject}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledArguments, CompiledViewClasses, NativeCompiledModel, compiled_record, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::{RelationKeyRow, relation_key_row}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::relation_keys::relation_key_row; +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::RelationKeyRow; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::records::projected_text::projected_text; use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; -use sqlbuild_python_front::model_config::models::HeaderAudits; -use sqlbuild_python_front::model_config::models::HeaderColumns; -use sqlbuild_python_front::model_config::models::HeaderFamilies; use sqlbuild_python_front::render::main::calls::span_tuple::span_tuple; use sqlbuild_python_front::render::types::SpanRow; @@ -39,93 +34,11 @@ const MODEL_RELATION_KIND: &str = "ref"; #[pyo3(from_item_all)] pub(crate) struct InputViewClasses<'py> { pub(crate) compiled: CompiledViewClasses<'py>, - sql_reference: Bound<'py, PyAny>, - reference_kind: Bound<'py, PyAny>, - schema_model_entry: Bound<'py, PyAny>, - sql_expansion: Bound<'py, PyAny>, - expansion_span: Bound<'py, PyAny>, -} - -/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. -pub(crate) type SchemaRecord = project_models::SchemaRecord; - -/// What the model loop finished for one model beyond its facts. -pub(crate) type ModelInputRecord = project_models::ModelInputRecord; - -/// A model schema's header columns and specs, which carry user-authored audit arguments. -pub(crate) struct SchemaHandles { - pub(crate) columns: Arc, - pub(crate) audits: Arc, - pub(crate) dynamic_columns: Arc, -} - -/// A model input's user values and the attribute projections its views build. -pub(crate) struct ModelInputHandles { - /// The discovered model file, owned by the discovery inputs. - pub(crate) model_file: Py, - /// The resolved config, whose values hold user values. - pub(crate) config: Py, - pub(crate) rejected_opt_out: Option>, - /// The model's own enums and constants, whose values are user values. - pub(crate) enum_declarations: Py, - pub(crate) constant_declarations: Py, - pub(crate) enum_columns: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) macro_usages: Py, - pub(crate) declaration_usages: Py, - /// Attribute projections built on first access, shared by every view of the model. - pub(crate) projections: InputProjections, -} - -/// A model input's Python attribute projections, each built once on first access. -pub(crate) struct InputProjections { - query_sql: PyOnceLock>, - references: PyOnceLock>, - schema_entry: PyOnceLock>, - sql_expansion: PyOnceLock>, - lint_expansion: PyOnceLock>, -} - -impl Default for InputProjections { - fn default() -> Self { - Self { - query_sql: PyOnceLock::new(), - references: PyOnceLock::new(), - schema_entry: PyOnceLock::new(), - sql_expansion: PyOnceLock::new(), - lint_expansion: PyOnceLock::new(), - } - } -} - -/// One model's compile input, projected from the owner on demand. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeModelInput { - owner: Py, - index: usize, -} - -impl NativeModelInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } - - pub(crate) fn read<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, - ) -> PyResult { - read_input(&self.owner, self.index, py, read) - } - - /// Read the input record together with the project that owns it. - pub(crate) fn read_with_owner<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, - ) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, read) - } + pub(crate) sql_reference: Bound<'py, PyAny>, + pub(crate) reference_kind: Bound<'py, PyAny>, + pub(crate) schema_model_entry: Bound<'py, PyAny>, + pub(crate) sql_expansion: Bound<'py, PyAny>, + pub(crate) expansion_span: Bound<'py, PyAny>, } /// Read the input record at `index` of `owner` together with `owner`. @@ -158,171 +71,6 @@ pub(crate) fn view_classes<'py>( owner.bind(py).borrow().input_classes(py)?.extract() } -#[pymethods] -impl NativeModelInput { - #[getter] - pub(crate) fn model_file(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.model_file.clone_ref(py).into_any()) - }) - } - - #[getter] - pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) - } - - #[getter] - pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(query_sql_text(py, record))) - } - - #[getter] - pub(crate) fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| { - Ok(PyString::new(py, &record.macro_source_sql)) - }) - } - - #[getter] - pub(crate) fn references(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| references(&classes, record)) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| schema_entry(&classes, record)) - } - - /// Each reference as `(kind, name, package, call argument count)`. - #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| { - Ok(record - .references - .iter() - .cloned() - .map(reference_row_tuple) - .collect()) - }) - } - - /// The model's declared key facts, for deriving its unique keys. - #[getter] - pub(crate) fn relation_key_row(&self, py: Python<'_>) -> PyResult { - self.read_with_owner(py, |owner, record| { - model_relation_key_row(owner, record, py) - }) - } - - /// The name of the schema entry attached to the model, or `None`. - #[getter] - pub(crate) fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(attached_schema_name(record))) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> Py { - py.None() - } - - #[getter] - pub(crate) fn sql_validation_enabled(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _| Ok(record.sql_validation_enabled)) - } - - #[getter] - pub(crate) fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(rejected_opt_out(record, py))) - } - - #[getter] - pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.enum_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.constant_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) - } - - #[getter] - pub(crate) fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| PyTuple::new(py, &record.macro_deps)) - } - - #[getter] - pub(crate) fn macro_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.macro_usages.clone_ref(py))) - } - - #[getter] - pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.declaration_usages.clone_ref(py)) - }) - } - - /// Record what assembly finished for the model; return its compiled model view. - fn compiled_model( - &self, - py: Python<'_>, - arguments: CompiledArguments, - ) -> PyResult { - let mut owner = self.owner.bind(py).borrow_mut(); - let record: &mut ModelInputRecord = owner.input_mut(self.index)?; - let name: String = record.name.clone(); - record.compiled = Some(compiled_record(py, record, &name, arguments)?); - Ok(NativeCompiledModel::new( - self.owner.clone_ref(py), - self.index, - )) - } - - /// The authored-to-expanded passes, when interpolation left the authored query unchanged. - #[getter] - pub(crate) fn sql_expansion(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| authored_expansion(&classes, record)) - } - - /// The whole expansion, variables included, which early SQL lint reuses. - #[getter] - pub(crate) fn lint_expansion(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, py| { - record - .host - .projections - .lint_expansion - .get_or_try_init(py, || { - let expansion: &ExpansionRecord = &record.expansion; - expansion_object( - &classes, - expansion, - &[ - &expansion.interpolation_spans, - &expansion.declaration_spans, - &expansion.macro_spans, - ], - ) - }) - .map(|expansion| expansion.clone_ref(py)) - }) - } -} - /// The compiled query as the Python string every reader shares; the authored query's when equal. pub(crate) fn query_sql_text<'py>( py: Python<'py>, @@ -463,7 +211,7 @@ pub(crate) fn schema_entry( } /// `CompiledSqlExpansion(authored_sql, expanded_sql, passes)` for these passes. -fn expansion_object( +pub(crate) fn expansion_object( classes: &InputViewClasses<'_>, expansion: &ExpansionRecord, passes: &[&Vec], @@ -488,13 +236,6 @@ fn expansion_object( Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } -/// A model input's attached schema: its name and dynamic column families. -pub(crate) struct AttachedSchema { - pub(crate) name: String, - pub(crate) dynamic_columns: Vec, - pub(crate) type_enforcement: bool, -} - /// The attached schema of `model_input`; a native input answers without building its entry. pub(crate) fn attached_schema<'py>( model_input: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_reads.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/model_reads.rs index 93e1daa526..ecc8195755 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_reads.rs @@ -1,24 +1,16 @@ //! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. +use crate::compiled_views::models::LineageFactRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, LineageInput, NativeCompiledModel, -}; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::types::DeclaredRow; +use crate::compiled_views::types::LineageInput; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// One lineage output with its transform kind and confidence values, and its sources. -pub(crate) struct LineageFactRow { - pub(crate) output_column: String, - pub(crate) transform: String, - pub(crate) confidence: String, - /// `(resource type, resource, column)` of each source. - pub(crate) sources: Vec<(String, String, String)>, -} - const TRANSFORM_VALUES: [&str; 6] = [ "direct", "cast", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs index 046ad1d0f2..ea763d064d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs @@ -2,36 +2,17 @@ use std::fmt::{Debug, Formatter, Result as FmtResult}; -use sqlbuild_project::types::ProjectHost; - -use crate::bindings::_helpers::compiled_views::audit_views::{AuditHandles, AuditProjections}; -use crate::bindings::_helpers::compiled_views::compiled_model_analysis::AnalysisProjections; -use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledProjections; -use crate::bindings::_helpers::compiled_views::function_views::{ - FunctionHandles, FunctionProjections, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputHandles, SchemaHandles, -}; -use crate::bindings::_helpers::compiled_views::seed_views::SeedHandles; -use crate::bindings::_helpers::compiled_views::source_views::{SourceHandles, SourceProjections}; - -/// The extension's host: Python handles and the views built from them. -pub(crate) struct PyProjectHost; - -impl ProjectHost for PyProjectHost { - type ModelInput = ModelInputHandles; - type Schema = SchemaHandles; - type CompiledModel = CompiledProjections; - type Analysis = AnalysisProjections; - type AttachedSource = SourceHandles; - type Source = SourceProjections; - type AttachedFunction = FunctionHandles; - type Function = FunctionProjections; - type AttachedAudit = AuditHandles; - type Audit = AuditProjections; - type Seed = SeedHandles; -} +use crate::compiled_views::_helpers::audit_views::AuditProjections; +use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisProjections; +use crate::compiled_views::_helpers::compiled_model_view::CompiledProjections; +use crate::compiled_views::_helpers::function_views::FunctionProjections; +use crate::compiled_views::_helpers::seed_views::SeedHandles; +use crate::compiled_views::_helpers::source_views::SourceProjections; +use crate::compiled_views::models::AuditHandles; +use crate::compiled_views::models::FunctionHandles; +use crate::compiled_views::models::ModelInputHandles; +use crate::compiled_views::models::SchemaHandles; +use crate::compiled_views::models::SourceHandles; impl Debug for ModelInputHandles { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_reads.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project_reads.rs index b3f0994076..4d92f81fe6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_reads.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::compiled_views::models::NativeProjectView; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs new file mode 100644 index 0000000000..4d4b2e5fc0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs @@ -0,0 +1,270 @@ +//! A compiled project, read from the native compiled project that owns its models. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::graph::models::ResourceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +const MODELS_FIELD: &str = "models"; +const RUN_ID_FIELD: &str = "run_id"; +const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; +const KEY_ATTRIBUTE: &str = "key"; +const DEPS_ATTRIBUTE: &str = "deps"; +pub(crate) const TAGS_ATTRIBUTE: &str = "tags"; +pub(crate) const SCHEMA_ENTRY_ATTRIBUTE: &str = "schema_entry"; +const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; +const NAME_ATTRIBUTE: &str = "name"; + +/// Project inputs held as Python objects: user code, discovery or an integration owns them. +pub(crate) struct BoundaryInputs { + /// The resolved connection, adapter integration values. + pub(crate) effective_connection: Py, + /// Project variables, user values. + pub(crate) effective_vars: Py, + /// The native binding catalog. + pub(crate) binding_catalog: Py, + /// The adapter's lexical syntax. + pub(crate) sql_lexical_syntax: Py, + /// The compile cache directory path. + pub(crate) compile_cache_dir: Py, + /// Discovered loader functions, user callables. + pub(crate) loader_functions: Py, + /// Task/asset/check source locations, without retaining additional Python callables. + pub(crate) python_execution_inputs: BTreeMap, + /// Discovered Python hooks, user callables. + pub(crate) hook_functions: Py, + /// Discovered SQL hook files. + pub(crate) sql_hook_files: Py, + /// Discovered custom materializations. + pub(crate) materialization_files: Py, + /// Public enums, user declaration values. + pub(crate) public_enums: Py, + /// Public constants, user declaration values. + pub(crate) public_constants: Py, + /// Loaded macro modules, user code. + pub(crate) loaded_macros: Py, + /// The external reference resolver integration, or `None`. + pub(crate) external_sql_reference_resolver: Py, + /// The native analysis session, or `None`. + pub(crate) native_session: Py, +} + +/// Compiler-derived project parts still held as Python objects until they are owner facts. +pub(crate) struct CompilerHandles { + /// Effective settings. + pub(crate) settings: Py, + /// Effective scenario config. + pub(crate) scenario: Py, + /// Compiled sources. + pub(crate) sources: Py, + /// Compiled seeds. + pub(crate) seeds: Py, + /// Compiled SQL functions. + pub(crate) functions: Py, + /// Compiled audits. + pub(crate) audits: Py, + /// Compiled SQL tests. + pub(crate) sql_tests: Py, + /// Compiled SQL scenarios. + pub(crate) sql_scenarios: Py, + /// Compiler diagnostics. + pub(crate) diagnostics: Py, + /// Literal relations no resource matched. + pub(crate) unmatched_literal_sql_relations: Py, + /// The declaration scope index. + pub(crate) scope_index: Py, +} + +/// The project-level values of one compiled project view. +pub(crate) struct ProjectFields { + pub(crate) run_id: String, + pub(crate) effective_target_name: Option, + pub(crate) effective_target_database: Option, + pub(crate) effective_target_schema: Option, + pub(crate) sql_analysis_dialect: Option, + pub(crate) enforce_explicit_references: bool, + pub(crate) models: Py, + pub(crate) boundary: BoundaryInputs, + pub(crate) compiler: CompilerHandles, +} + +impl std::fmt::Debug for ProjectFields { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ProjectFields") + .finish_non_exhaustive() + } +} + +impl ProjectFields { + /// The fields named in `fields`, each required. + pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + effective_target_database: required(fields, "effective_target_database")?.extract()?, + effective_target_schema: required(fields, "effective_target_schema")?.extract()?, + sql_analysis_dialect: required(fields, "sql_analysis_dialect")?.extract()?, + enforce_explicit_references: required(fields, ENFORCE_EXPLICIT_FIELD)?.extract()?, + models: required(fields, MODELS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + binding_catalog: required(fields, "binding_catalog")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + loader_functions: required(fields, "loader_functions")?.unbind(), + python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, + hook_functions: required(fields, "hook_functions")?.unbind(), + sql_hook_files: required(fields, "sql_hook_files")?.unbind(), + materialization_files: required(fields, "materialization_files")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + native_session: required(fields, "native_session")?.unbind(), + }, + compiler: CompilerHandles { + settings: required(fields, "settings")?.unbind(), + scenario: required(fields, "scenario")?.unbind(), + sources: required(fields, "sources")?.unbind(), + seeds: required(fields, "seeds")?.unbind(), + functions: required(fields, "functions")?.unbind(), + audits: required(fields, "audits")?.unbind(), + sql_tests: required(fields, "sql_tests")?.unbind(), + sql_scenarios: required(fields, "sql_scenarios")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + unmatched_literal_sql_relations: required( + fields, + "unmatched_literal_sql_relations", + )? + .unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + }, + }) + } + + /// These fields with `changes` applied. + pub(crate) fn changed(&self, py: Python<'_>, changes: &Bound<'_, PyDict>) -> PyResult { + let current: Bound<'_, PyDict> = self.as_fields(py)?; + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown project field {name}" + ))); + } + current.set_item(name, value)?; + } + Self::from_fields(¤t) + } + + /// Every field by name. + pub(crate) fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item("effective_target_database", &self.effective_target_database)?; + fields.set_item("effective_target_schema", &self.effective_target_schema)?; + fields.set_item("sql_analysis_dialect", &self.sql_analysis_dialect)?; + fields.set_item(ENFORCE_EXPLICIT_FIELD, self.enforce_explicit_references)?; + fields.set_item(MODELS_FIELD, self.models.bind(py))?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("binding_catalog", self.boundary.binding_catalog.bind(py))?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; + fields.set_item( + "python_execution_inputs", + &self.boundary.python_execution_inputs, + )?; + fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; + fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; + fields.set_item( + "materialization_files", + self.boundary.materialization_files.bind(py), + )?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item("native_session", self.boundary.native_session.bind(py))?; + fields.set_item("settings", self.compiler.settings.bind(py))?; + fields.set_item("scenario", self.compiler.scenario.bind(py))?; + fields.set_item("sources", self.compiler.sources.bind(py))?; + fields.set_item("seeds", self.compiler.seeds.bind(py))?; + fields.set_item("functions", self.compiler.functions.bind(py))?; + fields.set_item("audits", self.compiler.audits.bind(py))?; + fields.set_item("sql_tests", self.compiler.sql_tests.bind(py))?; + fields.set_item("sql_scenarios", self.compiler.sql_scenarios.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item( + "unmatched_literal_sql_relations", + self.compiler.unmatched_literal_sql_relations.bind(py), + )?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing project field {name}"))) +} + +/// A source, seed or function's graph row, read from its compiled object. +pub(crate) fn resource_row( + resource: &Bound<'_, PyAny>, + tags: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut deps: Vec<(String, String)> = Vec::new(); + for key in native_field(resource, DEPS_ATTRIBUTE)?.try_iter()? { + deps.push(key_row(&key?)?); + } + let mut tag_names: Vec = Vec::new(); + if let Some(tags) = tags { + for tag in tags.try_iter()? { + tag_names.push(tag?.str()?.extract()?); + } + } + Ok(ResourceRow { + key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, + deps, + tags: tag_names, + folder: None, + }) +} + +/// `(resource type, name)` of a `CompiledObjectKey`. +pub(crate) fn key_row(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(key, RESOURCE_TYPE_ATTRIBUTE)? + .str()? + .extract()?, + native_field(key, NAME_ATTRIBUTE)?.extract()?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs index 17ecfeb899..12a33aa75b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs @@ -9,14 +9,15 @@ use pyo3::types::{ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; -use crate::bindings::_helpers::compiled_project::project::{ - AnalysisPy, NativeCompiledProject, ProofPy, model_facts, -}; -use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::compiled_project::main::project::model_facts::model_facts; +use crate::compiled_project::main::schema_facts::entry_schema_fact::entry_schema_fact; +use crate::compiled_project::models::AnalysisPy; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_project::models::ProofPy; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -557,3 +558,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { } Ok(()) } + +/// `declaration_rows` as a Python function. +pub(crate) fn declaration_rows_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(declaration_rows, py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs index 19829fd9a8..7e66f2f7a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs @@ -1,6 +1,7 @@ //! Declared unique keys of project relations, derived natively from their key facts. -use sqlbuild_python_base::positional_record; +use crate::compiled_views::models::RelationKeyRow; +use crate::compiled_views::types::RelationKeys; use std::collections::BTreeMap; use std::path::PathBuf; @@ -13,14 +14,13 @@ use sqlbuild_discovery::project_validation::models::CsvSettings; use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - NativeModelInput, model_relation_key_row, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - SourceKeyFacts, SourceViews, read_source_view, -}; +use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use crate::compiled_views::_helpers::model_input_view::model_relation_key_row; +use crate::compiled_views::_helpers::source_views::SourceKeyFacts; +use crate::compiled_views::_helpers::source_views::read_source_view; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::types::SourceViews; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; @@ -36,22 +36,6 @@ const RELATION_KIND_SOURCE: &str = "source"; /// Native declared columns and model audits used to derive relation keys. type DeclaredKeyFacts<'a> = (&'a [SchemaColumnFact], Vec<(String, Option)>); -positional_record! { - extract, into; - /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. - pub(crate) struct RelationKeyRow { - kind: String, - name: String, - materialized: String, - mode: String, - strategy: String, - has_cursor: bool, - unique_key: Option>, - columns: Vec<(String, Vec, bool)>, - audits: Vec<(String, Option)>, - } -} - /// The key row of one relation from its config values and declared column and audit specs. pub(crate) fn relation_key_row( kind: &str, @@ -150,7 +134,7 @@ fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { } /// `unique_key` as a name list, when it is a name or a non-empty list or tuple of names. -fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { +pub(crate) fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { let Some(value) = values.get_item(UNIQUE_KEY_CONFIG)? else { return Ok(None); }; @@ -200,9 +184,6 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult>)>; - /// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. pub(crate) fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs index 95ebe5b0d6..f463c3b8de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs @@ -3,14 +3,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, model_graph_row, -}; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::compiled_views::_helpers::compiled_model_view::model_graph_row; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSource; +use crate::graph::main::keys::native_key::native_key; +use crate::graph::models::ResourceRow; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs index 1912c33f0f..e0aae31e96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSeed; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs new file mode 100644 index 0000000000..c0ee62033b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs @@ -0,0 +1,128 @@ +//! Seeds paired with their declarations, held by the compiled project, and their views. + +use crate::compiled_views::models::SeedNamespace; +use crate::compiled_views::models::SeedParts; +use crate::compiled_views::types::SeedRecord; +use crate::compiled_views::types::SeedViews; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; + +use crate::compiled_project::models::NativeCompiledProject; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +pub(crate) const SEED_RESOURCE: &str = "seed"; +pub(crate) const TAGS_ATTRIBUTE: &str = "tags"; +const NAME_ATTRIBUTE: &str = "name"; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const STEM_ATTRIBUTE: &str = "stem"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; + +/// A seed's discovered objects and the key its views build. +pub struct SeedHandles { + /// The discovered CSV file. + pub(crate) seed_file: Py, + /// The discovered declaration, whose spec carries user-authored column metadata. + pub(crate) schema_entry: Py, + /// The schema file declaring the seed. + pub(crate) schema_file: Py, + pub(crate) key: PyOnceLock>, +} + +/// Pair each declaration with the CSV file of its name, raising for the first without one. +pub(crate) fn paired_seeds( + declarations: Vec<(Py, Py)>, + seed_files: &[Py], + compile_error: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = compile_error.py(); + let mut names: Vec = Vec::with_capacity(declarations.len()); + for (entry, _) in &declarations { + names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); + } + let mut stems: Vec = Vec::with_capacity(seed_files.len()); + for seed_file in seed_files { + stems.push( + native_field( + &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, + STEM_ATTRIBUTE, + )? + .extract()?, + ); + } + let pairs: Vec = match pair_seed_declarations(&names, &stems) { + Ok(pairs) => pairs, + Err(missing) => { + let schema_file: &Py = &declarations[missing].1; + let relative_path: String = + native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? + .str()? + .extract()?; + let message: String = format!( + "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", + names[missing] + ); + return Err(PyErr::from_value(compile_error.call1((message,))?)); + } + }; + let mut seeds: Vec = Vec::with_capacity(pairs.len()); + for (((schema_entry, schema_file), name), file_index) in + declarations.into_iter().zip(names).zip(pairs) + { + seeds.push(SeedRecord { + name, + destination: None, + host: SeedHandles { + seed_file: seed_files[file_index].clone_ref(py), + schema_entry, + schema_file, + key: PyOnceLock::new(), + }, + }); + } + Ok(seeds) +} + +impl SeedViews for SeedRecord { + fn parts(&self) -> SeedParts<'_> { + SeedParts { + name: &self.name, + seed_file: &self.host.seed_file, + schema_entry: &self.host.schema_entry, + schema_file: &self.host.schema_file, + } + } + + fn set_namespace(&mut self, namespace: SeedNamespace) { + let SeedNamespace { + database, + schema, + logical_database, + logical_schema, + } = namespace; + self.destination = Some(DestinationFact { + database, + schema, + name: self.name.clone(), + qualified_name: None, + logical_schema, + logical_database, + }); + } +} + +pub(crate) fn read_seed<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SeedRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let seed: &SeedRecord = owner + .seeds() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(seed, py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs index 46dc34fa56..ef85363ea9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; +use crate::compiled_views::models::NativeCompiledSource; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs new file mode 100644 index 0000000000..3891fe4ffd --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs @@ -0,0 +1,163 @@ +//! Sources attached by the native loop, held by the compiled project, and their views. + +use crate::compiled_views::models::NativeCompiledSource; +use crate::compiled_views::models::NativeSourceInput; +use crate::compiled_views::types::AttachedSource; +use crate::compiled_views::types::SourceRecord; +use crate::compiled_views::types::SourceViews; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_python_base::positional_record; + +use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::relation_keys::audit_key_rows; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; + +pub(crate) const SOURCE_RESOURCE: &str = "source"; + +positional_record! { + into; + /// `(name, has a loader, database, schema)` of one source. + pub(crate) struct SourceAssemblyRow { + pub(crate) name: String, + pub(crate) managed: bool, + pub(crate) database: Option, + pub(crate) schema: Option, + } +} + +/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. +pub struct SourceKeyFacts { + pub columns: Vec, + pub(crate) audits: Vec<(String, Option)>, +} + +/// A source record's key facts and entries, each built once from its Python values. +pub struct SourceProjections { + key_facts: PyOnceLock, + attached_entry: PyOnceLock>, + assembled_entry: PyOnceLock>, + pub(crate) key: PyOnceLock>, +} + +/// The source record of an attached source, before assembly resolves its namespace. +pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { + SourceRecord { + attached, + namespace: None, + host: SourceProjections { + key_facts: PyOnceLock::new(), + attached_entry: PyOnceLock::new(), + assembled_entry: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } +} + +impl SourceViews for SourceRecord { + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { + self.host.key_facts.get_or_try_init(py, || { + Ok(SourceKeyFacts { + columns: python_column_facts(self.attached.host.columns.bind(py).as_any(), None)?, + audits: audit_key_rows(self.attached.host.audits.bind(py).as_any())?, + }) + }) + } + + fn attached_entry(&self, py: Python<'_>) -> PyResult> { + self.host + .attached_entry + .get_or_try_init(py, || { + let source: &AttachedSource = &self.attached; + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("contract", &source.contract)?; + changes.set_item("database", &source.database)?; + changes.set_item("schema", &source.schema)?; + changes.set_item("table", &source.table)?; + changes.set_item("expression", &source.expression)?; + changes.set_item("description", &source.description)?; + changes.set_item("meta", source.host.meta.bind(py))?; + changes.set_item("columns", source.host.columns.bind(py))?; + changes.set_item("audits", source.host.audits.bind(py))?; + replaced(py, source.host.entry.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } + + fn assembled_entry(&self, py: Python<'_>) -> PyResult> { + let attached: Py = self.attached_entry(py)?; + let Some(Some((database, schema))) = &self.namespace else { + return Ok(attached); + }; + self.host + .assembled_entry + .get_or_try_init(py, || { + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("database", database)?; + changes.set_item("schema", schema)?; + replaced(py, attached.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } +} + +/// `dataclasses.replace(value, **changes)`. +pub(crate) fn replaced( + _py: Python<'_>, + value: &Bound<'_, PyAny>, + changes: &Bound<'_, PyDict>, +) -> PyResult> { + Ok(replaced_keywords(value, changes)?.unbind()) +} + +/// Read the source record behind a native source view, or `None` for any other object. +pub(crate) fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + let py: Python<'py> = view.py(); + if let Ok(input) = view.cast::() { + let input: &NativeSourceInput = input.get(); + return Some(read_source(&input.owner, input.index, py, read)); + } + let Ok(compiled) = view.cast::() else { + return None; + }; + let compiled: &NativeCompiledSource = compiled.get(); + Some(read_source(&compiled.owner, compiled.index, py, read)) +} + +pub(crate) fn read_source<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let source: &SourceRecord = owner + .sources() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(source, py) +} + +/// Keep `attached` in `owner`; return each source's view. +pub(crate) fn retain_sources( + owner: &Bound<'_, NativeCompiledProject>, + attached: Vec, +) -> Vec { + let mut project = owner.borrow_mut(); + let first: usize = project.sources().len(); + let count: usize = attached.len(); + for source in attached { + project.retain_source(source_record(source)); + } + drop(project); + (first..first + count) + .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/constants.rs b/crates/sqlbuild-python-project/src/compiled_views/constants.rs new file mode 100644 index 0000000000..0c64a3a13a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/constants.rs @@ -0,0 +1,80 @@ +//! Public constants of compiled views. + +use crate::compiled_views::_helpers::getters::read_native_audit_input; +use crate::compiled_views::_helpers::getters::read_native_compiled_audit; +use crate::compiled_views::_helpers::getters::read_native_compiled_function; +use crate::compiled_views::_helpers::getters::read_native_compiled_model; +use crate::compiled_views::_helpers::getters::read_native_compiled_seed; +use crate::compiled_views::_helpers::getters::read_native_compiled_source; +use crate::compiled_views::_helpers::getters::read_native_function_input; +use crate::compiled_views::_helpers::getters::read_native_inputs_view; +use crate::compiled_views::_helpers::getters::read_native_model_input; +use crate::compiled_views::_helpers::getters::read_native_project_view; +use crate::compiled_views::_helpers::getters::read_native_seed_input; +use crate::compiled_views::_helpers::getters::read_native_source_input; +use crate::compiled_views::models::NativeAuditInput; +use crate::compiled_views::models::NativeCompiledAudit; +use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeCompiledSeed; +use crate::compiled_views::models::NativeCompiledSource; +use crate::compiled_views::models::NativeFunctionInput; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::models::NativeSeedInput; +use crate::compiled_views::models::NativeSourceInput; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_native_audit_input, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_audit, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_model, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_function, + }, + GetterEntry { + class: class_of::, + read: read_native_function_input, + }, + GetterEntry { + class: class_of::, + read: read_native_inputs_view, + }, + GetterEntry { + class: class_of::, + read: read_native_model_input, + }, + GetterEntry { + class: class_of::, + read: read_native_project_view, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_seed, + }, + GetterEntry { + class: class_of::, + read: read_native_seed_input, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_source, + }, + GetterEntry { + class: class_of::, + read: read_native_source_input, + }, +]; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs new file mode 100644 index 0000000000..b258240250 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `always_run`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `always_run`, as its Python attribute reads it. +#[inline] +pub fn always_run<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::always_run(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs new file mode 100644 index 0000000000..0c6556b10a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `attached_column_name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `attached_column_name`, as its Python attribute reads it. +#[inline] +pub fn attached_column_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::attached_column_name(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs new file mode 100644 index 0000000000..bf80d2713c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `attached_target_kind`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `attached_target_kind`, as its Python attribute reads it. +#[inline] +pub fn attached_target_kind<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::attached_target_kind(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs new file mode 100644 index 0000000000..2dc16e7c44 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `attached_target_name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `attached_target_name`, as its Python attribute reads it. +#[inline] +pub fn attached_target_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::attached_target_name(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs new file mode 100644 index 0000000000..35503fc542 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `definition_name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `definition_name`, as its Python attribute reads it. +#[inline] +pub fn definition_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::definition_name(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs new file mode 100644 index 0000000000..545c290bd7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `description`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `description`, as its Python attribute reads it. +#[inline] +pub fn description<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::description(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs new file mode 100644 index 0000000000..edc2036c70 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `evaluation_mode`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `evaluation_mode`, as its Python attribute reads it. +#[inline] +pub fn evaluation_mode<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::evaluation_mode(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs new file mode 100644 index 0000000000..a21294a9ef --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `evidence_limit`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `evidence_limit`, as its Python attribute reads it. +#[inline] +pub fn evidence_limit<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::evidence_limit(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs new file mode 100644 index 0000000000..9f5441be22 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `evidence_sql`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `evidence_sql`, as its Python attribute reads it. +#[inline] +pub fn evidence_sql<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::evidence_sql(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs new file mode 100644 index 0000000000..9dd0b1121b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `key`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `key`, as its Python attribute reads it. +#[inline] +pub fn key<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::key(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs new file mode 100644 index 0000000000..33dafbc8cb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `measurement_contract`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `measurement_contract`, as its Python attribute reads it. +#[inline] +pub fn measurement_contract<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::measurement_contract(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs new file mode 100644 index 0000000000..1c5abbc2b2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `meta`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `meta`, as its Python attribute reads it. +#[inline] +pub fn meta<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::meta(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs new file mode 100644 index 0000000000..33e8be110a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `minimum_samples`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `minimum_samples`, as its Python attribute reads it. +#[inline] +pub fn minimum_samples<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::minimum_samples(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs new file mode 100644 index 0000000000..16994fd38a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs @@ -0,0 +1,20 @@ +pub mod always_run; +pub mod attached_column_name; +pub mod attached_target_kind; +pub mod attached_target_name; +pub mod definition_name; +pub mod description; +pub mod evaluation_mode; +pub mod evidence_limit; +pub mod evidence_sql; +pub mod key; +pub mod measurement_contract; +pub mod meta; +pub mod minimum_samples; +pub mod name; +pub mod references; +pub mod run_scope; +pub mod scope_deps; +pub mod severity; +pub mod sql_body; +pub mod thresholds; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs new file mode 100644 index 0000000000..8fdc26131f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `name`, as its Python attribute reads it. +#[inline] +pub fn name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::name(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs new file mode 100644 index 0000000000..2a3f18f444 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `references`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `references`, as its Python attribute reads it. +#[inline] +pub fn references<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::references(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs new file mode 100644 index 0000000000..f0eb9d41a8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `run_scope`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `run_scope`, as its Python attribute reads it. +#[inline] +pub fn run_scope<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::run_scope(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs new file mode 100644 index 0000000000..7afcfd5752 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `scope_deps`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `scope_deps`, as its Python attribute reads it. +#[inline] +pub fn scope_deps<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::scope_deps(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs new file mode 100644 index 0000000000..1a5fc0609f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `severity`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `severity`, as its Python attribute reads it. +#[inline] +pub fn severity<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::severity(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs new file mode 100644 index 0000000000..4b1bed68cb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `sql_body`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `sql_body`, as its Python attribute reads it. +#[inline] +pub fn sql_body<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::sql_body(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs new file mode 100644 index 0000000000..e043d67cf1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs @@ -0,0 +1,12 @@ +//! The compiled audit's `thresholds`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::audit_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled audit's `thresholds`, as its Python attribute reads it. +#[inline] +pub fn thresholds<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + audit_reads::thresholds(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs new file mode 100644 index 0000000000..36c30e7160 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs @@ -0,0 +1,11 @@ +//! The audit record of an attached audit, before assembly resolves its scope dependencies. + +use crate::compiled_views::_helpers::audit_views; +use crate::compiled_views::types::AttachedAudit; +use crate::compiled_views::types::AuditRecord; + +/// The audit record of an attached audit, before assembly resolves its scope dependencies. +#[inline] +pub fn audit_record(attached: AttachedAudit) -> AuditRecord { + audit_views::audit_record(attached) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs new file mode 100644 index 0000000000..ef7a0e56da --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs @@ -0,0 +1,13 @@ +//! A native audit view's reference rows, or `None` for a hand-built audit record. + +use crate::compiled_views::_helpers::audit_views; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_project::models::ReferenceRow; + +/// A native audit view's reference rows, or `None` for a hand-built audit record. +#[inline] +pub fn audit_reference_rows(audit: &Bound<'_, PyAny>) -> PyResult>> { + audit_views::audit_reference_rows(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs new file mode 100644 index 0000000000..598f39715c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs @@ -0,0 +1,2 @@ +pub mod audit_record; +pub mod audit_reference_rows; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs new file mode 100644 index 0000000000..f1ead33bf9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs @@ -0,0 +1,12 @@ +//! The compiled function's `arguments`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `arguments`, as its Python attribute reads it. +#[inline] +pub fn arguments<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::arguments(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs new file mode 100644 index 0000000000..0871c4ab5f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs @@ -0,0 +1,12 @@ +//! The compiled function's `body_sql`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `body_sql`, as its Python attribute reads it. +#[inline] +pub fn body_sql<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::body_sql(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs new file mode 100644 index 0000000000..6cce87ce83 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs @@ -0,0 +1,12 @@ +//! The compiled function's `deps`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `deps`, as its Python attribute reads it. +#[inline] +pub fn deps<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::deps(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs new file mode 100644 index 0000000000..cf57c8a8ea --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs @@ -0,0 +1,12 @@ +//! The compiled function's `destination`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `destination`, as its Python attribute reads it. +#[inline] +pub fn destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::destination(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs new file mode 100644 index 0000000000..ec8b8520b2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs @@ -0,0 +1,12 @@ +//! The compiled function's `entry_point`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `entry_point`, as its Python attribute reads it. +#[inline] +pub fn entry_point<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::entry_point(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs new file mode 100644 index 0000000000..80f5d1f42b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs @@ -0,0 +1,12 @@ +//! The compiled function's `fingerprint_destination`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `fingerprint_destination`, as its Python attribute reads it. +#[inline] +pub fn fingerprint_destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::fingerprint_destination(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs new file mode 100644 index 0000000000..e4a166a6fa --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs @@ -0,0 +1,12 @@ +//! The compiled function's `key`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `key`, as its Python attribute reads it. +#[inline] +pub fn key<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::key(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs new file mode 100644 index 0000000000..f460494416 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs @@ -0,0 +1,12 @@ +//! The compiled function's `language`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `language`, as its Python attribute reads it. +#[inline] +pub fn language<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::language(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs new file mode 100644 index 0000000000..04cf303460 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs @@ -0,0 +1,15 @@ +pub mod arguments; +pub mod body_sql; +pub mod deps; +pub mod destination; +pub mod entry_point; +pub mod fingerprint_destination; +pub mod key; +pub mod language; +pub mod name; +pub mod packages; +pub mod relative_path; +pub mod return_columns; +pub mod returns; +pub mod runtime_version; +pub mod source_file_path; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs new file mode 100644 index 0000000000..08624d0217 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs @@ -0,0 +1,12 @@ +//! The compiled function's `name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `name`, as its Python attribute reads it. +#[inline] +pub fn name<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::name(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs new file mode 100644 index 0000000000..2feefc0e07 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs @@ -0,0 +1,12 @@ +//! The compiled function's `packages`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `packages`, as its Python attribute reads it. +#[inline] +pub fn packages<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::packages(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs new file mode 100644 index 0000000000..5a1d4e69ed --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs @@ -0,0 +1,12 @@ +//! The compiled function's `relative_path`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `relative_path`, as its Python attribute reads it. +#[inline] +pub fn relative_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::relative_path(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs new file mode 100644 index 0000000000..1bc51ce82c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs @@ -0,0 +1,12 @@ +//! The compiled function's `return_columns`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `return_columns`, as its Python attribute reads it. +#[inline] +pub fn return_columns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::return_columns(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs new file mode 100644 index 0000000000..96ead2f322 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs @@ -0,0 +1,12 @@ +//! The compiled function's `returns`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `returns`, as its Python attribute reads it. +#[inline] +pub fn returns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::returns(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs new file mode 100644 index 0000000000..560c226b93 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs @@ -0,0 +1,12 @@ +//! The compiled function's `runtime_version`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `runtime_version`, as its Python attribute reads it. +#[inline] +pub fn runtime_version<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::runtime_version(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs new file mode 100644 index 0000000000..b0493f3389 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs @@ -0,0 +1,12 @@ +//! The compiled function's `source_file_path`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::function_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled function's `source_file_path`, as its Python attribute reads it. +#[inline] +pub fn source_file_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + function_reads::source_file_path(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs new file mode 100644 index 0000000000..faecf3109d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs @@ -0,0 +1,11 @@ +//! The function record of an attached function, before assembly resolves its dependencies. + +use crate::compiled_views::_helpers::function_views; +use crate::compiled_views::types::AttachedFunction; +use crate::compiled_views::types::FunctionRecord; + +/// The function record of an attached function, before assembly resolves its dependencies. +#[inline] +pub fn function_record(attached: AttachedFunction) -> FunctionRecord { + function_views::function_record(attached) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs new file mode 100644 index 0000000000..6d63b8eef2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs @@ -0,0 +1 @@ +pub mod function_record; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs new file mode 100644 index 0000000000..acf736cc6e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs @@ -0,0 +1,14 @@ +pub mod audit_reads; +pub mod audit_views; +pub mod function_reads; +pub mod function_views; +pub mod model_input_view; +pub mod model_reads; +pub mod project_reads; +pub mod records; +pub mod relation_keys; +pub mod resource_reads; +pub mod seed_reads; +pub mod seed_views; +pub mod source_reads; +pub mod source_views; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs new file mode 100644 index 0000000000..ab2ee2cd25 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs @@ -0,0 +1,13 @@ +//! The declared column facts of `model_input`'s attached schema, empty without one. + +use crate::compiled_views::_helpers::model_input_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; + +/// The declared column facts of `model_input`'s attached schema, empty without one. +#[inline] +pub fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { + model_input_view::attached_columns(model_input) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs new file mode 100644 index 0000000000..6bcb65e7e3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs @@ -0,0 +1,13 @@ +//! The attached schema of `model_input`; a native input answers without building its entry. + +use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::models::AttachedSchema; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The attached schema of `model_input`; a native input answers without building its entry. +#[inline] +pub fn attached_schema<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { + model_input_view::attached_schema(model_input) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs new file mode 100644 index 0000000000..c1084c5926 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs @@ -0,0 +1,12 @@ +//! The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. + +use crate::compiled_views::_helpers::model_input_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. +#[inline] +pub fn input_analysis_sql(model_input: &Bound<'_, PyAny>, dialect: &str) -> PyResult { + model_input_view::input_analysis_sql(model_input, dialect) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs new file mode 100644 index 0000000000..3b4df41a90 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs @@ -0,0 +1,12 @@ +//! The name of `model_input`, its file stem; a native input answers from its record. + +use crate::compiled_views::_helpers::model_input_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The name of `model_input`, its file stem; a native input answers from its record. +#[inline] +pub fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { + model_input_view::input_model_name(model_input) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs new file mode 100644 index 0000000000..f8ab027bb2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs @@ -0,0 +1,13 @@ +//! Each reference of `model_input` as `(kind, name, package, call argument count)`. + +use crate::compiled_views::_helpers::model_input_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_project::models::ReferenceRow; + +/// Each reference of `model_input` as `(kind, name, package, call argument count)`. +#[inline] +pub fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { + model_input_view::input_reference_rows(model_input) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs new file mode 100644 index 0000000000..c08c043e93 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs @@ -0,0 +1,13 @@ +//! What the analysed SQL of a native model input must show for its SQL macro calls to fit. + +use crate::compiled_views::_helpers::model_input_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; + +/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. +#[inline] +pub fn input_sql_macro_checks(model_input: &Bound<'_, PyAny>) -> PyResult> { + model_input_view::input_sql_macro_checks(model_input) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs new file mode 100644 index 0000000000..7c4abf2a27 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs @@ -0,0 +1,6 @@ +pub mod attached_columns; +pub mod attached_schema; +pub mod input_analysis_sql; +pub mod input_model_name; +pub mod input_reference_rows; +pub mod input_sql_macro_checks; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs new file mode 100644 index 0000000000..55c38f638d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs @@ -0,0 +1,12 @@ +//! Whether the native session validated the model's bindings. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether the native session validated the model's bindings. +#[inline] +pub fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult { + model_reads::binding_validated(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs new file mode 100644 index 0000000000..2b5aa18c10 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs @@ -0,0 +1,12 @@ +//! The model's `CompileModelConfig`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's `CompileModelConfig`. +#[inline] +pub fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::config(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs new file mode 100644 index 0000000000..aefe69ce68 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs @@ -0,0 +1,13 @@ +//! Each declared column of the attached schema as `(name, type)`, `None` without a schema. + +use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::types::DeclaredRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each declared column of the attached schema as `(name, type)`, `None` without a schema. +#[inline] +pub fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { + model_reads::declared_rows(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs new file mode 100644 index 0000000000..f39e51333f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs @@ -0,0 +1,12 @@ +//! `(resource type, name)` of each dependency. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(resource type, name)` of each dependency. +#[inline] +pub fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { + model_reads::dep_rows(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs new file mode 100644 index 0000000000..802de4e9c9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs @@ -0,0 +1,12 @@ +//! The model's dependency `CompiledObjectKey` tuple. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's dependency `CompiledObjectKey` tuple. +#[inline] +pub fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::deps(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs new file mode 100644 index 0000000000..86a11cee6a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs @@ -0,0 +1,12 @@ +//! The model's destination `CompiledRelationLocation`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's destination `CompiledRelationLocation`. +#[inline] +pub fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::destination(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs new file mode 100644 index 0000000000..5477819682 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs @@ -0,0 +1,12 @@ +//! Whether the attached schema declares dynamic column families. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether the attached schema declares dynamic column families. +#[inline] +pub fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { + model_reads::dynamic_columns_declared(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs new file mode 100644 index 0000000000..524a0f1fcc --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs @@ -0,0 +1,12 @@ +//! The model's enum columns by name. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's enum columns by name. +#[inline] +pub fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::enum_columns(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs new file mode 100644 index 0000000000..f81ca7ebdd --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs @@ -0,0 +1,12 @@ +//! The model's `InferredColumn` tuple, or `None`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's `InferredColumn` tuple, or `None`. +#[inline] +pub fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::inferred_columns(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs new file mode 100644 index 0000000000..bd2e6593ff --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs @@ -0,0 +1,12 @@ +//! The names of the inferred output columns, `None` when the shape is unknown. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The names of the inferred output columns, `None` when the shape is unknown. +#[inline] +pub fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { + model_reads::inferred_names(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs new file mode 100644 index 0000000000..4a9f368c29 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs @@ -0,0 +1,13 @@ +//! Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_project::types::InferredRow; + +/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. +#[inline] +pub fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { + model_reads::inferred_rows(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs new file mode 100644 index 0000000000..52b0bd4e51 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs @@ -0,0 +1,13 @@ +//! Each lineage output with its transform kind and confidence values, `None` without lineage. + +use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::models::LineageFactRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each lineage output with its transform kind and confidence values, `None` without lineage. +#[inline] +pub fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { + model_reads::lineage_facts(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs new file mode 100644 index 0000000000..8ab719d265 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs @@ -0,0 +1,13 @@ +//! Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. + +use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::types::LineageInput; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. +#[inline] +pub fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult> { + model_reads::lineage_inputs(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs new file mode 100644 index 0000000000..96e507a09d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs @@ -0,0 +1,24 @@ +pub mod binding_validated; +pub mod config; +pub mod declared_rows; +pub mod dep_rows; +pub mod deps; +pub mod destination; +pub mod dynamic_columns_declared; +pub mod enum_columns; +pub mod inferred_columns; +pub mod inferred_names; +pub mod inferred_rows; +pub mod lineage_facts; +pub mod lineage_inputs; +pub mod model_key; +pub mod model_name; +pub mod name_object; +pub mod output_proven; +pub mod proven_inputs; +pub mod query_sql; +pub mod references; +pub mod relative_path; +pub mod schema_entry; +pub mod star_flags; +pub mod unchecked_columns; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs new file mode 100644 index 0000000000..844a3329d1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs @@ -0,0 +1,12 @@ +//! The model's `CompiledObjectKey`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's `CompiledObjectKey`. +#[inline] +pub fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::model_key(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs new file mode 100644 index 0000000000..596bb0fcd7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs @@ -0,0 +1,12 @@ +//! The model's name. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's name. +#[inline] +pub fn model_name(model: &Bound<'_, PyAny>) -> PyResult { + model_reads::model_name(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs new file mode 100644 index 0000000000..08b1863df9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs @@ -0,0 +1,12 @@ +//! The model's name as a Python string. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's name as a Python string. +#[inline] +pub fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::name_object(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs new file mode 100644 index 0000000000..9140f3c6d6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs @@ -0,0 +1,12 @@ +//! Whether the model's dynamic pivot proof proved its output columns. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether the model's dynamic pivot proof proved its output columns. +#[inline] +pub fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { + model_reads::output_proven(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs new file mode 100644 index 0000000000..24c16211a8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs @@ -0,0 +1,12 @@ +//! The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. +#[inline] +pub fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { + model_reads::proven_inputs(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs new file mode 100644 index 0000000000..93e054b538 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs @@ -0,0 +1,12 @@ +//! The model's compiled query SQL. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's compiled query SQL. +#[inline] +pub fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::query_sql(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs new file mode 100644 index 0000000000..68f59ec296 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs @@ -0,0 +1,12 @@ +//! The model's `CompileSqlReference` tuple. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's `CompileSqlReference` tuple. +#[inline] +pub fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::references(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs new file mode 100644 index 0000000000..cede5426b5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs @@ -0,0 +1,12 @@ +//! The model file's relative path. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model file's relative path. +#[inline] +pub fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::relative_path(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs new file mode 100644 index 0000000000..7c7e9881e9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs @@ -0,0 +1,12 @@ +//! The model's attached `SchemaModelEntry`, or `None`. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model's attached `SchemaModelEntry`, or `None`. +#[inline] +pub fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model_reads::schema_entry(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs new file mode 100644 index 0000000000..21851ebf26 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs @@ -0,0 +1,12 @@ +//! `(has star, star resolved)` of the model's analysed query. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(has star, star resolved)` of the model's analysed query. +#[inline] +pub fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { + model_reads::star_flags(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs new file mode 100644 index 0000000000..ad414a8c0d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs @@ -0,0 +1,12 @@ +//! Output columns whose types semantic recovery forgot, in any order. + +use crate::compiled_views::_helpers::model_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Output columns whose types semantic recovery forgot, in any order. +#[inline] +pub fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { + model_reads::unchecked_columns(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs new file mode 100644 index 0000000000..d62361d306 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs @@ -0,0 +1,12 @@ +//! The project's compiled audits. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled audits. +#[inline] +pub fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::audits(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs new file mode 100644 index 0000000000..d4109dbf56 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs @@ -0,0 +1,12 @@ +//! The project's resource collection named `field`; any other field is read by attribute. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's resource collection named `field`; any other field is read by attribute. +#[inline] +pub fn collection<'py>(project: &Bound<'py, PyAny>, field: &str) -> PyResult> { + project_reads::collection(project, field) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs new file mode 100644 index 0000000000..e828b81594 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs @@ -0,0 +1,12 @@ +//! The project's compiled functions. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled functions. +#[inline] +pub fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::functions(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs new file mode 100644 index 0000000000..6672cad240 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs @@ -0,0 +1,7 @@ +pub mod audits; +pub mod collection; +pub mod functions; +pub mod models; +pub mod seeds; +pub mod sources; +pub mod sql_tests; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs new file mode 100644 index 0000000000..3084971bb6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs @@ -0,0 +1,12 @@ +//! The project's compiled models, in compile order. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled models, in compile order. +#[inline] +pub fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::models(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs new file mode 100644 index 0000000000..2035f6fd86 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs @@ -0,0 +1,12 @@ +//! The project's compiled seeds. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled seeds. +#[inline] +pub fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::seeds(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs new file mode 100644 index 0000000000..8c142e37e9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs @@ -0,0 +1,12 @@ +//! The project's compiled sources. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled sources. +#[inline] +pub fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::sources(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs new file mode 100644 index 0000000000..62008ee4de --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs @@ -0,0 +1,12 @@ +//! The project's compiled SQL tests. + +use crate::compiled_views::_helpers::project_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project's compiled SQL tests. +#[inline] +pub fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + project_reads::sql_tests(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs new file mode 100644 index 0000000000..d89f15e85d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs @@ -0,0 +1,17 @@ +//! The native owner's inputs view when it holds every model input; else a record. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The native owner's inputs view when it holds every model input; else a record. +#[inline] +pub fn compile_inputs_view<'py>( + fields: &Bound<'py, PyDict>, + native_project: Option>, +) -> PyResult> { + records::compile_inputs_view(fields, native_project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs new file mode 100644 index 0000000000..4ac5081e9b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs @@ -0,0 +1,12 @@ +//! The audit as a `CompiledAudit`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. +#[inline] +pub fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + records::compiled_audit_record(audit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs new file mode 100644 index 0000000000..55b535f477 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs @@ -0,0 +1,12 @@ +//! The function as a `CompiledFunction`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The function as a `CompiledFunction`, projected from its owner when it is a native view. +#[inline] +pub fn compiled_function_record<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + records::compiled_function_record(function) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs new file mode 100644 index 0000000000..1aec08f0c2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs @@ -0,0 +1,12 @@ +//! The model as a `CompiledModel`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model as a `CompiledModel`, projected from its owner when it is a native view. +#[inline] +pub fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + records::compiled_model_record(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs new file mode 100644 index 0000000000..ba6c6ac76b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs @@ -0,0 +1,15 @@ +//! The compile's retained project, or one recorded from a hand-built project. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compile's retained project, or one recorded from a hand-built project. +#[inline] +pub fn compiled_project_facts<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + records::compiled_project_facts(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs new file mode 100644 index 0000000000..784aa64fdb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs @@ -0,0 +1,12 @@ +//! The seed as a `CompiledSeed`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. +#[inline] +pub fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + records::compiled_seed_record(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs new file mode 100644 index 0000000000..001211900c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs @@ -0,0 +1,12 @@ +//! The source as a `CompiledSource`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The source as a `CompiledSource`, projected from its owner when it is a native view. +#[inline] +pub fn compiled_source_record<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + records::compiled_source_record(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs new file mode 100644 index 0000000000..1b23b33be0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs @@ -0,0 +1,16 @@ +//! Enum and constant declarations as native rows, in the given order. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyList; + +/// Enum and constant declarations as native rows, in the given order. +#[inline] +pub fn declaration_rows<'py>( + enums: &Bound<'py, PyAny>, + constants: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + records::declaration_rows(enums, constants) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs new file mode 100644 index 0000000000..87b1d93dac --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs @@ -0,0 +1,11 @@ +//! `declaration_rows` as a Python function. + +use crate::compiled_views::_helpers::records; + +/// `declaration_rows` as a Python function. +#[inline] +pub fn declaration_rows_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + records::declaration_rows_function(py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs new file mode 100644 index 0000000000..c0bcd3fbe1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs @@ -0,0 +1,16 @@ +//! The compile inputs with `changes`; a native view shares its owner's model inputs. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The compile inputs with `changes`; a native view shares its owner's model inputs. +#[inline] +pub fn inputs_with<'py>( + inputs: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + records::inputs_with(inputs, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs new file mode 100644 index 0000000000..10dc87056b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs @@ -0,0 +1,17 @@ +pub mod compile_inputs_view; +pub mod compiled_audit_record; +pub mod compiled_function_record; +pub mod compiled_model_record; +pub mod compiled_project_facts; +pub mod compiled_seed_record; +pub mod compiled_source_record; +pub mod declaration_rows; +pub mod declaration_rows_function; +pub mod inputs_with; +pub mod plain_typed_value; +pub mod project_with; +pub mod record_model_analyses; +pub mod register_records; +pub mod replace; +pub mod with_binding_diagnostics; +pub mod with_type_recovery; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs new file mode 100644 index 0000000000..c6e38b4743 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs @@ -0,0 +1,12 @@ +//! A typed SQL value as the plain value native consumers read. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A typed SQL value as the plain value native consumers read. +#[inline] +pub fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + records::plain_typed_value(value) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs new file mode 100644 index 0000000000..b078c9acd5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs @@ -0,0 +1,16 @@ +//! The project with `changes`; a native view shares its owner's models. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The project with `changes`; a native view shares its owner's models. +#[inline] +pub fn project_with<'py>( + project: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + records::project_with(project, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs new file mode 100644 index 0000000000..bd5c8b11fe --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs @@ -0,0 +1,16 @@ +//! Transfer each assembled model's typed analysis to its retained facts; return how many. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Transfer each assembled model's typed analysis to its retained facts; return how many. +#[inline] +pub fn record_model_analyses<'py>( + project: &Bound<'py, NativeCompiledProject>, + models: &Bound<'py, PyAny>, +) -> PyResult { + records::record_model_analyses(project, models) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs new file mode 100644 index 0000000000..d431a08dc5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs @@ -0,0 +1,12 @@ +//! Register the records functions and classes. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the records functions and classes. +#[inline] +pub fn register_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + records::register(module) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs new file mode 100644 index 0000000000..112eec91fb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs @@ -0,0 +1,15 @@ +//! `dataclasses.replace(value, **changes)`. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `dataclasses.replace(value, **changes)`. +#[inline] +pub fn replace<'py>( + value: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + records::replace(value, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs new file mode 100644 index 0000000000..b666daaa7e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs @@ -0,0 +1,15 @@ +//! The model with these final binding diagnostics. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The model with these final binding diagnostics. +#[inline] +pub fn with_binding_diagnostics<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + records::with_binding_diagnostics(model, diagnostics) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs b/crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs new file mode 100644 index 0000000000..6a4253eeda --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs @@ -0,0 +1,17 @@ +//! The model with its recovered bindings and its poisoned outputs' types forgotten. + +use crate::compiled_views::_helpers::records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// The model with its recovered bindings and its poisoned outputs' types forgotten. +#[inline] +pub fn with_type_recovery<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, + poisoned: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + records::with_type_recovery(model, diagnostics, poisoned) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs new file mode 100644 index 0000000000..b0f600536b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs @@ -0,0 +1,19 @@ +//! The key row of a seed, source or hand-built model from its config values and schema entry. + +use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::models::RelationKeyRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The key row of a seed, source or hand-built model from its config values and schema entry. +#[inline] +pub fn entry_relation_key_row( + kind: &str, + name: String, + values: &Bound<'_, PyDict>, + entry: &Bound<'_, PyAny>, +) -> PyResult { + relation_keys::entry_relation_key_row(kind, name, values, entry) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs new file mode 100644 index 0000000000..29ef53105c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs @@ -0,0 +1,7 @@ +pub mod entry_relation_key_row; +pub mod model_key_row; +pub mod one_row_seed_names; +pub mod project_relation_keys; +pub mod register_relation_keys; +pub mod relation_keys; +pub mod source_key_input; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs new file mode 100644 index 0000000000..59f9850952 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs @@ -0,0 +1,13 @@ +//! The declared key facts of a model input or compiled model. + +use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::models::RelationKeyRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The declared key facts of a model input or compiled model. +#[inline] +pub fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { + relation_keys::model_key_row(model) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs new file mode 100644 index 0000000000..86580705ae --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs @@ -0,0 +1,12 @@ +//! Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. + +use crate::compiled_views::_helpers::relation_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. +#[inline] +pub fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { + relation_keys::one_row_seed_names(seeds) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs new file mode 100644 index 0000000000..02e71c03f3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs @@ -0,0 +1,13 @@ +//! Declared keys of a compiled project's models, seeds and sources. + +use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::types::RelationKeys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Declared keys of a compiled project's models, seeds and sources. +#[inline] +pub fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { + relation_keys::project_relation_keys(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs new file mode 100644 index 0000000000..843dca6bc3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs @@ -0,0 +1,12 @@ +//! Register the relation keys functions and classes. + +use crate::compiled_views::_helpers::relation_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the relation keys functions and classes. +#[inline] +pub fn register_relation_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { + relation_keys::register(module) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs new file mode 100644 index 0000000000..e8b531ca28 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs @@ -0,0 +1,20 @@ +//! Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. + +use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::models::RelationKeyRow; +use crate::compiled_views::types::RelationKeys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. +#[inline] +pub fn relation_keys<'py>( + py: Python<'py>, + rows: Vec, + (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + one_row_seeds: &[String], +) -> PyResult { + relation_keys::relation_keys(py, rows, (seeds, sources), one_row_seeds) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs new file mode 100644 index 0000000000..42c8554a90 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs @@ -0,0 +1,12 @@ +//! What `relation_keys` reads for a source: its native view itself, or its `source_entry`. + +use crate::compiled_views::_helpers::relation_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. +#[inline] +pub fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + relation_keys::source_key_input(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs new file mode 100644 index 0000000000..50785fde2e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs @@ -0,0 +1,12 @@ +//! A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::_helpers::resource_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. +#[inline] +pub fn destination<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + resource_reads::destination(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs new file mode 100644 index 0000000000..f49f015bce --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs @@ -0,0 +1,13 @@ +//! A resource's graph facts without constructing its public keys or dependency tuple. + +use crate::compiled_views::_helpers::resource_reads; +use crate::graph::models::ResourceRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's graph facts without constructing its public keys or dependency tuple. +#[inline] +pub fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { + resource_reads::graph_row(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs new file mode 100644 index 0000000000..1ec3c3cf2f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs @@ -0,0 +1,12 @@ +//! A resource's `key` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::_helpers::resource_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's `key` from whichever native view it is; any other object is read by attribute, counted. +#[inline] +pub fn key<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + resource_reads::key(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs new file mode 100644 index 0000000000..0fc77e0a6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs @@ -0,0 +1,6 @@ +pub mod destination; +pub mod graph_row; +pub mod key; +pub mod name; +pub mod references; +pub mod schema_entry; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs new file mode 100644 index 0000000000..c5914046aa --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs @@ -0,0 +1,12 @@ +//! A resource's `name` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::_helpers::resource_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's `name` from whichever native view it is; any other object is read by attribute, counted. +#[inline] +pub fn name<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + resource_reads::name(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs new file mode 100644 index 0000000000..35cce1730e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs @@ -0,0 +1,12 @@ +//! A resource's `references` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::_helpers::resource_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's `references` from whichever native view it is; any other object is read by attribute, counted. +#[inline] +pub fn references<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + resource_reads::references(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs new file mode 100644 index 0000000000..c2815e2fcf --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs @@ -0,0 +1,12 @@ +//! A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::_helpers::resource_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. +#[inline] +pub fn schema_entry<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + resource_reads::schema_entry(resource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs new file mode 100644 index 0000000000..ff40d3e05a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `destination`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `destination`, as its Python attribute reads it. +#[inline] +pub fn destination<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::destination(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs new file mode 100644 index 0000000000..9bd052d3e3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `external`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `external`, as its Python attribute reads it. +#[inline] +pub fn external<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::external(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs new file mode 100644 index 0000000000..dd87458694 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `key`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `key`, as its Python attribute reads it. +#[inline] +pub fn key<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::key(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs new file mode 100644 index 0000000000..f935135d3b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs @@ -0,0 +1,6 @@ +pub mod destination; +pub mod external; +pub mod key; +pub mod name; +pub mod schema_entry; +pub mod seed_file; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs new file mode 100644 index 0000000000..287b4b749b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `name`, as its Python attribute reads it. +#[inline] +pub fn name<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::name(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs new file mode 100644 index 0000000000..e13a77f862 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `schema_entry`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `schema_entry`, as its Python attribute reads it. +#[inline] +pub fn schema_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::schema_entry(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs new file mode 100644 index 0000000000..48efeea871 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs @@ -0,0 +1,12 @@ +//! The compiled seed's `seed_file`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::seed_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled seed's `seed_file`, as its Python attribute reads it. +#[inline] +pub fn seed_file<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + seed_reads::seed_file(seed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs new file mode 100644 index 0000000000..4ad797daa4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs @@ -0,0 +1 @@ +pub mod paired_seeds; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs new file mode 100644 index 0000000000..3b5ea5ef51 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs @@ -0,0 +1,18 @@ +//! Pair each declaration with the CSV file of its name, raising for the first without one. + +use crate::compiled_views::_helpers::seed_views; +use crate::compiled_views::types::SeedRecord; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Pair each declaration with the CSV file of its name, raising for the first without one. +#[inline] +pub fn paired_seeds( + declarations: Vec<(Py, Py)>, + seed_files: &[Py], + compile_error: &Bound<'_, PyAny>, +) -> PyResult> { + seed_views::paired_seeds(declarations, seed_files, compile_error) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs new file mode 100644 index 0000000000..75c51ec5f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs @@ -0,0 +1,12 @@ +//! The compiled source's `key`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::source_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled source's `key`, as its Python attribute reads it. +#[inline] +pub fn key<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + source_reads::key(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs new file mode 100644 index 0000000000..92c5d37228 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs @@ -0,0 +1,4 @@ +pub mod key; +pub mod name; +pub mod source_entry; +pub mod source_file; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs new file mode 100644 index 0000000000..43b6240a06 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs @@ -0,0 +1,12 @@ +//! The compiled source's `name`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::source_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled source's `name`, as its Python attribute reads it. +#[inline] +pub fn name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + source_reads::name(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs new file mode 100644 index 0000000000..069d75749e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs @@ -0,0 +1,12 @@ +//! The compiled source's `source_entry`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::source_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled source's `source_entry`, as its Python attribute reads it. +#[inline] +pub fn source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + source_reads::source_entry(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs new file mode 100644 index 0000000000..6efdf8210f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs @@ -0,0 +1,12 @@ +//! The compiled source's `source_file`, as its Python attribute reads it. + +use crate::compiled_views::_helpers::source_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The compiled source's `source_file`, as its Python attribute reads it. +#[inline] +pub fn source_file<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + source_reads::source_file(source) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs new file mode 100644 index 0000000000..5a8c6e25e0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs @@ -0,0 +1,3 @@ +pub mod read_source_view; +pub mod replaced; +pub mod retain_sources; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs new file mode 100644 index 0000000000..011b217b41 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs @@ -0,0 +1,17 @@ +//! Read the source record behind a native source view, or `None` for any other object. + +use crate::compiled_views::_helpers::source_views; +use crate::compiled_views::types::SourceRecord; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Read the source record behind a native source view, or `None` for any other object. +#[inline] +pub fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + source_views::read_source_view(view, read) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs new file mode 100644 index 0000000000..c6e02aa97c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs @@ -0,0 +1,19 @@ +//! `dataclasses.replace(value, **changes)`. + +use crate::compiled_views::_helpers::source_views; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// `dataclasses.replace(value, **changes)`. +#[inline] +pub fn replaced( + _py: Python<'_>, + value: &Bound<'_, PyAny>, + changes: &Bound<'_, PyDict>, +) -> PyResult> { + source_views::replaced(_py, value, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs new file mode 100644 index 0000000000..9837fa8c19 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs @@ -0,0 +1,16 @@ +//! Keep `attached` in `owner`; return each source's view. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::source_views; +use crate::compiled_views::models::NativeSourceInput; +use crate::compiled_views::types::AttachedSource; +use pyo3::prelude::Bound; + +/// Keep `attached` in `owner`; return each source's view. +#[inline] +pub fn retain_sources( + owner: &Bound<'_, NativeCompiledProject>, + attached: Vec, +) -> Vec { + source_views::retain_sources(owner, attached) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/mod.rs new file mode 100644 index 0000000000..0cdb75ac22 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/mod.rs @@ -0,0 +1,7 @@ +//! Python views of the compiled project: models, inputs, audits, functions, seeds, sources and their reads. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs new file mode 100644 index 0000000000..5eac042e4e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -0,0 +1,2252 @@ +//! Public records of compiled views. + +use crate::compiled_project::main::rows::reference_row_tuple::reference_row_tuple; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_project::models::ReferenceTuple; +use crate::compiled_views::_helpers::audit_views::AuditProjections; +use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisProjections; +use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisRecord; +use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisViews; +use crate::compiled_views::_helpers::compiled_model_view::AuditedSchema; +use crate::compiled_views::_helpers::compiled_model_view::CompiledArguments; +use crate::compiled_views::_helpers::compiled_model_view::CompiledProjections; +use crate::compiled_views::_helpers::compiled_model_view::CompiledRecord; +use crate::compiled_views::_helpers::compiled_model_view::DestinationRow; +use crate::compiled_views::_helpers::compiled_model_view::MODEL_RESOURCE; +use crate::compiled_views::_helpers::compiled_model_view::ModelOverlay; +use crate::compiled_views::_helpers::compiled_model_view::NULLABLE; +use crate::compiled_views::_helpers::compiled_model_view::SQL_ANALYSIS_KEY; +use crate::compiled_views::_helpers::compiled_model_view::ViewProjections; +use crate::compiled_views::_helpers::compiled_model_view::casefolded_families; +use crate::compiled_views::_helpers::compiled_model_view::compiled_of; +use crate::compiled_views::_helpers::compiled_model_view::compiled_record; +use crate::compiled_views::_helpers::compiled_model_view::defaulted_destination; +use crate::compiled_views::_helpers::compiled_model_view::lineage_input; +use crate::compiled_views::_helpers::compiled_model_view::model_graph_row; +use crate::compiled_views::_helpers::compiled_model_view::model_name; +use crate::compiled_views::_helpers::compiled_model_view::object_key; +use crate::compiled_views::_helpers::compiled_model_view::relation_location; +use crate::compiled_views::_helpers::function_views::FunctionProjections; +use crate::compiled_views::_helpers::function_views::NOT_ASSEMBLED; +use crate::compiled_views::_helpers::function_views::read_function; +use crate::compiled_views::_helpers::inputs_view::InputsFields; +use crate::compiled_views::_helpers::model_input_view::InputViewClasses; +use crate::compiled_views::_helpers::model_input_view::attached_schema_name; +use crate::compiled_views::_helpers::model_input_view::authored_expansion; +use crate::compiled_views::_helpers::model_input_view::expansion_object; +use crate::compiled_views::_helpers::model_input_view::model_relation_key_row; +use crate::compiled_views::_helpers::model_input_view::query_sql_text; +use crate::compiled_views::_helpers::model_input_view::read_input; +use crate::compiled_views::_helpers::model_input_view::read_input_with_owner; +use crate::compiled_views::_helpers::model_input_view::references; +use crate::compiled_views::_helpers::model_input_view::rejected_opt_out; +use crate::compiled_views::_helpers::model_input_view::schema_entry; +use crate::compiled_views::_helpers::model_input_view::view_classes; +use crate::compiled_views::_helpers::project_view::CompilerHandles; +use crate::compiled_views::_helpers::project_view::ProjectFields; +use crate::compiled_views::_helpers::project_view::SCHEMA_ENTRY_ATTRIBUTE; +use crate::compiled_views::_helpers::project_view::resource_row; +use crate::compiled_views::_helpers::seed_views::SEED_RESOURCE; +use crate::compiled_views::_helpers::seed_views::SeedHandles; +use crate::compiled_views::_helpers::seed_views::read_seed; +use crate::compiled_views::_helpers::source_views::SOURCE_RESOURCE; +use crate::compiled_views::_helpers::source_views::SourceAssemblyRow; +use crate::compiled_views::_helpers::source_views::SourceProjections; +use crate::compiled_views::_helpers::source_views::read_source; +use crate::compiled_views::types::AttachedSource; +use crate::compiled_views::types::DeclaredRow; +use crate::compiled_views::types::LineageInput; +use crate::compiled_views::types::ModelInputRecord; +use crate::compiled_views::types::SourceViews; +use crate::graph::main::project_graph::project_graph::project_graph; +use crate::graph::models::NativeProjectGraph; +use crate::graph::models::ResourceRow; +use pyo3::FromPyObject; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyFrozenSet; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use pyo3::types::PyTupleMethods; +use sqlbuild_analysis::assembly::analysis_session::main::contract_proof_json::contract_proof_json; +use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::compiled_project::models::DynamicProofFact; +use sqlbuild_analysis::compiled_project::models::ModelAnalysisFact; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::models::ExpansionRecord; +use sqlbuild_project::models::ModelMetadataTexts; +use sqlbuild_project::types::InferredRow; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_project::types::ProjectHost; +use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::HeaderFamilies; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::path::PathBuf; +use std::sync::Arc; + +/// An attached audit's user values. +pub struct AuditHandles { + /// The discovered (or built-in) audit file and block the SQL comes from. + pub audit_file: Py, + pub audit_block: Py, + /// The authored measurement policy specs, carrying user values. + pub thresholds: Py, + pub minimum_samples: Py, + pub evidence_limit: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// One attached audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeAuditInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeAuditInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledAudit { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledAudit { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +impl NativeCompiledAudit { + /// The `CompiledAudit` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.scope_deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.definition_name(py)?.into_bound_py_any(py)?, + self.audit_file(py)?.into_bound(py), + self.audit_block(py)?.into_bound(py), + self.sql_body(py)?.into_bound_py_any(py)?, + self.evaluation_mode(py)?.into_bound(py), + self.measurement_contract(py)?.into_bound(py), + self.thresholds(py)?.into_bound(py), + self.minimum_samples(py)?.into_bound(py), + self.measure_sql(py)?.into_bound_py_any(py)?, + self.evidence_sql(py)?.into_bound_py_any(py)?, + self.evidence_limit(py)?.into_bound(py), + self.references(py)?.into_bound(py).into_any(), + self.attached_target_kind(py)?, + self.attached_target_name(py)?.into_bound_py_any(py)?, + self.attached_column_name(py)?.into_bound_py_any(py)?, + self.severity(py)?, + self.run_scope(py)?.into_bound_py_any(py)?, + pyo3::types::PyBool::new(py, self.always_run(py)?) + .to_owned() + .into_any(), + self.description(py)?.into_bound_py_any(py)?, + self.meta(py)?.into_bound(py), + ]) + } +} + +/// A model's SQL rule inputs; `excluded` is `None` when its config values could not be read natively. +#[derive(Debug)] +pub struct SqlRuleInputs { + pub posix_path: String, + pub excluded: Option, + pub key: Option, +} + +/// What a model's SQL rule cache key reads. +#[derive(Debug)] +pub struct SqlRuleKey { + pub authored_sql: String, + pub query_sql: String, + /// The canonical JSON text of the model's dynamic pivot proof. + pub contract: Option, +} + +/// The Python classes a compiled model's projected attributes are built with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct CompiledViewClasses<'py> { + pub(crate) object_key: Bound<'py, PyAny>, + pub(crate) relation_location: Bound<'py, PyAny>, + pub inferred_column: Bound<'py, PyAny>, + pub nullability: Bound<'py, PyAny>, + pub compiler_diagnostic: Bound<'py, PyAny>, + pub source_location: Bound<'py, PyAny>, + pub compact_lineage_facts: Bound<'py, PyAny>, + pub lineage_column_fact: Bound<'py, PyAny>, + pub lineage_source_fact: Bound<'py, PyAny>, + pub contract_proof: Bound<'py, PyAny>, + pub family_proof: Bound<'py, PyAny>, +} + +/// One compiled model, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledModel { + pub(crate) owner: Py, + pub(crate) index: usize, + pub(crate) overlay: ModelOverlay, + pub(crate) projections: ViewProjections, +} + +impl NativeCompiledModel { + pub fn new(owner: Py, index: usize) -> Self { + Self::with_overlay(owner, index, ModelOverlay::default()) + } + + pub(crate) fn with_overlay( + owner: Py, + index: usize, + overlay: ModelOverlay, + ) -> Self { + Self { + owner, + index, + overlay, + projections: ViewProjections::default(), + } + } + + pub(crate) fn changed(&self, py: Python<'_>, change: impl FnOnce(&mut ModelOverlay)) -> Self { + let mut overlay: ModelOverlay = self.overlay.clone_ref(py); + change(&mut overlay); + Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) + } + + pub fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } + + pub(crate) fn compiled<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&CompiledRecord, Python<'py>) -> PyResult, + ) -> PyResult { + self.read(py, |record, py| read(compiled_of(record)?, py)) + } + + /// Read the effective destination, including namespace overlays, without a Python projection. + pub fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + + /// This view's typed analysis as the owner's model facts read it, and the model's name. + pub fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { + self.read(py, |record, _| { + let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; + let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); + let mut inferred_columns: Vec<(String, Option)> = + Vec::with_capacity(rows.len()); + for (name, column_type, _) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + inferred_columns.push((name.clone(), column_type.clone().filter(|_| !poisoned))); + } + let dynamic_proof: Option = + analysis.proof.as_ref().map(|proof| DynamicProofFact { + output_proven: proof.output_proven, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + families: casefolded_families(&proof.families), + failure_reason: proof.failure_reason.clone(), + named_families: proof.families.clone(), + }); + let fact: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof, + inferred: analysis.inferred.is_some(), + nullable_columns: rows + .iter() + .map(|(_, _, nullability)| nullability == NULLABLE) + .collect(), + fast_lineage_has_star: analysis.has_star, + unchecked_output_columns: self.overlay.poisoned.clone(), + }; + Ok((model_name(record), fact)) + }) + } + + /// Each output column of the kept lineage with its `(resource name, column)` sources. + pub fn lineage_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .lineage_rows() + .map(|rows| rows.iter().map(lineage_input).collect())) + }) + } + + /// Whether the model's dynamic pivot proof proved its output columns. + pub fn output_proven(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .is_some_and(|proof| proof.output_proven)) + }) + } + + /// Output columns whose types semantic recovery forgot. + pub fn unchecked_columns(&self) -> &[String] { + &self.overlay.poisoned + } + + /// The kept lineage rows with their transform and confidence codes, or `None`. + pub fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) + }) + } + + /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. + pub fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .filter(|proof| proof.output_proven) + .map(|proof| proof.input_relations.clone())) + }) + } + + /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. + pub fn declared_columns(&self, py: Python<'_>) -> PyResult>> { + self.read(py, |record, py| { + record + .schema + .as_ref() + .map(|schema| schema.host.columns.project(py).map(Bound::unbind)) + .transpose() + }) + } + + /// The attached schema's entry audits, its column specs and each audited column's index and name. + pub fn schema_audits_and_columns(&self, py: Python<'_>) -> PyResult> { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + let (Some(schema), Some(fact)) = (&record.schema, owner.input_schema_fact(record)) + else { + return Ok(None); + }; + let audited: Vec<(usize, String)> = fact + .columns + .iter() + .enumerate() + .filter(|(_, column)| !column.audit_definitions.is_empty()) + .map(|(index, column)| (index, column.name.clone())) + .collect(); + Ok(Some(( + schema.host.audits.clone(), + schema.host.columns.clone(), + audited, + ))) + }) + } + + /// What SQL rules select the model by, plus its cache key inputs when `keyed`. + pub fn sql_rule_inputs(&self, py: Python<'_>, keyed: bool) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + let facts: &CompiledModelFacts = + model_facts(owner.facts(), &record.name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + let excluded: Option = match &facts.config.encoding_error { + Some(_) => None, + None => Some( + facts.config.values.get(SQL_ANALYSIS_KEY) + == Some(&serde_json::Value::Bool(false)), + ), + }; + let key = keyed + .then(|| -> PyResult { + Ok(SqlRuleKey { + authored_sql: facts.authored_sql.to_string(), + query_sql: facts.query_sql.to_string(), + contract: compiled_of(record)? + .analysis + .proof + .as_ref() + .map(contract_proof_json), + }) + }) + .transpose()?; + Ok(SqlRuleInputs { + posix_path: facts.relative_path.clone(), + excluded, + key, + }) + }) + } + + /// The model's name, `str(relative_path)` and authored file text. + pub fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { + self.read(py, |record, py| { + let model_file: &NativeModelFile = record.host.model_file.get(); + Ok(( + model_name(record), + model_file.relative_text(py)?, + Arc::clone(model_file.contents_text()), + )) + }) + } + + /// The name, compiled query and authored file of the model, and whether it calls a function. + pub fn metadata_texts( + &self, + py: Python<'_>, + function_kinds: &[&str], + ) -> PyResult { + self.read(py, |record, _| { + Ok(ModelMetadataTexts { + name: model_name(record), + query_sql: record.query_sql.to_string(), + authored_sql: record.host.model_file.get().contents_text().to_string(), + calls_functions: record + .references + .iter() + .any(|reference| function_kinds.contains(&reference.kind.as_str())), + }) + }) + } + + /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. + pub fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { + self.read(py, |record, py| { + Ok(( + record.host.model_file.get().posix_path(py)?, + Arc::clone(&record.query_sql), + )) + }) + } + + /// The model's authored file text and expanded query, from its recorded facts. + pub fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) + } + + pub(crate) fn model_file( + &self, + py: Python<'_>, + read: impl FnOnce(&NativeModelFile) -> PyResult, + ) -> PyResult { + self.read(py, |record, _| read(record.host.model_file.get())) + } +} + +#[pymethods] +impl NativeCompiledModel { + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + compiled + .host + .key + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record)) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + compiled + .host + .deps + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut keys: Vec> = Vec::with_capacity(compiled.deps.len()); + for (kind, name) in &compiled.deps { + keys.push(object_key(&classes.compiled, kind, name)?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, keys)?.unbind()) + }) + .map(|deps| deps.clone_ref(py)) + }) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _| Ok(model_name(record))) + } + + #[getter] + pub fn relative_path(&self, py: Python<'_>) -> PyResult> { + self.model_file(py, |file| file.relative_path(py)) + } + + #[getter] + pub fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) + } + + #[getter] + pub fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) + } + + #[getter] + pub fn destination(&self, py: Python<'_>) -> PyResult> { + self.read_destination(py, |destination| { + self.projections + .destination + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + relation_location(&classes.compiled, destination) + }) + .map(|destination| destination.clone_ref(py)) + }) + } + + #[getter] + pub fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + pub fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + pub fn relation_key_row(&self, py: Python<'_>) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + model_relation_key_row(owner, record, py) + }) + } + + /// `(resource type, name)` of the model's key. + #[getter] + pub fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.read(py, |record, _| { + Ok((MODEL_RESOURCE.to_owned(), model_name(record))) + }) + } + + /// `(resource type, name)` of each dependency. + #[getter] + pub fn dep_rows(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) + } + + /// `(database, schema, name)` of the model's destination. + #[getter] + pub fn destination_row(&self, py: Python<'_>) -> PyResult { + self.read_destination(py, |destination| { + Ok(( + destination.database.clone(), + destination.schema.clone(), + destination.name.clone(), + )) + }) + } + + /// Each inferred column as `(name, type, nullability)`, or `None` when none were inferred. + #[getter] + pub fn inferred_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + let Some(rows) = &compiled.analysis.inferred else { + return Ok(None); + }; + let mut shown: Vec = Vec::with_capacity(rows.len()); + for (name, column_type, nullability) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + let column_type: Option = if poisoned { None } else { column_type.clone() }; + shown.push((name.clone(), column_type, nullability.clone())); + } + Ok(Some(shown)) + }) + } + + /// Each declared column of the attached schema as `(name, type)`, or `None` without one. + #[getter] + pub fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + let Some(schema) = &facts.schema else { + return Ok(None); + }; + Ok(Some( + schema + .columns + .iter() + .map(|column| (column.name.clone(), column.data_type.clone())) + .collect(), + )) + } + + /// Whether the attached schema declares dynamic columns. + #[getter(has_dynamic_columns)] + pub fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _py| { + Ok(record + .schema + .as_ref() + .is_some_and(|schema| !schema.host.dynamic_columns.specs.is_empty())) + }) + } + + /// Inferred columns, else the attached schema's declared columns, else zero. + #[getter] + pub fn output_column_count(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _py| { + if let Some(rows) = &compiled_of(record)?.analysis.inferred { + return Ok(rows.len()); + } + Ok(record + .schema + .as_ref() + .map_or(0, |schema| schema.host.columns.specs.len())) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + #[getter] + pub fn inferred_columns(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + self.projections + .inferred_columns + .get_or_try_init(py, || { + let Some(rows) = &compiled.analysis.inferred else { + return Ok::<_, pyo3::PyErr>(py.None()); + }; + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut columns: Vec> = Vec::with_capacity(rows.len()); + for row in rows { + let poisoned_row: InferredRow; + let row = if self.overlay.poisoned.contains(&row.0) { + poisoned_row = (row.0.clone(), None, row.2.clone()); + &poisoned_row + } else { + row + }; + columns.push(self.owner.bind(py).borrow().inferred_column( + py, + &classes.compiled, + row, + )?); + } + Ok(PyTuple::new(py, columns)?.into_any().unbind()) + }) + .map(|columns| columns.clone_ref(py)) + }) + } + + /// The upstream source count of the model's lineage facts, `None` without them. + pub fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) + } + + #[getter] + pub fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.lineage_object(py, &classes.compiled) + }) + } + + #[getter] + pub fn fast_lineage_has_star(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.has_star)) + } + + #[getter] + pub fn fast_lineage_star_resolved(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.star_resolved)) + } + + #[getter] + pub fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.contents(py))) + } + + #[getter] + pub fn authored_query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.query_sql(py))) + } + + #[getter] + pub fn output_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| file.output_column_locations(py)) + } + + #[getter] + pub fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { + self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) + } + + #[getter] + pub fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.compiled(py, |compiled, py| PyTuple::new(py, &compiled.macro_deps)) + } + + #[getter] + pub fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) + } + + #[getter] + pub fn binding_diagnostics(&self, py: Python<'_>) -> PyResult> { + if let Some(diagnostics) = &self.overlay.binding_diagnostics { + return Ok(diagnostics.clone_ref(py)); + } + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + let name: String = model_name(record); + let path: Bound<'_, PyAny> = record.host.model_file.get().path_objects(py)?.1; + compiled_of(record)? + .analysis + .binding_tuple(py, &classes.compiled, (&name, &path)) + }) + } + + #[getter] + pub fn binding_validated(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.binding_validated)) + } + + #[getter] + pub fn dynamic_column_contract(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.proof_object(py, &classes.compiled) + }) + } + + #[getter] + pub fn unchecked_output_columns<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + PyFrozenSet::new(py, &self.overlay.poisoned) + } + + #[getter] + pub fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + /// This model with its final binding diagnostics. + pub(crate) fn with_binding_diagnostics( + &self, + py: Python<'_>, + diagnostics: Py, + ) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics) + }) + } + + /// This model after semantic type recovery: its bindings and the outputs left unchecked. + pub(crate) fn with_type_recovery( + &self, + py: Python<'_>, + diagnostics: Py, + poisoned: Vec, + ) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics); + overlay.poisoned = poisoned; + }) + } + + /// This model with the adapter's default schema and database filled into its destination. + pub fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let destination: DestinationFact = self.compiled(py, |compiled, _| { + let current: &DestinationFact = self + .overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination); + defaulted_destination(current, defaults, &render) + })?; + Ok(self.changed(py, |overlay| overlay.destination = Some(destination))) + } +} + +/// An attached function's user values. +pub struct FunctionHandles { + /// The discovered function file. + pub function_file: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// One attached function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeFunctionInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeFunctionInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledFunction { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination and fingerprint destination with target defaults applied. + pub(crate) defaulted: Option<(DestinationFact, DestinationFact)>, + pub(crate) destination: PyOnceLock>, + pub(crate) fingerprint_destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledFunction { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledFunction") + .finish_non_exhaustive() + } +} + +impl NativeCompiledFunction { + pub fn new(owner: Py, index: usize) -> Self { + Self::defaulted(owner, index, None) + } + + /// The function's project graph row: its key, the resources it reads and its tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + Ok(ResourceRow { + key: ( + function.attached.node_type().to_owned(), + function.attached.name.clone(), + ), + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) + }) + } + + pub(crate) fn defaulted( + owner: Py, + index: usize, + defaulted: Option<(DestinationFact, DestinationFact)>, + ) -> Self { + Self { + owner, + index, + defaulted, + destination: PyOnceLock::new(), + fingerprint_destination: PyOnceLock::new(), + } + } + + /// The destination and fingerprint destination, with target defaults once applied. + pub fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_function(&self.owner, self.index, py, |function, _| { + Ok(( + function.attached.destination.clone(), + function.attached.fingerprint_destination.clone(), + )) + }) + } +} + +impl NativeCompiledFunction { + /// The `CompiledFunction` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.relative_path(py)?.into_bound(py), + self.arguments(py)?.into_bound(py).into_any(), + self.returns(py)?.into_bound_py_any(py)?, + self.body_sql(py)?.into_bound_py_any(py)?, + self.destination(py)?.into_bound(py), + self.fingerprint_destination(py)?.into_bound(py), + self.return_columns(py)?.into_bound(py).into_any(), + self.references(py)?.into_bound(py).into_any(), + self.language(py)?, + self.source_file_path(py)?.into_bound(py), + self.runtime_version(py)?.into_bound_py_any(py)?, + self.entry_point(py)?.into_bound_py_any(py)?, + self.packages(py)?.into_any(), + self.tags(py)?.into_any(), + self.description(py)?.into_bound_py_any(py)?, + ]) + } +} + +/// One compile's attached inputs, its model inputs projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeInputsView { + pub(crate) owner: Py, + pub(crate) fields: InputsFields, +} + +impl NativeInputsView { + /// The view over `owner` with these input fields. + pub fn new(owner: Py, fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + owner, + fields: InputsFields::from_fields(fields)?, + }) + } +} + +#[pymethods] +impl NativeInputsView { + /// These inputs with `changes` applied; the owner's model inputs are shared. + #[pyo3(signature = (**changes))] + pub(crate) fn with_changes( + &self, + py: Python<'_>, + changes: Option>, + ) -> PyResult { + let current: Bound<'_, PyDict> = self.fields.as_fields(py)?; + if let Some(changes) = changes { + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown compile input field {name}" + ))); + } + current.set_item(name, value)?; + } + } + Self::new(self.owner.clone_ref(py), ¤t) + } + + /// Every field by name, for an explicit `CompileProjectInputs` projection. + pub(crate) fn fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + pub fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + pub fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter(no_sql_validation)] + pub fn is_sql_validation_skipped(&self) -> bool { + self.fields.no_sql_validation + } + + #[getter] + pub fn model_inputs(&self, py: Python<'_>) -> Py { + self.fields.model_inputs.clone_ref(py) + } + + #[getter] + pub fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + pub fn project_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.project_config.clone_ref(py) + } + + #[getter] + pub fn local_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.local_config.clone_ref(py) + } + + #[getter] + pub fn discovered_inputs(&self, py: Python<'_>) -> Py { + self.fields.boundary.discovered_inputs.clone_ref(py) + } + + #[getter] + pub fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + pub fn effective_target(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_target.clone_ref(py) + } + + #[getter] + pub fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + pub fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + pub fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + pub fn macro_context(&self, py: Python<'_>) -> Py { + self.fields.boundary.macro_context.clone_ref(py) + } + + #[getter] + pub fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + pub fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + pub fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + pub fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + pub fn effective_settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.effective_settings.clone_ref(py) + } + + #[getter] + pub fn seed_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.seed_inputs.clone_ref(py) + } + + #[getter] + pub fn source_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.source_inputs.clone_ref(py) + } + + #[getter] + pub fn sql_function_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_function_inputs.clone_ref(py) + } + + #[getter] + pub fn test_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.test_inputs.clone_ref(py) + } + + #[getter] + pub fn scenario_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario_inputs.clone_ref(py) + } + + #[getter] + pub fn audit_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.audit_inputs.clone_ref(py) + } + + #[getter] + pub fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + pub fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + #[getter] + pub fn declaration_scope(&self, py: Python<'_>) -> Py { + self.fields.compiler.declaration_scope.clone_ref(py) + } +} + +/// A model schema's header columns and specs, which carry user-authored audit arguments. +pub struct SchemaHandles { + pub columns: Arc, + pub audits: Arc, + pub dynamic_columns: Arc, +} + +/// A model input's user values and the attribute projections its views build. +pub struct ModelInputHandles { + /// The discovered model file, owned by the discovery inputs. + pub model_file: Py, + /// The resolved config, whose values hold user values. + pub config: Py, + pub rejected_opt_out: Option>, + /// The model's own enums and constants, whose values are user values. + pub enum_declarations: Py, + pub constant_declarations: Py, + pub enum_columns: Py, + /// Usage records, the identities the scope index is built from. + pub macro_usages: Py, + pub declaration_usages: Py, + /// Attribute projections built on first access, shared by every view of the model. + pub projections: InputProjections, +} + +/// A model input's Python attribute projections, each built once on first access. +pub struct InputProjections { + pub(crate) query_sql: PyOnceLock>, + pub(crate) references: PyOnceLock>, + pub(crate) schema_entry: PyOnceLock>, + pub(crate) sql_expansion: PyOnceLock>, + pub(crate) lint_expansion: PyOnceLock>, +} + +impl std::fmt::Debug for InputProjections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputProjections") + .finish_non_exhaustive() + } +} + +impl Default for InputProjections { + fn default() -> Self { + Self { + query_sql: PyOnceLock::new(), + references: PyOnceLock::new(), + schema_entry: PyOnceLock::new(), + sql_expansion: PyOnceLock::new(), + lint_expansion: PyOnceLock::new(), + } + } +} + +/// One model's compile input, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeModelInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeModelInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + pub fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } + + /// Read the input record together with the project that owns it. + pub fn read_with_owner<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, + ) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, read) + } +} + +#[pymethods] +impl NativeModelInput { + #[getter] + pub fn model_file(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.model_file.clone_ref(py).into_any()) + }) + } + + #[getter] + pub fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) + } + + #[getter] + pub fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) + } + + #[getter] + pub fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| { + Ok(PyString::new(py, &record.macro_source_sql)) + }) + } + + #[getter] + pub fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + pub fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + pub fn relation_key_row(&self, py: Python<'_>) -> PyResult { + self.read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }) + } + + /// The name of the schema entry attached to the model, or `None`. + #[getter] + pub fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| Ok(attached_schema_name(record))) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> Py { + py.None() + } + + #[getter] + pub fn sql_validation_enabled(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _| Ok(record.sql_validation_enabled)) + } + + #[getter] + pub fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + #[getter] + pub fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) + } + + #[getter] + pub fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| PyTuple::new(py, &record.macro_deps)) + } + + #[getter] + pub fn macro_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.macro_usages.clone_ref(py))) + } + + #[getter] + pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.declaration_usages.clone_ref(py)) + }) + } + + /// Record what assembly finished for the model; return its compiled model view. + pub(crate) fn compiled_model( + &self, + py: Python<'_>, + arguments: CompiledArguments, + ) -> PyResult { + let mut owner = self.owner.bind(py).borrow_mut(); + let record: &mut ModelInputRecord = owner.input_mut(self.index)?; + let name: String = record.name.clone(); + record.compiled = Some(compiled_record(py, record, &name, arguments)?); + Ok(NativeCompiledModel::new( + self.owner.clone_ref(py), + self.index, + )) + } + + /// The authored-to-expanded passes, when interpolation left the authored query unchanged. + #[getter] + pub fn sql_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| authored_expansion(&classes, record)) + } + + /// The whole expansion, variables included, which early SQL lint reuses. + #[getter] + pub fn lint_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + record + .host + .projections + .lint_expansion + .get_or_try_init(py, || { + let expansion: &ExpansionRecord = &record.expansion; + expansion_object( + &classes, + expansion, + &[ + &expansion.interpolation_spans, + &expansion.declaration_spans, + &expansion.macro_spans, + ], + ) + }) + .map(|expansion| expansion.clone_ref(py)) + }) + } +} + +/// A model input's attached schema: its name and dynamic column families. +#[derive(Debug)] +pub struct AttachedSchema { + pub name: String, + pub dynamic_columns: Vec, + pub type_enforcement: bool, +} + +/// One lineage output with its transform kind and confidence values, and its sources. +#[derive(Debug)] +pub struct LineageFactRow { + pub output_column: String, + pub transform: String, + pub confidence: String, + /// `(resource type, resource, column)` of each source. + pub sources: Vec<(String, String, String)>, +} + +/// The extension's host: Python handles and the views built from them. +#[derive(Debug)] +pub struct PyProjectHost; + +impl ProjectHost for PyProjectHost { + type ModelInput = ModelInputHandles; + type Schema = SchemaHandles; + type CompiledModel = CompiledProjections; + type Analysis = AnalysisProjections; + type AttachedSource = SourceHandles; + type Source = SourceProjections; + type AttachedFunction = FunctionHandles; + type Function = FunctionProjections; + type AttachedAudit = AuditHandles; + type Audit = AuditProjections; + type Seed = SeedHandles; +} + +/// One compiled project, its models projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeProjectView { + pub(crate) owner: Py, + pub(crate) fields: ProjectFields, + pub(crate) lineage_graph: PyOnceLock>, + pub(crate) sql_expansions: PyOnceLock>, +} + +impl std::fmt::Debug for NativeProjectView { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeProjectView") + .finish_non_exhaustive() + } +} + +impl NativeProjectView { + /// The view over `owner` with these project fields. + pub fn new(owner: Py, fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self::with_fields( + owner, + ProjectFields::from_fields(fields)?, + )) + } + + /// The project field custom-rule facts and rules runs read by `name`, or `None` for any other. + pub fn fact_field(&self, py: Python<'_>, name: &str) -> Option> { + let field: &Py = match name { + "settings" => &self.fields.compiler.settings, + "sql_scenarios" => &self.fields.compiler.sql_scenarios, + "scope_index" => &self.fields.compiler.scope_index, + "diagnostics" => &self.fields.compiler.diagnostics, + "sources" => &self.fields.compiler.sources, + "seeds" => &self.fields.compiler.seeds, + "functions" => &self.fields.compiler.functions, + "audits" => &self.fields.compiler.audits, + "sql_tests" => &self.fields.compiler.sql_tests, + "public_enums" => &self.fields.boundary.public_enums, + "public_constants" => &self.fields.boundary.public_constants, + "models" => return Some(self.fields.models.clone_ref(py).into_any()), + _ => return None, + }; + Some(field.clone_ref(py)) + } + + /// The compiled project this view projects. + pub fn owner(&self) -> &Py { + &self.owner + } + + pub(crate) fn with_fields(owner: Py, fields: ProjectFields) -> Self { + Self { + owner, + fields, + lineage_graph: PyOnceLock::new(), + sql_expansions: PyOnceLock::new(), + } + } +} + +#[pymethods] +impl NativeProjectView { + /// This project with `changes` applied; the owner's models are shared. + #[pyo3(signature = (**changes))] + pub(crate) fn with_changes( + &self, + py: Python<'_>, + changes: Option>, + ) -> PyResult { + let fields: ProjectFields = match changes { + Some(changes) => self.fields.changed(py, &changes)?, + None => self.fields.changed(py, &PyDict::new(py))?, + }; + Ok(Self::with_fields(self.owner.clone_ref(py), fields)) + } + + /// Every field by name, for an explicit `CompiledProject` projection. + #[pyo3(signature = (include_sql_expansions = true))] + pub fn fields<'py>( + &self, + py: Python<'py>, + include_sql_expansions: bool, + ) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item( + "sql_expansions", + if include_sql_expansions { + self.sql_expansions(py)? + } else { + PyDict::new(py).unbind() + }, + )?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + pub fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + pub fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter] + pub fn effective_target_database(&self) -> Option { + self.fields.effective_target_database.clone() + } + + #[getter] + pub fn effective_target_schema(&self) -> Option { + self.fields.effective_target_schema.clone() + } + + #[getter] + pub fn sql_analysis_dialect(&self) -> Option { + self.fields.sql_analysis_dialect.clone() + } + + #[getter(enforce_explicit_references)] + pub fn is_explicit_reference_enforced(&self) -> bool { + self.fields.enforce_explicit_references + } + + #[getter] + pub fn models(&self, py: Python<'_>) -> Py { + self.fields.models.clone_ref(py) + } + + #[getter] + pub fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + pub fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + pub fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + pub fn binding_catalog(&self, py: Python<'_>) -> Py { + self.fields.boundary.binding_catalog.clone_ref(py) + } + + #[getter] + pub fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + pub fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + pub fn loader_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.loader_functions.clone_ref(py) + } + + #[getter] + pub fn python_execution_inputs(&self) -> BTreeMap { + self.fields.boundary.python_execution_inputs.clone() + } + + #[getter] + pub fn hook_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.hook_functions.clone_ref(py) + } + + #[getter] + pub fn sql_hook_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_hook_files.clone_ref(py) + } + + #[getter] + pub fn materialization_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.materialization_files.clone_ref(py) + } + + #[getter] + pub fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + pub fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + pub fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + pub fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + pub fn native_session(&self, py: Python<'_>) -> Py { + self.fields.boundary.native_session.clone_ref(py) + } + + #[getter] + pub fn settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.settings.clone_ref(py) + } + + #[getter] + pub fn scenario(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario.clone_ref(py) + } + + #[getter] + pub fn sources(&self, py: Python<'_>) -> Py { + self.fields.compiler.sources.clone_ref(py) + } + + #[getter] + pub fn seeds(&self, py: Python<'_>) -> Py { + self.fields.compiler.seeds.clone_ref(py) + } + + #[getter] + pub fn functions(&self, py: Python<'_>) -> Py { + self.fields.compiler.functions.clone_ref(py) + } + + #[getter] + pub fn audits(&self, py: Python<'_>) -> Py { + self.fields.compiler.audits.clone_ref(py) + } + + #[getter] + pub fn sql_tests(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_tests.clone_ref(py) + } + + #[getter] + pub fn sql_scenarios(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_scenarios.clone_ref(py) + } + + #[getter] + pub fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + pub fn unmatched_literal_sql_relations(&self, py: Python<'_>) -> Py { + self.fields + .compiler + .unmatched_literal_sql_relations + .clone_ref(py) + } + + #[getter] + pub fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + /// Each model's authored-to-expanded passes by file path, where interpolation kept the query. + #[getter] + pub fn sql_expansions(&self, py: Python<'_>) -> PyResult> { + self.sql_expansions + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let owner = self.owner.bind(py).borrow(); + let expansions: Bound<'_, PyDict> = PyDict::new(py); + for record in owner.inputs() { + let expansion: Py = authored_expansion(&classes, record)?; + if !expansion.is_none(py) { + let path: Bound<'_, PyAny> = + record.host.model_file.get().path_objects(py)?.0; + expansions.set_item(path, expansion)?; + } + } + Ok::<_, pyo3::PyErr>(expansions.unbind()) + }) + .map(|expansions| expansions.clone_ref(py)) + } + + /// The lineage edges, tags, model folders and names, indexed natively once per project. + #[getter] + pub fn lineage_graph(&self, py: Python<'_>) -> PyResult> { + self.lineage_graph + .get_or_try_init(py, || { + let mut rows: Vec = Vec::new(); + for model in self.fields.models.bind(py).iter() { + let model: Bound<'_, NativeCompiledModel> = model.downcast_into()?; + rows.push(model_graph_row(model.get(), py)?); + } + let compiler: &CompilerHandles = &self.fields.compiler; + for source in compiler.sources.bind(py).try_iter()? { + let source: Bound<'_, PyAny> = source?; + rows.push(match source.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => resource_row(&source, None)?, + }); + } + for seed in compiler.seeds.bind(py).try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + rows.push(match seed.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = native_field( + &native_field(&seed, SCHEMA_ENTRY_ATTRIBUTE)?, + crate::compiled_views::_helpers::project_view::TAGS_ATTRIBUTE, + )?; + resource_row(&seed, Some(&tags))? + } + }); + } + for function in compiler.functions.bind(py).try_iter()? { + let function: Bound<'_, PyAny> = function?; + rows.push(match function.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = native_field( + &function, + crate::compiled_views::_helpers::project_view::TAGS_ATTRIBUTE, + )?; + resource_row(&function, Some(&tags))? + } + }); + } + self.owner.bind(py).borrow_mut().record_graph_build(); + count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; + Py::new(py, project_graph(rows)?) + }) + .map(|graph| graph.clone_ref(py)) + } +} + +/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. +#[derive(Debug)] +pub struct RelationKeyRow { + pub kind: String, + pub name: String, + pub materialized: String, + pub mode: String, + pub strategy: String, + pub has_cursor: bool, + pub unique_key: Option>, + pub columns: Vec<(String, Vec, bool)>, + pub audits: Vec<(String, Option)>, +} + +impl<'py> pyo3::FromPyObject<'py> for RelationKeyRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits): ( + String, + String, + String, + String, + String, + bool, + Option>, + Vec<(String, Vec, bool)>, + Vec<(String, Option)>, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for RelationKeyRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.kind, + self.name, + self.materialized, + self.mode, + self.strategy, + self.has_cursor, + self.unique_key, + self.columns, + self.audits, + ), + py, + ) + } +} + +/// `(database, schema, logical database, logical schema)` of a seed's destination. +#[derive(Debug)] +pub struct SeedNamespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for SeedNamespace { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (database, schema, logical_database, logical_schema): ( + Option, + Option, + Option, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + database, + schema, + logical_database, + logical_schema, + }) + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +#[derive(Debug)] +pub struct SeedParts<'a> { + pub name: &'a str, + pub seed_file: &'a Py, + pub schema_entry: &'a Py, + pub schema_file: &'a Py, +} + +/// One attached seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSeedInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSeedInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSeedInput { + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } +} + +/// One assembled seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledSeed { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination with target defaults applied, replacing the assembled one. + pub(crate) defaulted: Option, + pub(crate) destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledSeed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledSeed") + .finish_non_exhaustive() + } +} + +impl NativeCompiledSeed { + pub fn new(owner: Py, index: usize) -> Self { + Self { + owner, + index, + defaulted: None, + destination: PyOnceLock::new(), + } + } + + /// The seed's project graph row: its key and its declared tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, py| { + let mut tags: Vec = Vec::new(); + for tag in native_field( + seed.host.schema_entry.bind(py), + crate::compiled_views::_helpers::seed_views::TAGS_ATTRIBUTE, + )? + .try_iter()? + { + tags.push(tag?.str()?.extract()?); + } + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), + tags, + folder: None, + }) + }) + } + + pub fn current_destination(&self, py: Python<'_>) -> PyResult { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_seed(&self.owner, self.index, py, |seed, _| { + seed.destination + .clone() + .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) + }) + } +} + +#[pymethods] +impl NativeCompiledSeed { + /// The seed's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_seed(&self.owner, self.index, py, |seed, py| { + seed.host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SEED_RESOURCE, &seed.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) + } + + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } + + /// The seed's planner-facing `CompiledRelationLocation`. + #[getter] + pub fn destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destination(py)?) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// This seed with the adapter's default schema and database filled into its destination. + pub fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let defaulted: DestinationFact = + defaulted_destination(&self.current_destination(py)?, defaults, &render)?; + Ok(Self { + owner: self.owner.clone_ref(py), + index: self.index, + defaulted: Some(defaulted), + destination: PyOnceLock::new(), + }) + } + + #[getter(external)] + pub fn is_external(&self) -> bool { + false + } +} + +impl NativeCompiledSeed { + /// The `CompiledSeed` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.seed_file(py)?.into_bound(py), + self.schema_entry(py)?.into_bound(py), + self.schema_file(py)?.into_bound(py), + self.destination(py)?.into_bound(py), + pyo3::types::PyBool::new(py, self.is_external()) + .to_owned() + .into_any(), + ]) + } +} + +/// An attached source's user values. +pub struct SourceHandles { + /// The discovered source file. + pub source_file: Py, + /// The discovered entry; its spec carries user-authored metadata. + pub entry: Py, + /// Template-expanded user metadata. + pub meta: Py, + /// Template-expanded column specs, which carry user metadata and audit arguments. + pub columns: Py, + /// Template-expanded table audit specs, which carry user audit arguments. + pub audits: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// One attached source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSourceInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSourceInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSourceInput { + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.attached_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } + + #[getter] + pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.usages.clone_ref(py)) + }) + } + + /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. + #[getter] + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, py| { + let attached: &AttachedSource = &source.attached; + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) + }) + } +} + +/// One assembled source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledSource { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledSource { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + /// The source's project graph row: its key, with no dependencies, tags or folder. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) + }) + } +} + +#[pymethods] +impl NativeCompiledSource { + /// The source's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_source(&self.owner, self.index, py, |source, py| { + source + .host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(source.attached.name.clone()) + }) + } + + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.assembled_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } +} + +impl NativeCompiledSource { + /// The `CompiledSource` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.source_entry(py)?.into_bound(py), + self.source_file(py)?.into_bound(py), + ]) + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/types.rs b/crates/sqlbuild-python-project/src/compiled_views/types.rs new file mode 100644 index 0000000000..e65215546d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/types.rs @@ -0,0 +1,68 @@ +//! Public traits and type aliases of compiled views. + +use crate::compiled_views::_helpers::source_views::SourceKeyFacts; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::models::SeedNamespace; +use crate::compiled_views::models::SeedParts; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_project::models as project_models; + +/// What the audit loop attached for one singular audit block or attached generic audit. +pub type AttachedAudit = project_models::AttachedAudit; + +/// One attached audit held by the compiled project. +pub type AuditRecord = project_models::AuditRecord; + +/// `(name, type)` of a declared column. +pub type DeclaredRow = (String, Option); + +/// `(output column, [(resource name, column name)])` of one lineage output. +pub type LineageInput = (String, Vec<(String, String)>); + +/// What the function loop attached for one SQL or Python function. +pub type AttachedFunction = project_models::AttachedFunction; + +/// One attached function held by the compiled project. +pub type FunctionRecord = project_models::FunctionRecord; + +/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. +pub type SchemaRecord = project_models::SchemaRecord; + +/// What the model loop finished for one model beyond its facts. +pub type ModelInputRecord = project_models::ModelInputRecord; + +/// Declared keys by `kind:name` identity, in identity order. +pub type RelationKeys = Vec<(String, Vec>)>; + +/// One seed CSV paired with its declaration; the discovered objects are discovery's. +pub type SeedRecord = project_models::SeedRecord; + +/// What a seed record reads and keeps through its Python values. +pub trait SeedViews { + /// The seed's name, CSV file, declaration and declaring schema file. + fn parts(&self) -> SeedParts<'_>; + + /// Keep the destination native project assembly resolved for the seed. + fn set_namespace(&mut self, namespace: SeedNamespace); +} + +/// What the source loop attached for one source: its expanded metadata and usages. +pub type AttachedSource = project_models::AttachedSource; + +/// One attached source held by the compiled project. +pub type SourceRecord = project_models::SourceRecord; + +/// What a source record projects from its Python values, each built once. +pub trait SourceViews { + /// The source's declared column facts and audit key rows, read once from its expanded specs. + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; + + /// The attached `SourceEntry`: the discovered entry with its expanded metadata. + fn attached_entry(&self, py: Python<'_>) -> PyResult>; + + /// The assembled `SourceEntry`: the attached entry in its resolved namespace. + fn assembled_entry(&self, py: Python<'_>) -> PyResult>; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs index 361d2c5738..a6d408cadc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs @@ -18,12 +18,13 @@ use sqlbuild_analysis::contracts::models::{ use sqlbuild_core::constants::STAGED_PROMOTION_MODE; use sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::model_reads::enum_columns; -use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; -use crate::bindings::_helpers::contracts::model_contracts::{ - ConflictRow, DiagnosticRow, evaluate_project_model_contracts, project_promotion_conflicts, -}; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::model_reads::enum_columns::enum_columns; +use crate::compiled_views::main::records::compiled_project_facts::compiled_project_facts; +use crate::contracts::_helpers::model_contracts::ConflictRow; +use crate::contracts::_helpers::model_contracts::DiagnosticRow; +use crate::contracts::_helpers::model_contracts::evaluate_project_model_contracts; +use crate::contracts::_helpers::model_contracts::project_promotion_conflicts; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs new file mode 100644 index 0000000000..7d9a72637e --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod contract_diagnostics; +pub(crate) mod exception_types; +pub(crate) mod model_contracts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs index e3f1ba85ca..d334f5b780 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::contracts::models::{ use sqlbuild_analysis::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use sqlbuild_analysis::type_system::errors::TypeNormalizationError; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::compiled_project::models::NativeCompiledProject; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::type_system::main::normalization_error::normalization_error; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs new file mode 100644 index 0000000000..e215029be9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs @@ -0,0 +1,18 @@ +//! Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. + +use crate::contracts::_helpers::contract_diagnostics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. +#[inline] +pub fn evaluate_model_contracts<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + adapter_default: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + contract_diagnostics::evaluate_model_contracts(py, project, dialect, adapter_default) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/mod.rs b/crates/sqlbuild-python-project/src/contracts/main/mod.rs new file mode 100644 index 0000000000..37e04c6f41 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/mod.rs @@ -0,0 +1,4 @@ +pub mod evaluate_model_contracts; +pub mod promotion_conflict_diagnostics; +pub mod register_contract_diagnostics; +pub mod register_exception_types; diff --git a/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs new file mode 100644 index 0000000000..c233ef8ec7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs @@ -0,0 +1,24 @@ +//! One K011 error per enforced-contract table model under immediate promotion. + +use crate::contracts::_helpers::contract_diagnostics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// One K011 error per enforced-contract table model under immediate promotion. +#[inline] +pub fn promotion_conflict_diagnostics<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + adapter_default: &Bound<'py, PyAny>, + settings_file: String, +) -> PyResult> { + contract_diagnostics::promotion_conflict_diagnostics( + py, + project, + adapter_default, + settings_file, + ) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs new file mode 100644 index 0000000000..f237d80000 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs @@ -0,0 +1,12 @@ +//! Register the contract diagnostics functions and classes. + +use crate::contracts::_helpers::contract_diagnostics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the contract diagnostics functions and classes. +#[inline] +pub fn register_contract_diagnostics(module: &Bound<'_, PyModule>) -> PyResult<()> { + contract_diagnostics::register(module) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs new file mode 100644 index 0000000000..7640032d19 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs @@ -0,0 +1,12 @@ +//! Register the exception types functions and classes. + +use crate::contracts::_helpers::exception_types; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the exception types functions and classes. +#[inline] +pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + exception_types::register(module) +} diff --git a/crates/sqlbuild-python-project/src/contracts/mod.rs b/crates/sqlbuild-python-project/src/contracts/mod.rs new file mode 100644 index 0000000000..c1664aaf89 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/mod.rs @@ -0,0 +1,4 @@ +//! Model contracts, their diagnostics and exception types. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs new file mode 100644 index 0000000000..ee85866068 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod latest_rows; +pub(crate) mod query_hashes; +pub(crate) mod storage_io; +pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs index 6bf94189fa..482711d083 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs @@ -14,12 +14,10 @@ use sqlbuild_analysis::fingerprint_storage::models::RelevantRead; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::fingerprints::latest_rows::{ - fingerprint_input_error, fingerprint_set_from_rows, -}; -use crate::bindings::_helpers::fingerprints::storage_sql::{ - fingerprint_insert_batches, fingerprint_insert_sql, -}; +use crate::fingerprints::_helpers::latest_rows::fingerprint_input_error; +use crate::fingerprints::_helpers::latest_rows::fingerprint_set_from_rows; +use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_batches; +use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_sql; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs index d1fccc23c7..bbc9a407c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs @@ -7,7 +7,7 @@ use sqlbuild_analysis::fingerprint_storage::main::insert_batch_lengths::insert_b use sqlbuild_analysis::fingerprint_storage::main::insert_sql::insert_sql; use sqlbuild_analysis::fingerprint_storage::models::StoredFingerprint; -use crate::bindings::_helpers::fingerprints::latest_rows::fingerprint_input_error; +use crate::fingerprints::_helpers::latest_rows::fingerprint_input_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One `INSERT` of a single fingerprint. diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs new file mode 100644 index 0000000000..86111b2adb --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs @@ -0,0 +1,15 @@ +//! The persisted SHA-256 fingerprint of the query's canonical token stream. + +use crate::fingerprints::_helpers::query_hashes; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The persisted SHA-256 fingerprint of the query's canonical token stream. +#[inline] +pub fn compute_query_hash( + py: Python<'_>, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + query_hashes::compute_query_hash(py, query_sql, dialect) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs new file mode 100644 index 0000000000..ea51d13f10 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs @@ -0,0 +1,7 @@ +pub mod compute_query_hash; +pub mod read_latest_fingerprints; +pub mod register_latest_rows; +pub mod register_query_hashes; +pub mod register_storage_io; +pub mod row_fingerprint; +pub mod select_latest_fingerprints; diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs new file mode 100644 index 0000000000..b2ca7b0b3c --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs @@ -0,0 +1,15 @@ +//! The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. + +use crate::fingerprints::_helpers::storage_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. +#[inline] +pub fn read_latest_fingerprints<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + storage_io::read_latest_fingerprints(request) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs new file mode 100644 index 0000000000..11b572ab79 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs @@ -0,0 +1,12 @@ +//! Register the latest rows functions and classes. + +use crate::fingerprints::_helpers::latest_rows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the latest rows functions and classes. +#[inline] +pub fn register_latest_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { + latest_rows::register(module) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs new file mode 100644 index 0000000000..997ac33603 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs @@ -0,0 +1,12 @@ +//! Register the query hashes functions and classes. + +use crate::fingerprints::_helpers::query_hashes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the query hashes functions and classes. +#[inline] +pub fn register_query_hashes(module: &Bound<'_, PyModule>) -> PyResult<()> { + query_hashes::register(module) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs new file mode 100644 index 0000000000..863938c25b --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs @@ -0,0 +1,12 @@ +//! Register the storage io functions and classes. + +use crate::fingerprints::_helpers::storage_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the storage io functions and classes. +#[inline] +pub fn register_storage_io(module: &Bound<'_, PyModule>) -> PyResult<()> { + storage_io::register(module) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs new file mode 100644 index 0000000000..939bb6fe8c --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs @@ -0,0 +1,17 @@ +//! Row fingerprint. + +use crate::fingerprints::_helpers::latest_rows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Row fingerprint. +#[inline] +pub fn row_fingerprint<'py>( + fingerprint_class: &Bound<'py, PyAny>, + from_iso: &Bound<'py, PyAny>, + row: &Bound<'py, PyAny>, + qualified_name: &str, +) -> PyResult> { + latest_rows::row_fingerprint(fingerprint_class, from_iso, row, qualified_name) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs new file mode 100644 index 0000000000..4a60426de9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs @@ -0,0 +1,18 @@ +//! `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. + +use crate::fingerprints::_helpers::latest_rows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. +#[inline] +pub fn select_latest_fingerprints<'py>( + py: Python<'py>, + fingerprint_set: &Bound<'py, PyAny>, + node_names: Vec, + filtered_node_types: Vec, +) -> PyResult> { + latest_rows::select_latest_fingerprints(py, fingerprint_set, node_names, filtered_node_types) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/mod.rs new file mode 100644 index 0000000000..94f3b1fa2a --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/mod.rs @@ -0,0 +1,4 @@ +//! Fingerprint rows: query hashes and their storage. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs index 53b334f5bf..880f655b82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs @@ -8,7 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::graph_walk::main::directed_path_nodes::directed_path_nodes; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; +use crate::graph::models::PyNodeSet; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs b/crates/sqlbuild-python-project/src/graph/_helpers/getters.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/getters.rs index fde060ec0b..fe8f1996e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::graph::models::CompiledProjectGraph; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_compiled_project_graph, -}]; - -fn read_compiled_project_graph<'py>( +pub(crate) fn read_compiled_project_graph<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/keys.rs b/crates/sqlbuild-python-project/src/graph/_helpers/keys.rs new file mode 100644 index 0000000000..ce305b908c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/keys.rs @@ -0,0 +1,12 @@ +//! `CompiledObjectKey`s for native `(resource type, name)` pairs, converted once per pair. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The native `(resource type, name)` pair of a `CompiledObjectKey`. +pub(crate) fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(key, "resource_type")?.str()?.extract()?, + native_field(key, "name")?.extract()?, + )) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs new file mode 100644 index 0000000000..11fd00d1b8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod algorithms; +pub(crate) mod execution_gates; +pub(crate) mod getters; +pub(crate) mod keys; +pub(crate) mod project_graph; +pub(crate) mod python_lifecycle; +pub(crate) mod python_node_graph; +pub(crate) mod python_selection; +pub(crate) mod views; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs new file mode 100644 index 0000000000..d67a57bfa9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs @@ -0,0 +1,76 @@ +//! The compiled project graph and selector resolution, held natively for Python's graph users. + +use crate::graph::models::CompiledProjectGraph; +use crate::graph::models::NativeProjectGraph; +use crate::graph::models::ResourceRow; +use sqlbuild_python_base::positional_record; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; +use sqlbuild_analysis::graph::models::GraphResource; + +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; + +type Edges = Vec<(ObjectKey, Vec)>; +positional_record! { + extract, into; + /// `(names, upstream, downstream, tags, paths)` in Python's dict order. + pub(crate) struct IndexesRow { + pub(crate) names: Vec<(String, ObjectKey)>, + pub(crate) upstream: Edges, + pub(crate) downstream: Edges, + pub(crate) tags: Vec<(String, Vec)>, + pub(crate) paths: Vec<(ObjectKey, String)>, + } +} +/// `(unique id, neighbour unique ids)` in graph order. +pub(crate) type UniqueIdMap = Vec<(String, Vec)>; +/// `PlannerInputError`'s `(code, message, help)`. +type FailureRow = (&'static str, String, Option); +/// The keys, or the `PlannerInputError` Python raises instead. +pub(crate) type KeysOutcome = (Option>, Option); + +pub(crate) fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { + deps.iter().map(|dep| (dep.clone(), key.clone())).collect() +} + +pub(crate) fn outcome(result: Result, SelectorError>) -> KeysOutcome { + match result { + Ok(keys) => (Some(keys), None), + Err(error) => (None, Some(failure(error))), + } +} + +fn failure(error: SelectorError) -> FailureRow { + (error.code, error.message, error.help) +} + +/// The project graph indexes of these resources, in project order. +pub(crate) fn project_graph(resources: Vec) -> PyResult { + compiler_guard(|| { + let resources: Vec = resources + .into_iter() + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) + .collect(); + Ok(NativeProjectGraph::new(build_project_graph(&resources))) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs index cfebc0428e..9906d242ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs @@ -8,8 +8,8 @@ use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNod use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; -use crate::bindings::_helpers::graph::views::compiled_graph_keys; +use crate::graph::_helpers::python_selection::resolve_python_sql_run_selection; +use crate::graph::_helpers::views::compiled_graph_keys; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs index d578631e41..c9e90a972f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs @@ -2,7 +2,6 @@ use sqlbuild_python_base::positional_record; use std::borrow::Cow; -use std::collections::BTreeSet; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -14,19 +13,17 @@ use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; -use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; -use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, SqlSelectionGraph, }; -use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +use crate::compiled_project::main::job_catalog::needed_jobs::needed_jobs; +use crate::graph::models::CompiledProjectGraph; +use crate::graph::models::NativeProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -104,7 +101,7 @@ fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { } /// A `PythonNodeGraph` as the native selection graph. -fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { Ok(PythonSelectionGraph { nodes: native_field(graph, "nodes")? .try_iter()? @@ -122,7 +119,9 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { } /// `(name, loader)` of every source in the compiled project. -fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>> { +pub(crate) fn project_sources( + project: &Bound<'_, PyAny>, +) -> PyResult)>> { native_field(project, "sources")? .try_iter()? .map(|source| { @@ -137,101 +136,6 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, - pub(crate) nodes: PythonSelectionGraph, -} - -impl ProjectSelection { - pub(crate) fn new( - project: &Bound<'_, PyAny>, - discovered_inputs: &Bound<'_, PyAny>, - ) -> PyResult { - Ok(Self { - sources: project_sources(project)?, - nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, - }) - } - - /// The `kind:name` keys `sqb build --select` runs for these selectors. - pub(crate) fn run_keys( - &self, - graph: &ProjectGraph, - select: &[String], - exclude: &[String], - ) -> Result, String> { - self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) - } - - /// The `kind:name` keys these selectors name, Python checks included. - pub(crate) fn node_keys( - &self, - graph: &ProjectGraph, - select: &[String], - exclude: &[String], - ) -> Result, String> { - self.keys(resolve_python_sql_nodes( - self.sql(graph), - &self.nodes, - select, - exclude, - )) - } - - /// The published key of a Python node, such as `task:export_orders`. - pub(crate) fn node_key(&self, name: &str) -> Option { - self.nodes - .nodes - .iter() - .rev() - .find(|node| node.name == name) - .map(|node| format!("{}:{name}", node.kind)) - } - - /// The key a selected Python node's run reports under; a loader reports as the source it writes. - fn run_key(&self, name: &str) -> Option { - let node = self - .nodes - .nodes - .iter() - .rev() - .find(|node| node.name == name)?; - Some(python_run_key(&self.sources, (&node.kind, name))) - } - - /// Whether a loader writes no declared source and so runs only as part of another loader's chain. - pub(crate) fn intermediate_loader(&self, name: &str) -> bool { - !self - .sources - .iter() - .any(|(_, loader)| loader.as_deref() == Some(name)) - } - - fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { - SqlSelectionGraph { - graph, - sources: &self.sources, - } - } - - fn keys( - &self, - selection: Result, - ) -> Result, String> { - let selection: RunSelection = selection.map_err(|error| error.message)?; - let mut keys: BTreeSet = selection - .sql_keys - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); - for name in &selection.python_node_names { - keys.extend(self.run_key(name)); - } - Ok(keys) - } -} - /// The Python node names `select` minus `exclude` resolve to; every node when unselected. pub(crate) fn python_node_names<'py>( graph: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python-project/src/graph/_helpers/views.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/views.rs index d76f28574d..441551c7d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/views.rs @@ -1,5 +1,6 @@ //! The native project graph's indexes as the Python dicts graph consumers read, and back. +use crate::graph::models::GraphViewMappings; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; @@ -9,8 +10,9 @@ use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::{GraphIndexes, ProjectGraph}; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::graph::_helpers::keys::native_key; +use crate::graph::models::KeyCache; +use crate::graph::models::NativeProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -141,15 +143,6 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult { - pub(crate) all_keys: &'a Bound<'py, PyAny>, - pub(crate) upstream: &'a Bound<'py, PyAny>, - pub(crate) downstream: &'a Bound<'py, PyAny>, - pub(crate) tag_index: &'a Bound<'py, PyAny>, - pub(crate) path_index: &'a Bound<'py, PyAny>, -} - /// A native graph over Python's index mappings, keeping their order. pub(crate) fn graph_from_view_mappings( mappings: GraphViewMappings<'_, '_>, diff --git a/crates/sqlbuild-python-project/src/graph/constants.rs b/crates/sqlbuild-python-project/src/graph/constants.rs new file mode 100644 index 0000000000..3d1c7c916d --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of graph. + +use crate::graph::_helpers::getters::read_compiled_project_graph; +use crate::graph::models::CompiledProjectGraph; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_compiled_project_graph, +}]; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs new file mode 100644 index 0000000000..659cac1686 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs @@ -0,0 +1,17 @@ +//! Keys within `max_depth` edges of any start, plus the starts when `include_starts`. + +use crate::graph::_helpers::algorithms; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PySet; + +/// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. +#[inline] +pub fn closure<'py>( + starts: Vec>, + edges: &Bound<'py, PyAny>, + (include_starts, max_depth): (bool, Option), +) -> PyResult> { + algorithms::closure(starts, edges, (include_starts, max_depth)) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs new file mode 100644 index 0000000000..7dceb79c46 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs @@ -0,0 +1,13 @@ +//! Reversed edges: every source key first, then each new target in first-seen order. + +use crate::graph::_helpers::algorithms; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Reversed edges: every source key first, then each new target in first-seen order. +#[inline] +pub fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { + algorithms::inverted(edges) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs new file mode 100644 index 0000000000..af1ff0b160 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs @@ -0,0 +1,4 @@ +pub mod closure; +pub mod inverted; +pub mod register_algorithms; +pub mod transitive_closure_many; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs new file mode 100644 index 0000000000..c8f4b55a2a --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs @@ -0,0 +1,12 @@ +//! Register the algorithms functions and classes. + +use crate::graph::_helpers::algorithms; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the algorithms functions and classes. +#[inline] +pub fn register_algorithms(module: &Bound<'_, PyModule>) -> PyResult<()> { + algorithms::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs new file mode 100644 index 0000000000..a122938835 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs @@ -0,0 +1,18 @@ +//! Keys within `max_depth` edges of any start, plus the starts when requested. + +use crate::graph::_helpers::algorithms; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// Keys within `max_depth` edges of any start, plus the starts when requested. +#[inline] +pub fn transitive_closure_many<'py>( + starts: &Bound<'py, PyAny>, + edges: &Bound<'py, PyAny>, + include_starts: bool, + max_depth: Option, +) -> PyResult> { + algorithms::transitive_closure_many(starts, edges, include_starts, max_depth) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs b/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs new file mode 100644 index 0000000000..d3bd9b5e81 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs @@ -0,0 +1,3 @@ +pub mod model_python_hook_names; +pub mod register_execution_gates; +pub mod sql_ref_key; diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs b/crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs new file mode 100644 index 0000000000..d58fedf433 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs @@ -0,0 +1,17 @@ +//! The Python hooks attached to one model, in lifecycle order without repeats. + +use crate::graph::_helpers::execution_gates; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyTuple; + +/// The Python hooks attached to one model, in lifecycle order without repeats. +#[inline] +pub fn model_python_hook_names<'py>( + py: Python<'py>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + execution_gates::model_python_hook_names(py, model) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs b/crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs new file mode 100644 index 0000000000..e4e4d60ad3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs @@ -0,0 +1,12 @@ +//! Register the execution gates functions and classes. + +use crate::graph::_helpers::execution_gates; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the execution gates functions and classes. +#[inline] +pub fn register_execution_gates(module: &Bound<'_, PyModule>) -> PyResult<()> { + execution_gates::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs b/crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs new file mode 100644 index 0000000000..d8fcec2bfb --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs @@ -0,0 +1,16 @@ +//! The compiled graph key a typed model, source or seed reference names. + +use crate::graph::_helpers::execution_gates; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The compiled graph key a typed model, source or seed reference names. +#[inline] +pub fn sql_ref_key<'py>( + py: Python<'py>, + reference: &Bound<'py, PyAny>, +) -> PyResult> { + execution_gates::sql_ref_key(py, reference) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs b/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs new file mode 100644 index 0000000000..b92019db42 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs @@ -0,0 +1 @@ +pub mod native_key; diff --git a/crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs b/crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs new file mode 100644 index 0000000000..32bd82849b --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs @@ -0,0 +1,12 @@ +//! The native `(resource type, name)` pair of a `CompiledObjectKey`. + +use crate::graph::_helpers::keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The native `(resource type, name)` pair of a `CompiledObjectKey`. +#[inline] +pub fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + keys::native_key(key) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/mod.rs b/crates/sqlbuild-python-project/src/graph/main/mod.rs new file mode 100644 index 0000000000..139c906297 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/mod.rs @@ -0,0 +1,8 @@ +pub mod algorithms; +pub mod execution_gates; +pub mod keys; +pub mod project_graph; +pub mod python_lifecycle; +pub mod python_node_graph; +pub mod python_selection; +pub mod views; diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs b/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs new file mode 100644 index 0000000000..523a463166 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs @@ -0,0 +1,2 @@ +pub mod project_graph; +pub mod register_project_graph; diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs new file mode 100644 index 0000000000..c9c83f9d74 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs @@ -0,0 +1,12 @@ +//! The project graph indexes of these resources, in project order. + +use crate::graph::_helpers::project_graph; +use crate::graph::models::NativeProjectGraph; +use crate::graph::models::ResourceRow; +use pyo3::prelude::PyResult; + +/// The project graph indexes of these resources, in project order. +#[inline] +pub fn project_graph(resources: Vec) -> PyResult { + project_graph::project_graph(resources) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs new file mode 100644 index 0000000000..bb053d40c1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs @@ -0,0 +1,12 @@ +//! Register the project graph functions and classes. + +use crate::graph::_helpers::project_graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the project graph functions and classes. +#[inline] +pub fn register_project_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + project_graph::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs new file mode 100644 index 0000000000..a4d3f8863b --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs @@ -0,0 +1,15 @@ +//! `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. + +use crate::graph::_helpers::python_lifecycle; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. +#[inline] +pub fn build_python_sql_run_lifecycle<'py>( + selection: &Bound<'py, PyAny>, + python_graph: &Bound<'py, PyAny>, +) -> PyResult> { + python_lifecycle::build_python_sql_run_lifecycle(selection, python_graph) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs new file mode 100644 index 0000000000..83049aebc7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs @@ -0,0 +1,2 @@ +pub mod build_python_sql_run_lifecycle; +pub mod register_python_lifecycle; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs new file mode 100644 index 0000000000..ece0f909a4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs @@ -0,0 +1,12 @@ +//! Register the python lifecycle functions and classes. + +use crate::graph::_helpers::python_lifecycle; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python lifecycle functions and classes. +#[inline] +pub fn register_python_lifecycle(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_lifecycle::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs new file mode 100644 index 0000000000..34b988f757 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs @@ -0,0 +1,3 @@ +pub mod python_node_graph; +pub mod register_python_node_graph; +pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs new file mode 100644 index 0000000000..2d9e69b330 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs @@ -0,0 +1,14 @@ +//! The internal executable Python-node graph of discovered project inputs. + +use crate::graph::_helpers::python_node_graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The internal executable Python-node graph of discovered project inputs. +#[inline] +pub fn python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + python_node_graph::python_node_graph(discovered_inputs) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs new file mode 100644 index 0000000000..fe05b78fa1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs @@ -0,0 +1,12 @@ +//! Register the python node graph functions and classes. + +use crate::graph::_helpers::python_node_graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python node graph functions and classes. +#[inline] +pub fn register_python_node_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_node_graph::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs new file mode 100644 index 0000000000..7a3218c23c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs @@ -0,0 +1,20 @@ +//! `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. + +use crate::graph::_helpers::python_node_graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. +#[inline] +pub fn run_selection_from_inputs<'py>( + (select, exclude): (Vec, Vec), + project_graph: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + python_node_graph::run_selection_from_inputs( + (select, exclude), + project_graph, + discovered_inputs, + ) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs new file mode 100644 index 0000000000..e98ca2796b --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs @@ -0,0 +1,3 @@ +pub mod python_node_names; +pub mod register_python_selection; +pub mod resolve_python_check_excludes; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs b/crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs new file mode 100644 index 0000000000..60c8fe0ea4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs @@ -0,0 +1,16 @@ +//! The Python node names `select` minus `exclude` resolve to; every node when unselected. + +use crate::graph::_helpers::python_selection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// The Python node names `select` minus `exclude` resolve to; every node when unselected. +#[inline] +pub fn python_node_names<'py>( + graph: &Bound<'py, PyAny>, + (select, exclude): (Vec, Vec), +) -> PyResult> { + python_selection::python_node_names(graph, (select, exclude)) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs b/crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs new file mode 100644 index 0000000000..fcdcebefff --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs @@ -0,0 +1,12 @@ +//! Register the python selection functions and classes. + +use crate::graph::_helpers::python_selection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the python selection functions and classes. +#[inline] +pub fn register_python_selection(module: &Bound<'_, PyModule>) -> PyResult<()> { + python_selection::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs b/crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs new file mode 100644 index 0000000000..7ef0c30a2e --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs @@ -0,0 +1,16 @@ +//! The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. + +use crate::graph::_helpers::python_selection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. +#[inline] +pub fn resolve_python_check_excludes<'py>( + graph: &Bound<'py, PyAny>, + exclude: Vec, +) -> PyResult> { + python_selection::resolve_python_check_excludes(graph, exclude) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs b/crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs new file mode 100644 index 0000000000..83322b6b1c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs @@ -0,0 +1,17 @@ +//! `CompiledObjectKey`s for native graph key pairs. + +use crate::graph::_helpers::views; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// `CompiledObjectKey`s for native graph key pairs. +#[inline] +pub fn compiled_graph_keys<'py>( + py: Python<'py>, + keys: Vec, +) -> PyResult> { + views::compiled_graph_keys(py, keys) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs b/crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs new file mode 100644 index 0000000000..c63899f901 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs @@ -0,0 +1,17 @@ +//! Edge dict. + +use crate::graph::_helpers::views; +use crate::graph::models::KeyCache; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Edge dict. +#[inline] +pub fn edge_dict<'py>( + cache: &KeyCache<'py>, + edges: &[(ObjectKey, Vec)], +) -> PyResult> { + views::edge_dict(cache, edges) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs b/crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs new file mode 100644 index 0000000000..b835290885 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs @@ -0,0 +1,14 @@ +//! A native graph over Python's index mappings, keeping their order. + +use crate::graph::_helpers::views; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::PyResult; + +/// A native graph over Python's index mappings, keeping their order. +#[inline] +pub fn graph_from_view_mappings( + mappings: GraphViewMappings<'_, '_>, +) -> PyResult { + views::graph_from_view_mappings(mappings) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/mod.rs b/crates/sqlbuild-python-project/src/graph/main/views/mod.rs new file mode 100644 index 0000000000..22ae97ac95 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/mod.rs @@ -0,0 +1,6 @@ +pub mod compiled_graph_keys; +pub mod edge_dict; +pub mod graph_from_view_mappings; +pub mod native_graph_from_edges; +pub mod project_lineage_views; +pub mod register_views; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs b/crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs new file mode 100644 index 0000000000..ae31746421 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs @@ -0,0 +1,16 @@ +//! A native graph holding these edges in their order, with no selector indexes. + +use crate::graph::_helpers::views; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A native graph holding these edges in their order, with no selector indexes. +#[inline] +pub fn native_graph_from_edges( + upstream: &Bound<'_, PyAny>, + downstream: &Bound<'_, PyAny>, +) -> PyResult { + views::native_graph_from_edges(upstream, downstream) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs b/crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs new file mode 100644 index 0000000000..628079c7c3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs @@ -0,0 +1,16 @@ +//! The project's lineage edges and selector indexes, from its native graph. + +use crate::graph::_helpers::views; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The project's lineage edges and selector indexes, from its native graph. +#[inline] +pub fn project_lineage_views<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + views::project_lineage_views(py, project) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/views/register_views.rs b/crates/sqlbuild-python-project/src/graph/main/views/register_views.rs new file mode 100644 index 0000000000..fabae49000 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/views/register_views.rs @@ -0,0 +1,12 @@ +//! Register the views functions and classes. + +use crate::graph::_helpers::views; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the views functions and classes. +#[inline] +pub fn register_views(module: &Bound<'_, PyModule>) -> PyResult<()> { + views::register(module) +} diff --git a/crates/sqlbuild-python-project/src/graph/mod.rs b/crates/sqlbuild-python-project/src/graph/mod.rs new file mode 100644 index 0000000000..10b27983f7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/mod.rs @@ -0,0 +1,6 @@ +//! The project graph: keys, node sets, algorithms, execution gates and Python nodes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs new file mode 100644 index 0000000000..e23602141e --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -0,0 +1,496 @@ +//! Public records of graph. + +use crate::graph::_helpers::project_graph::IndexesRow; +use crate::graph::_helpers::project_graph::KeysOutcome; +use crate::graph::_helpers::project_graph::UniqueIdMap; +use crate::graph::_helpers::project_graph::dependency_edges; +use crate::graph::_helpers::project_graph::outcome; +use crate::graph::_helpers::project_graph::project_graph; +use crate::graph::_helpers::python_node_graph::python_node_graph; +use crate::graph::_helpers::python_selection::project_sources; +use crate::graph::_helpers::python_selection::selection_graph; +use crate::graph::_helpers::views::lineage_graph_views; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PySet; +use pyo3::types::PySetMethods; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::build_resources::build_resources; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; +use sqlbuild_analysis::graph::models::BuildResources; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_analysis::graph::models::GraphIndexes; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_analysis::python_selection::models::RunSelection; +use sqlbuild_analysis::python_selection::models::SqlSelectionGraph; +use sqlbuild_core::graph_walk::types::NodeSet; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use std::cell::RefCell; +use std::collections::BTreeSet; +use std::collections::HashMap; + +/// One `CompiledObjectKey` per native pair, so shared keys convert once. +#[derive(Debug)] +pub struct KeyCache<'py> { + pub(crate) key_type: Bound<'py, PyAny>, + pub(crate) resource_type: Bound<'py, PyAny>, + pub(crate) keys: RefCell>>, +} + +impl<'py> KeyCache<'py> { + pub fn new(py: Python<'py>) -> PyResult { + Ok(Self { + key_type: record_class(py, RecordId::CompiledObjectKey)?, + resource_type: record_class(py, RecordId::CompiledResourceType)?, + keys: RefCell::new(HashMap::new()), + }) + } + + pub fn py(&self) -> Python<'py> { + self.key_type.py() + } + + /// The `CompiledObjectKey` for one `(resource type, name)` pair. + pub fn key(&self, (resource_type, name): &(String, String)) -> PyResult> { + let pair = (resource_type.clone(), name.clone()); + if let Some(existing) = self.keys.borrow().get(&pair) { + return Ok(existing.bind(self.py()).clone()); + } + let created: Bound<'py, PyAny> = self + .key_type + .call1((self.resource_type.call1((resource_type,))?, name))?; + self.keys + .borrow_mut() + .insert(pair, created.clone().unbind()); + Ok(created) + } + + /// `CompiledObjectKey`s for `pairs`, in order. + pub fn keys(&self, pairs: &[(String, String)]) -> PyResult>> { + pairs.iter().map(|pair| self.key(pair)).collect() + } +} + +/// Python keys a walk visited, compared with Python hashing and equality. +#[derive(Debug)] +pub struct PyNodeSet<'py>(pub Bound<'py, PySet>); + +impl<'py> NodeSet> for PyNodeSet<'py> { + type Error = pyo3::PyErr; + + fn insert_new(&mut self, key: &Bound<'py, PyAny>) -> Result { + if self.0.contains(key)? { + return Ok(false); + } + self.0.add(key)?; + Ok(true) + } + + fn holds(&self, key: &Bound<'py, PyAny>) -> Result { + self.0.contains(key) + } +} + +/// `(key, deps, tags, model folder)`. +#[derive(Debug)] +pub struct ResourceRow { + pub key: ObjectKey, + pub deps: Vec, + pub tags: Vec, + pub folder: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for ResourceRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (key, deps, tags, folder): (ObjectKey, Vec, Vec, Option) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + key, + deps, + tags, + folder, + }) + } +} + +/// A project graph: lineage edges, tag, path and name indexes, and selector resolution. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeProjectGraph { + pub(crate) graph: ProjectGraph, +} + +/// Static compiled project graph held natively; dict views are built only when read. +#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] +pub struct CompiledProjectGraph { + pub(crate) project: Py, + pub(crate) native: Py, + pub(crate) views: PyOnceLock>, +} + +impl std::fmt::Debug for CompiledProjectGraph { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CompiledProjectGraph") + .finish_non_exhaustive() + } +} + +impl NativeProjectGraph { + /// The graph this object wraps. + pub fn graph(&self) -> &ProjectGraph { + &self.graph + } + + /// Wrap a graph another native stage built, such as project assembly. + pub fn new(graph: ProjectGraph) -> Self { + Self { graph } + } +} + +impl CompiledProjectGraph { + /// The graph of `project` over its native lineage graph. + pub fn new(project: Py, native: Py) -> Self { + Self { + project, + native, + views: PyOnceLock::new(), + } + } + + /// The compiled project this graph indexes. + pub fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.project.bind(py) + } + + /// The native lineage graph. + pub fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { + self.native.bind(py) + } + + pub(crate) fn view(&self, py: Python<'_>, name: &str) -> PyResult> { + Ok(native_field(self.lineage_views(py)?.bind(py), name)?.unbind()) + } +} + +#[pymethods] +impl CompiledProjectGraph { + #[new] + #[pyo3(signature = (*, project, native))] + pub(crate) fn py_new(project: Py, native: Py) -> Self { + Self::new(project, native) + } + + #[getter(project)] + pub fn project_attribute(&self, py: Python<'_>) -> Py { + self.project.clone_ref(py) + } + + #[getter(native)] + pub fn native_attribute(&self, py: Python<'_>) -> Py { + self.native.clone_ref(py) + } + + /// Lineage edges and selector indexes, converted once. + #[getter] + pub fn lineage_views(&self, py: Python<'_>) -> PyResult> { + self.views + .get_or_try_init(py, || { + Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) + }) + .map(|views| views.clone_ref(py)) + } + + #[getter] + pub fn upstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "upstream_deps") + } + + #[getter] + pub fn downstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "downstream_deps") + } + + #[getter] + pub fn tag_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "tag_index") + } + + #[getter] + pub fn path_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "path_index") + } + + #[getter] + pub fn all_keys(&self, py: Python<'_>) -> PyResult> { + self.view(py, "all_keys") + } +} + +#[pymethods] +impl NativeProjectGraph { + /// Build the indexes from resources in project order: models, sources, seeds, functions. + #[staticmethod] + pub(crate) fn from_resources(resources: Vec) -> PyResult { + project_graph(resources) + } + + /// Wrap indexes a caller already built, such as the planner's. + #[staticmethod] + pub(crate) fn from_indexes(indexes: IndexesRow) -> PyResult { + compiler_guard(|| { + let IndexesRow { + names, + upstream, + downstream, + tags, + paths, + } = indexes; + Ok(Self::new(graph_from_indexes(GraphIndexes { + names, + upstream, + downstream, + tags, + paths, + }))) + }) + } + + /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. + pub(crate) fn indexes(&self) -> IndexesRow { + let graph: &ProjectGraph = &self.graph; + IndexesRow { + names: graph.names.clone(), + upstream: graph.upstream.clone(), + downstream: graph.downstream.clone(), + tags: graph.tags.clone(), + paths: graph.paths.clone(), + } + } + + /// The key's direct upstream lineage keys, in authored dependency order. + pub fn upstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() + } + + /// The key's direct downstream keys, sorted by `(resource type, name)`. + pub(crate) fn downstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Downstream).to_vec() + } + + /// Every selectable key, in project order with later same-name resources replacing earlier. + pub(crate) fn keys(&self) -> Vec { + self.graph + .names + .iter() + .map(|(_, key)| key.clone()) + .collect() + } + + /// `(selector name, key)` in project order. + pub(crate) fn names(&self) -> Vec<(String, ObjectKey)> { + self.graph.names.clone() + } + + /// Every lineage edge as `(dependency, dependent)`, in upstream order. + pub(crate) fn edges(&self) -> Vec<(ObjectKey, ObjectKey)> { + self.graph + .upstream + .iter() + .flat_map(|(key, deps)| dependency_edges(key, deps)) + .collect() + } + + /// The manifest's `(parent_map, child_map)` entries as dbt-style unique ids, in graph order. + pub(crate) fn unique_id_maps( + &self, + project_name: &str, + prefixes: HashMap, + ) -> (UniqueIdMap, UniqueIdMap) { + ( + unique_id_edges( + &self.graph, + EdgeDirection::Upstream, + project_name, + &prefixes, + ), + unique_id_edges( + &self.graph, + EdgeDirection::Downstream, + project_name, + &prefixes, + ), + ) + } + + /// The compile report's execution layer count over models. + pub(crate) fn model_layer_count(&self) -> usize { + model_layer_count(&self.graph) + } + + /// `--select` minus `--exclude`, plus the functions they need to build. + pub(crate) fn resolve( + &self, + select: Vec, + exclude: Vec, + ) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selectors(&self.graph, &select, &exclude)))) + } + + /// The keys selector tokens match, without build resources. + pub(crate) fn tokens(&self, selectors: Vec) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selector_tokens(&self.graph, &selectors)))) + } + + /// The keys one `kind:value` selector names before `+` expansion. + pub(crate) fn matched(&self, kind: &str, value: &str) -> PyResult { + compiler_guard(|| Ok(outcome(match_selector(&self.graph, kind, value)))) + } + + /// The selected keys plus the functions and seeds Python adds to build them. + pub(crate) fn build_resources( + &self, + selected: Vec, + include: (bool, bool, bool), + ) -> PyResult> { + compiler_guard(|| { + let (upstream_functions, upstream_seeds, downstream_functions) = include; + Ok(build_resources( + &self.graph, + &selected, + BuildResources { + upstream_functions, + upstream_seeds, + downstream_functions, + }, + )) + }) + } +} + +/// A compiled project's sources and discovered Python nodes, which publication resolves selectors over. +#[derive(Debug)] +pub struct ProjectSelection { + pub(crate) sources: Vec<(String, Option)>, + pub nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub fn new(project: &Bound<'_, PyAny>, discovered_inputs: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + pub(crate) fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + pub(crate) fn sql<'graph>( + &'graph self, + graph: &'graph ProjectGraph, + ) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + pub(crate) fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.run_key(name)); + } + Ok(keys) + } +} + +/// Python's selector name, edge, tag and path mappings. +#[derive(Debug)] +pub struct GraphViewMappings<'a, 'py> { + pub all_keys: &'a Bound<'py, PyAny>, + pub upstream: &'a Bound<'py, PyAny>, + pub downstream: &'a Bound<'py, PyAny>, + pub tag_index: &'a Bound<'py, PyAny>, + pub path_index: &'a Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python-project/src/lib.rs b/crates/sqlbuild-python-project/src/lib.rs new file mode 100644 index 0000000000..79a2063373 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lib.rs @@ -0,0 +1,25 @@ +//! The project layer of the `sqlbuild._native` bindings: the compiled project and its views, graph, references, lineage, fingerprints, semantic shapes, planner scope, selection and strategy, and SQL test planning. + +#![forbid(unsafe_code)] + +pub mod analysis; +pub mod analysis_session; +pub mod auditing; +pub mod compile_attachment; +pub mod compiled_project; +pub mod compiled_views; +pub mod contracts; +pub mod fingerprints; +pub mod graph; +pub mod lineage; +pub mod native_host; +pub mod planner_identity; +pub mod planner_phases; +pub mod planner_scope; +pub mod planner_selection; +pub mod planner_strategy; +pub mod references; +pub mod relation_naming; +pub mod semantic_shapes; +pub mod sql_test_glue; +pub mod type_system; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs b/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs rename to crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs diff --git a/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs b/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs new file mode 100644 index 0000000000..a662b6c8bd --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod lineage_facts; diff --git a/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs new file mode 100644 index 0000000000..b9bd54bb61 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs @@ -0,0 +1,13 @@ +//! The `ColumnLineageConfidence` member of a native confidence code. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The `ColumnLineageConfidence` member of a native confidence code. +#[inline] +pub fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { + lineage_facts::confidence_member(py, code) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs new file mode 100644 index 0000000000..e71aa93d42 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs @@ -0,0 +1,19 @@ +//! `facts == other`: item-wise equality with any sequence. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `facts == other`: item-wise equality with any sequence. +#[inline] +pub fn facts_eq( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + lineage_facts::facts_eq(instance, args, _kwargs) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs new file mode 100644 index 0000000000..43e211872f --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs @@ -0,0 +1,19 @@ +//! `facts[index]`: one fact, or a tuple of facts for a slice. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `facts[index]`: one fact, or a tuple of facts for a slice. +#[inline] +pub fn facts_getitem( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + lineage_facts::facts_getitem(instance, args, _kwargs) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs new file mode 100644 index 0000000000..a26ff38bf9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs @@ -0,0 +1,19 @@ +//! `iter(facts)`. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `iter(facts)`. +#[inline] +pub fn facts_iter( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + lineage_facts::facts_iter(instance, _args, _kwargs) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs new file mode 100644 index 0000000000..f099913346 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs @@ -0,0 +1,19 @@ +//! `len(facts)`. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `len(facts)`. +#[inline] +pub fn facts_len( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + lineage_facts::facts_len(instance, _args, _kwargs) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs new file mode 100644 index 0000000000..089bad1fd7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs @@ -0,0 +1,13 @@ +//! `CompactLineageFacts.facts`: every row projected once, in order. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `CompactLineageFacts.facts`: every row projected once, in order. +#[inline] +pub fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { + lineage_facts::facts_projected(instance) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs new file mode 100644 index 0000000000..59dc28803e --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs @@ -0,0 +1,19 @@ +//! `CompactLineageFacts.resource_name(index)`. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `CompactLineageFacts.resource_name(index)`. +#[inline] +pub fn facts_resource_name( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + lineage_facts::facts_resource_name(instance, args, kwargs) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/mod.rs b/crates/sqlbuild-python-project/src/lineage/main/mod.rs new file mode 100644 index 0000000000..0f7319858c --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/mod.rs @@ -0,0 +1,8 @@ +pub mod confidence_member; +pub mod facts_eq; +pub mod facts_getitem; +pub mod facts_iter; +pub mod facts_len; +pub mod facts_projected; +pub mod facts_resource_name; +pub mod transform_kind_member; diff --git a/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs new file mode 100644 index 0000000000..4a9b2cc04f --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs @@ -0,0 +1,13 @@ +//! The `ColumnTransformKind` member of a native transform code. + +use crate::lineage::_helpers::lineage_facts; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The `ColumnTransformKind` member of a native transform code. +#[inline] +pub fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { + lineage_facts::transform_kind_member(py, code) +} diff --git a/crates/sqlbuild-python-project/src/lineage/mod.rs b/crates/sqlbuild-python-project/src/lineage/mod.rs new file mode 100644 index 0000000000..009b7f7a66 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/mod.rs @@ -0,0 +1,4 @@ +//! Lineage facts of compiled models. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs new file mode 100644 index 0000000000..b336f64d4d --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod orchestration_triggers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs rename to crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs index 5c789c7eaa..ca91f52089 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs @@ -1,5 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. +use crate::native_host::models::CalendarRequest; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; @@ -17,7 +18,7 @@ use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; +use crate::graph::models::ProjectSelection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -34,13 +35,6 @@ use sqlbuild_python_front::discovery::main::python_files::provider_by_name::prov use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; use sqlbuild_python_front::discovery::models::SecretPolicy; -/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. -pub(crate) struct CalendarRequest { - pub(crate) calendar: String, - pub(crate) timezone: String, - pub(crate) trigger: TriggerDefinition, -} - fn horizon(value: &str) -> PyResult { let invalid = || { PyValueError::new_err( diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs new file mode 100644 index 0000000000..b5be81b2b1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs @@ -0,0 +1,22 @@ +//! Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. + +use crate::native_host::_helpers::orchestration_triggers; +use crate::native_host::models::CalendarRequest; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::declarations::models::Definitions; +use std::collections::BTreeMap; +use std::path::Path; + +/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. +#[inline] +pub fn compile_calendars( + py: Python<'_>, + project_dir: &Path, + (calendars, requests): (&BTreeMap>, Vec), + definitions: Definitions, +) -> PyResult { + orchestration_triggers::compile_calendars(py, project_dir, (calendars, requests), definitions) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs new file mode 100644 index 0000000000..1003be8915 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs @@ -0,0 +1,22 @@ +//! Compile sensors. + +use crate::graph::models::ProjectSelection; +use crate::native_host::_helpers::orchestration_triggers; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::declarations::models::Definitions; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; +use std::path::Path; + +/// Compile sensors. +#[inline] +pub fn compile_sensors( + py: Python<'_>, + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), + authored: Vec, + definitions: Definitions, +) -> PyResult { + orchestration_triggers::compile_sensors(py, (project_dir, input, graph), authored, definitions) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs new file mode 100644 index 0000000000..9d80042f90 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -0,0 +1,2 @@ +pub mod compile_calendars; +pub mod compile_sensors; diff --git a/crates/sqlbuild-python-project/src/native_host/mod.rs b/crates/sqlbuild-python-project/src/native_host/mod.rs new file mode 100644 index 0000000000..9fbdf0b3ef --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/mod.rs @@ -0,0 +1,5 @@ +//! Model lookup and orchestration triggers. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs b/crates/sqlbuild-python-project/src/native_host/models.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs rename to crates/sqlbuild-python-project/src/native_host/models.rs index 78f7958d10..415afdd71a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -1,37 +1,40 @@ -//! Public model-map inputs and projections around the native planner's model lookup. - -use std::sync::Arc; +//! Public records of native host. +use crate::compiled_views::models::NativeCompiledModel; +use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::planner_scope::retained::ScopeData; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::declarations::models::TriggerDefinition; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use std::sync::Arc; /// Native compiled views stay typed; caller-authored protocol implementations remain public values. #[derive(Debug)] -pub(crate) enum ModelHandle { +pub enum ModelHandle { View(Py), Public(Py), } impl ModelHandle { - pub(crate) fn read(value: &Bound<'_, PyAny>) -> Self { + pub fn read(value: &Bound<'_, PyAny>) -> Self { match value.cast::() { Ok(value) => Self::View(value.clone().unbind()), Err(_) => Self::Public(value.clone().unbind()), } } - pub(crate) fn bind<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + pub fn bind<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { match self { Self::View(value) => value.bind(py).clone().into_any(), Self::Public(value) => value.bind(py).clone(), } } - pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + pub fn clone_ref(&self, py: Python<'_>) -> Self { match self { Self::View(value) => Self::View(value.clone_ref(py)), Self::Public(value) => Self::Public(value.clone_ref(py)), @@ -41,20 +44,20 @@ impl ModelHandle { /// A public caller's mapping, or the compiler-owned models shared by native planning. #[derive(Debug)] -pub(crate) enum ModelLookup { +pub enum ModelLookup { Public(Py), Native(Arc), } impl ModelLookup { - pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + pub fn clone_ref(&self, py: Python<'_>) -> Self { match self { Self::Public(value) => Self::Public(value.clone_ref(py)), Self::Native(value) => Self::Native(value.clone()), } } - pub(crate) fn get<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + pub fn get<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { match self { Self::Public(value) => mapping_get(value.bind(py), name), Self::Native(value) => Ok(value @@ -63,7 +66,7 @@ impl ModelLookup { } } - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { match self { Self::Public(value) => Ok(value.bind(py).clone()), Self::Native(value) => value.field(py, "models_by_name"), @@ -86,3 +89,11 @@ impl<'py> IntoPyObject<'py> for &ModelLookup { self.project(py) } } + +/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. +#[derive(Debug)] +pub struct CalendarRequest { + pub calendar: String, + pub timezone: String, + pub trigger: TriggerDefinition, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs rename to crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index a90cb8ea1e..bcd17e0ee7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -21,16 +21,22 @@ use sqlbuild_analysis::planner_identity::models::{ AvailableUpstreams, ExecutionTest, GraphKey, MissingUpstream, }; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::planner_identity::hashing::graph_key; -use crate::bindings::_helpers::references::explicit_references::{ - GateRow, HookRow, native_execution_gate_edges, -}; +use crate::compiled_views::main::project_reads::functions::functions; +use crate::compiled_views::main::project_reads::models::models; +use crate::compiled_views::main::project_reads::seeds::seeds; +use crate::compiled_views::main::project_reads::sources::sources; +use crate::compiled_views::main::project_reads::sql_tests::sql_tests; +use crate::compiled_views::main::resource_reads::graph_row::graph_row; +use crate::graph::main::algorithms::closure::closure; +use crate::graph::main::algorithms::inverted::inverted; +use crate::graph::main::keys::native_key::native_key; +use crate::graph::main::views::edge_dict::edge_dict; +use crate::graph::models::KeyCache; +use crate::graph::models::NativeProjectGraph; +use crate::planner_identity::_helpers::hashing::graph_key; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::models::GateRow; +use crate::references::models::HookRow; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -97,19 +103,19 @@ pub(crate) fn execution_deps( ) -> PyResult)>> { let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); let collections: [Bound<'_, PyAny>; 4] = [ - project_reads::models(project)?, - project_reads::sources(project)?, - project_reads::seeds(project)?, - project_reads::functions(project)?, + models(project)?, + sources(project)?, + seeds(project)?, + functions(project)?, ]; for collection in collections { for resource in collection.try_iter()? { - let row = resource_reads::graph_row(&resource?)?; + let row = graph_row(&resource?)?; resources.push((row.key, row.deps)); } } let mut tests: Vec = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test = test?; tests.push(ExecutionTest { key: native_key(&native_field(&test, "key")?)?, @@ -149,7 +155,7 @@ pub(crate) fn execution_origins( project: &Bound<'_, PyAny>, ) -> PyResult> { let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test = test?; let test_name: String = native_field(&test, "name")?.extract()?; for target_key in native_field(&test, "scope_deps")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs rename to crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..35817cbc58 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod graph; +pub(crate) mod hashing; diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs new file mode 100644 index 0000000000..fe7a7755e8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs @@ -0,0 +1,23 @@ +//! The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. +#[inline] +pub fn baselined_function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), + recorded: &Bound<'_, PyAny>, +) -> PyResult { + hashing::baselined_function_definition_hash( + py, + function_name, + (fingerprint_sql, language, dialect), + recorded, + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs new file mode 100644 index 0000000000..ac498893a9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs @@ -0,0 +1,18 @@ +//! The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. +#[inline] +pub fn baselined_model_query_hash( + py: Python<'_>, + model_name: &str, + (query_sql, dialect): (&str, Option<&str>), + recorded: &Bound<'_, PyAny>, +) -> PyResult { + hashing::baselined_model_query_hash(py, model_name, (query_sql, dialect), recorded) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs new file mode 100644 index 0000000000..e8814c91d3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs @@ -0,0 +1,13 @@ +//! Execution dependencies kept native until a public graph or plan asks for its dict view. + +use crate::planner_identity::_helpers::graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Execution dependencies kept native until a public graph or plan asks for its dict view. +#[inline] +pub fn execution_deps(project: &Bound<'_, PyAny>) -> PyResult)>> { + graph::execution_deps(project) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs new file mode 100644 index 0000000000..bbb02f50a2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs @@ -0,0 +1,16 @@ +//! Stable native execution order, projecting no Python graph or key objects. + +use crate::planner_identity::_helpers::graph; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Stable native execution order, projecting no Python graph or key objects. +#[inline] +pub fn execution_order( + py: Python<'_>, + upstream: &[(ObjectKey, Vec)], + origins: &[((ObjectKey, ObjectKey), String)], +) -> PyResult> { + graph::execution_order(py, upstream, origins) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs new file mode 100644 index 0000000000..17deec03e6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs @@ -0,0 +1,15 @@ +//! Injected-edge descriptions remain native for cycle and buildability diagnostics. + +use crate::planner_identity::_helpers::graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Injected-edge descriptions remain native for cycle and buildability diagnostics. +#[inline] +pub fn execution_origins( + project: &Bound<'_, PyAny>, +) -> PyResult> { + graph::execution_origins(project) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs new file mode 100644 index 0000000000..fef31d807d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs @@ -0,0 +1,15 @@ +//! The definition fingerprint of a function, or the planner input error naming the function. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The definition fingerprint of a function, or the planner input error naming the function. +#[inline] +pub fn function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), +) -> PyResult { + hashing::function_definition_hash(py, function_name, (fingerprint_sql, language, dialect)) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs new file mode 100644 index 0000000000..bee99c8141 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs @@ -0,0 +1,13 @@ +//! The graph key of a `CompiledObjectKey`-like value: its resource type text and name. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::planner_identity::models::GraphKey; + +/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. +#[inline] +pub fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { + hashing::graph_key(key) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..d007f0bb8b --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs @@ -0,0 +1,10 @@ +pub mod baselined_function_definition_hash; +pub mod baselined_model_query_hash; +pub mod execution_deps; +pub mod execution_order; +pub mod execution_origins; +pub mod function_definition_hash; +pub mod graph_key; +pub mod model_definition_hash; +pub mod register_graph; +pub mod register_hashing; diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs new file mode 100644 index 0000000000..1def34d9b3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs @@ -0,0 +1,16 @@ +//! Fingerprint a model query for change detection, naming the model when it cannot. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Fingerprint a model query for change detection, naming the model when it cannot. +#[inline] +pub fn model_definition_hash( + py: Python<'_>, + model_name: &str, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + hashing::model_definition_hash(py, model_name, query_sql, dialect) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs new file mode 100644 index 0000000000..d24ebee179 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs @@ -0,0 +1,12 @@ +//! Register the graph functions and classes. + +use crate::planner_identity::_helpers::graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the graph functions and classes. +#[inline] +pub fn register_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + graph::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs new file mode 100644 index 0000000000..7065675431 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs @@ -0,0 +1,12 @@ +//! Register the hashing functions and classes. + +use crate::planner_identity::_helpers::hashing; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the hashing functions and classes. +#[inline] +pub fn register_hashing(module: &Bound<'_, PyModule>) -> PyResult<()> { + hashing::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/mod.rs new file mode 100644 index 0000000000..5e15399939 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/mod.rs @@ -0,0 +1,4 @@ +//! Planner identities: the graph and its hashes. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs new file mode 100644 index 0000000000..d8ae2709c2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs rename to crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs index de5c5ebcdf..54dc4f9392 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs @@ -2,8 +2,7 @@ use sqlbuild_host::entries::models::HostId; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; +use crate::compiled_views::main::project_reads::collection::collection; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; use pyo3::{BoundObject, IntoPyObject}; @@ -144,9 +143,13 @@ pub(crate) fn by_name<'py>( field: &str, ) -> PyResult> { let nodes = pyo3::types::PyDict::new(project.py()); - for node in project_reads::collection(project, field)?.try_iter()? { + for node in collection(project, field)?.try_iter()? { let node = node?; - pyo3::types::PyDictMethods::set_item(&nodes, resource_reads::name(&node)?, node)?; + pyo3::types::PyDictMethods::set_item( + &nodes, + crate::compiled_views::main::resource_reads::name::name(&node)?, + node, + )?; } Ok(nodes) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs new file mode 100644 index 0000000000..e96b975c0c --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs @@ -0,0 +1,15 @@ +//! The project nodes of `field` by name. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project nodes of `field` by name. +#[inline] +pub fn by_name<'py>( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + objects::by_name(project, field) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs new file mode 100644 index 0000000000..baf636fa88 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs @@ -0,0 +1,12 @@ +//! A model config value as text, `None` when absent or not text. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A model config value as text, `None` when absent or not text. +#[inline] +pub fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { + objects::config_str(model, key) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs new file mode 100644 index 0000000000..2b86df19cc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs @@ -0,0 +1,18 @@ +//! The staying-Python class `class(**fields)`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_host::entries::models::HostId; + +/// The staying-Python class `class(**fields)`. +#[inline] +pub fn construct<'py>( + py: Python<'py>, + class: impl Into, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + objects::construct(py, class, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs new file mode 100644 index 0000000000..5dc8500540 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs @@ -0,0 +1,13 @@ +//! A frozenset of the items. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A frozenset of the items. +#[inline] +pub fn frozenset<'py>(py: Python<'py>, items: &[Bound<'py, PyAny>]) -> PyResult> { + objects::frozenset(py, items) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs new file mode 100644 index 0000000000..0b9930f1c9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs @@ -0,0 +1,18 @@ +//! A staying-Python enum member from its value. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_host::entries::models::HostId; + +/// A staying-Python enum member from its value. +#[inline] +pub fn member<'py>( + py: Python<'py>, + class: impl Into, + value: &str, +) -> PyResult> { + objects::member(py, class, value) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs new file mode 100644 index 0000000000..f8acd3faea --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs @@ -0,0 +1,17 @@ +pub mod by_name; +pub mod config_str; +pub mod construct; +pub mod frozenset; +pub mod member; +pub mod model_key; +pub mod native; +pub mod native_member; +pub mod object; +pub mod optional_text; +pub mod planner_error; +pub mod planner_model; +pub mod replace; +pub mod sorted_names; +pub mod strings; +pub mod text; +pub mod tuple; diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs new file mode 100644 index 0000000000..b98b22f067 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs @@ -0,0 +1,13 @@ +//! The compiled object key of the model `name`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The compiled object key of the model `name`. +#[inline] +pub fn model_key<'py>(py: Python<'py>, name: &Bound<'py, PyAny>) -> PyResult> { + objects::model_key(py, name) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs new file mode 100644 index 0000000000..f44ccea1fb --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs @@ -0,0 +1,18 @@ +//! The native record `name(**fields)`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::RecordId; + +/// The native record `name(**fields)`. +#[inline] +pub fn native<'py>( + py: Python<'py>, + name: RecordId, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + objects::native(py, name, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs new file mode 100644 index 0000000000..83504d8648 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs @@ -0,0 +1,18 @@ +//! A native string enum member from its value. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::RecordId; + +/// A native string enum member from its value. +#[inline] +pub fn native_member<'py>( + py: Python<'py>, + class: RecordId, + value: &str, +) -> PyResult> { + objects::native_member(py, class, value) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/object.rs b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs new file mode 100644 index 0000000000..c79d808128 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs @@ -0,0 +1,19 @@ +//! A native helper's Rust result as a Python object. + +use crate::planner_phases::_helpers::objects; +use pyo3::IntoPyObject; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A native helper's Rust result as a Python object. +#[inline] +pub fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> +where + T: IntoPyObject<'py>, + PyErr: From, +{ + objects::object(py, value) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs new file mode 100644 index 0000000000..172453dc34 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs @@ -0,0 +1,12 @@ +//! `None`, or the text. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::Python; + +/// `None`, or the text. +#[inline] +pub fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + objects::optional_text(py, value) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs new file mode 100644 index 0000000000..6343200690 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs @@ -0,0 +1,15 @@ +//! `PlannerInputError(message, code=code, help=help)`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// `PlannerInputError(message, code=code, help=help)`. +#[inline] +pub fn planner_error( + py: Python<'_>, + message: &str, + (code, help): (Option<&str>, Option<&str>), +) -> PyErr { + objects::planner_error(py, message, (code, help)) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs new file mode 100644 index 0000000000..ce3819ce10 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs @@ -0,0 +1,18 @@ +//! A planner model dataclass. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::RecordId; + +/// A planner model dataclass. +#[inline] +pub fn planner_model<'py>( + py: Python<'py>, + name: RecordId, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + objects::planner_model(py, name, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs new file mode 100644 index 0000000000..b6666f0a7a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs @@ -0,0 +1,15 @@ +//! `dataclasses.replace(value, **fields)`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `dataclasses.replace(value, **fields)`. +#[inline] +pub fn replace<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + objects::replace(value, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs new file mode 100644 index 0000000000..cdddb0b35e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs @@ -0,0 +1,13 @@ +//! `sorted(items)` as a tuple of strings. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `sorted(items)` as a tuple of strings. +#[inline] +pub fn sorted_names<'py>(py: Python<'py>, names: Vec) -> PyResult> { + objects::sorted_names(py, names) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs new file mode 100644 index 0000000000..f2c03bea04 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs @@ -0,0 +1,12 @@ +//! The strings of any Python iterable, in iteration order. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The strings of any Python iterable, in iteration order. +#[inline] +pub fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + objects::strings(values) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs new file mode 100644 index 0000000000..6f33a4c39b --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs @@ -0,0 +1,12 @@ +//! A Python `str`. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::Python; + +/// A Python `str`. +#[inline] +pub fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + objects::text(py, value) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs new file mode 100644 index 0000000000..33b17ae72a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs @@ -0,0 +1,13 @@ +//! A tuple of the items. + +use crate::planner_phases::_helpers::objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A tuple of the items. +#[inline] +pub fn tuple<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + objects::tuple(py, items) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/mod.rs new file mode 100644 index 0000000000..efdc766b58 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/mod.rs @@ -0,0 +1,4 @@ +//! Planner phase objects and scopes. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs index 4d9bde263f..8d8b0206b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs @@ -1,10 +1,13 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::planner_scope::values::{ - dependency_names, edges, keys, loader_graph, loader_names, project_graph, -}; +use crate::graph::main::views::edge_dict::edge_dict; +use crate::graph::models::KeyCache; +use crate::planner_scope::_helpers::values::dependency_names; +use crate::planner_scope::_helpers::values::edges; +use crate::planner_scope::_helpers::values::keys; +use crate::planner_scope::_helpers::values::loader_graph; +use crate::planner_scope::_helpers::values::loader_names; +use crate::planner_scope::_helpers::values::project_graph; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; @@ -131,3 +134,24 @@ pub(crate) fn expand_selected_loader_dependencies<'py>( edge_dict(&cache, &upstream)?, )) } + +/// `build_intermediate_source_map` as a function of `module`. +pub(crate) fn build_intermediate_source_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_intermediate_source_map, module) +} + +/// `expand_selected_loader_dependencies` as a function of `module`. +pub(crate) fn expand_selected_loader_dependencies_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(expand_selected_loader_dependencies, module) +} + +/// `upstream_loader_dependency_names` as a function of `module`. +pub(crate) fn upstream_loader_dependency_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(upstream_loader_dependency_names, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs new file mode 100644 index 0000000000..df51068857 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod loaders; +pub(crate) mod retained; +pub(crate) mod scope; +pub(crate) mod sources; +pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs new file mode 100644 index 0000000000..40db90ad5a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -0,0 +1,94 @@ +//! Native planner scope ownership, with projection only at public record boundaries. + +use crate::planner_scope::models::RetainedScope; +use crate::planner_scope::models::ScopeData; +use std::collections::HashMap; +use std::sync::Arc; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use crate::native_host::models::ModelHandle; +use crate::planner_scope::_helpers::values::edges; +use crate::planner_scope::_helpers::values::keys; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::models::NativeRecord; +use sqlbuild_python_base::records::types::RecordId; + +/// Read native ownership directly, or import a caller-authored public scope at the API edge. +pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { + if let Ok(scope) = value.cast::() { + return Ok(scope.get().data.clone()); + } + let mut models: HashMap = HashMap::new(); + let mut model_order: Vec = Vec::new(); + for item in mapping_items(&field(value, "models_by_name")?)?.try_iter()? { + let item = item?; + let name: String = item.get_item(0)?.extract()?; + model_order.push(name.clone()); + models.insert(name, ModelHandle::read(&item.get_item(1)?)); + } + Ok(Arc::new(ScopeData { + upstream: edges(&field(value, "upstream_deps")?)?, + downstream: edges(&field(value, "downstream_deps")?)?, + names: mapping_items(&field(value, "all_keys")?)? + .try_iter()? + .map(|item| { + let item = item?; + Ok(( + item.get_item(0)?.extract()?, + crate::graph::main::keys::native_key::native_key(&item.get_item(1)?)?, + )) + }) + .collect::>()?, + models, + model_order, + selected: keys(&field(value, "selected_keys")?)?.into_iter().collect(), + order: keys(&field(value, "execution_order")?)?, + user_selected: keys(&field(value, "user_selected_keys")?)? + .into_iter() + .collect(), + python_reads: field(value, "python_read_source_names")?.extract()?, + tests: field(value, "sql_test_selection")?.extract()?, + load_sources: field(value, "load_sources")?.is_truthy()?, + end_audit_sources: keys(&field(value, "end_audit_source_keys")?)? + .into_iter() + .collect(), + })) +} + +/// Preserve public record identity and dataclass behavior on explicit projection. +pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs index 3926e1fe22..9892da4ce2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs @@ -1,17 +1,19 @@ //! Planner scope assembly using native graph and selector operations. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_identity::graph::{ - execution_deps, execution_order, execution_origins, -}; -use crate::bindings::_helpers::planner_scope::retained::ScopeData; -use crate::bindings::_helpers::planner_scope::values::{keys, project_graph}; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_keys; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::project_reads::sources::sources; +use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::graph::models::NativeProjectGraph; +use crate::native_host::models::ModelHandle; +use crate::planner_identity::main::execution_deps::execution_deps; +use crate::planner_identity::main::execution_order::execution_order; +use crate::planner_identity::main::execution_origins::execution_origins; +use crate::planner_scope::_helpers::values::keys; +use crate::planner_scope::_helpers::values::project_graph; +use crate::planner_scope::models::ScopeData; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; +use crate::planner_selection::models::Carrier; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; @@ -78,10 +80,12 @@ pub(crate) fn build_planner_scope<'py>( }; if auto_load { let mut managed: HashSet = HashSet::new(); - for source in project_reads::sources(&project)?.try_iter()? { + for source in sources(&project)?.try_iter()? { let source = source?; - if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { - managed.insert(source_reads::name(&source)?.extract()?); + if !native_field(&source_entry(&source)?, "loader")?.is_none() { + managed.insert( + crate::compiled_views::main::source_reads::name::name(&source)?.extract()?, + ); } } let loaded = managed_sources(&selected, &upstream, &managed); @@ -110,9 +114,9 @@ pub(crate) fn build_planner_scope<'py>( let mut models: std::collections::HashMap = std::collections::HashMap::new(); let mut model_order: Vec = Vec::new(); - for model in project_reads::models(&project)?.try_iter()? { + for model in crate::compiled_views::main::project_reads::models::models(&project)?.try_iter()? { let model = model?; - let name: String = model_reads::name_object(&model)?.extract()?; + let name: String = name_object(&model)?.extract()?; if !models.contains_key(&name) { model_order.push(name.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs index f603c490ee..aa0740f72d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs @@ -2,11 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; -use crate::bindings::_helpers::planner_scope::values::{edges, keys}; +use crate::compiled_views::main::project_reads::sources::sources; +use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::graph::main::keys::native_key::native_key; +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::planner_scope::_helpers::values::edges; +use crate::planner_scope::_helpers::values::keys; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyDictMethods; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -40,14 +41,16 @@ pub(crate) fn source_load_map<'py>( ) -> PyResult> { let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); let result = PyDict::new(project.py()); - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let source = source?; if let Some(keys) = &relevant - && !keys.contains(&native_key(&source_reads::key(&source)?)?) + && !keys.contains(&native_key( + &crate::compiled_views::main::source_reads::key::key(&source)?, + )?) { continue; } - let entry = source_reads::source_entry(&source)?; + let entry = source_entry(&source)?; result.set_item(native_field(&entry, "name")?, entry)?; } for upstream in [false, true] { @@ -131,3 +134,17 @@ pub(crate) fn build_source_load_entries<'py>( } PyTuple::new(py, entries) } + +/// `build_source_load_entries` as a function of `module`. +pub(crate) fn build_source_load_entries_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_source_load_entries, module) +} + +/// `build_source_load_map` as a function of `module`. +pub(crate) fn build_source_load_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_source_load_map, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs index 713ae4ab47..3d6bb63a4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs @@ -1,8 +1,8 @@ //! Python function identity and compiled graph conversion. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::native_key; +use crate::compiled_views::main::project_reads::sources::sources; +use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::graph::main::keys::native_key::native_key; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; @@ -69,8 +69,8 @@ pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; graph.sources.push(( native_field(&entry, "name")?.extract()?, native_field(&entry, "loader")?.extract()?, diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs new file mode 100644 index 0000000000..b0f5781bc7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs @@ -0,0 +1,16 @@ +//! Build intermediate source map. + +use crate::planner_scope::_helpers::loaders; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Build intermediate source map. +#[inline] +pub fn build_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + loaders::build_intermediate_source_map(project, selected_keys) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs new file mode 100644 index 0000000000..2ae00ddc8f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs @@ -0,0 +1,11 @@ +//! `build_intermediate_source_map` as a function of `module`. + +use crate::planner_scope::_helpers::loaders; + +/// `build_intermediate_source_map` as a function of `module`. +#[inline] +pub fn build_intermediate_source_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + loaders::build_intermediate_source_map_function(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs new file mode 100644 index 0000000000..54514bf26d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs @@ -0,0 +1,17 @@ +//! Build planner scope. + +use crate::planner_scope::_helpers::scope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Build planner scope. +#[inline] +pub fn build_planner_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + scope::build_planner_scope(py, options) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs new file mode 100644 index 0000000000..284b37217e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs @@ -0,0 +1,19 @@ +//! Build source load entries. + +use crate::planner_scope::_helpers::sources; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Build source load entries. +#[inline] +pub fn build_source_load_entries<'py>( + execution_order: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + is_reload: bool, +) -> PyResult> { + sources::build_source_load_entries(execution_order, selected_keys, source_map, is_reload) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs new file mode 100644 index 0000000000..7f38b930dc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs @@ -0,0 +1,11 @@ +//! `build_source_load_entries` as a function of `module`. + +use crate::planner_scope::_helpers::sources; + +/// `build_source_load_entries` as a function of `module`. +#[inline] +pub fn build_source_load_entries_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + sources::build_source_load_entries_function(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs new file mode 100644 index 0000000000..d5aad260b7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs @@ -0,0 +1,11 @@ +//! `build_source_load_map` as a function of `module`. + +use crate::planner_scope::_helpers::sources; + +/// `build_source_load_map` as a function of `module`. +#[inline] +pub fn build_source_load_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + sources::build_source_load_map_function(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs new file mode 100644 index 0000000000..11efd56bf0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs @@ -0,0 +1,16 @@ +//! Build upstream intermediate source map. + +use crate::planner_scope::_helpers::loaders; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Build upstream intermediate source map. +#[inline] +pub fn build_upstream_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + loaders::build_upstream_intermediate_source_map(project, selected_keys) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs new file mode 100644 index 0000000000..73eaf1822f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs @@ -0,0 +1,13 @@ +//! Edges. + +use crate::planner_scope::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Edges. +#[inline] +pub fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { + values::edges(values) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs new file mode 100644 index 0000000000..5718aeb8a1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs @@ -0,0 +1,11 @@ +//! `expand_selected_loader_dependencies` as a function of `module`. + +use crate::planner_scope::_helpers::loaders; + +/// `expand_selected_loader_dependencies` as a function of `module`. +#[inline] +pub fn expand_selected_loader_dependencies_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + loaders::expand_selected_loader_dependencies_function(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs new file mode 100644 index 0000000000..913a26182a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs @@ -0,0 +1,13 @@ +//! Keys. + +use crate::planner_scope::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Keys. +#[inline] +pub fn keys(values: &Bound<'_, PyAny>) -> PyResult> { + values::keys(values) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs new file mode 100644 index 0000000000..03a88c4a60 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs @@ -0,0 +1,16 @@ +pub mod build_intermediate_source_map; +pub mod build_intermediate_source_map_function; +pub mod build_planner_scope; +pub mod build_source_load_entries; +pub mod build_source_load_entries_function; +pub mod build_source_load_map_function; +pub mod build_upstream_intermediate_source_map; +pub mod edges; +pub mod expand_selected_loader_dependencies_function; +pub mod keys; +pub mod project_graph; +pub mod project_scope; +pub mod scope_data; +pub mod source_load_map; +pub mod upstream_loader_dependency_names; +pub mod upstream_loader_dependency_names_function; diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs new file mode 100644 index 0000000000..ccdab14bc1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs @@ -0,0 +1,13 @@ +//! Project graph. + +use crate::planner_scope::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::planner_scope::models::LoaderGraph; + +/// Project graph. +#[inline] +pub fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { + values::project_graph(project) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs new file mode 100644 index 0000000000..a05306cd50 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs @@ -0,0 +1,12 @@ +//! Preserve public record identity and dataclass behavior on explicit projection. + +use crate::planner_scope::_helpers::retained; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Preserve public record identity and dataclass behavior on explicit projection. +#[inline] +pub fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + retained::project_scope(value) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs new file mode 100644 index 0000000000..de17d920f6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs @@ -0,0 +1,14 @@ +//! Read native ownership directly, or import a caller-authored public scope at the API edge. + +use crate::planner_scope::_helpers::retained; +use crate::planner_scope::models::ScopeData; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use std::sync::Arc; + +/// Read native ownership directly, or import a caller-authored public scope at the API edge. +#[inline] +pub fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { + retained::scope_data(value) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs new file mode 100644 index 0000000000..dee238edc0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs @@ -0,0 +1,18 @@ +//! Select source entries from native reachability; the resulting entries cross the executor edge. + +use crate::planner_scope::_helpers::sources; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Select source entries from native reachability; the resulting entries cross the executor edge. +#[inline] +pub fn source_load_map<'py>( + project: &Bound<'py, PyAny>, + selected: &[ObjectKey], + upstream_deps: Option<&[(ObjectKey, Vec)]>, +) -> PyResult> { + sources::source_load_map(project, selected, upstream_deps) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs new file mode 100644 index 0000000000..892acbee20 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs @@ -0,0 +1,16 @@ +//! Upstream loader dependency names. + +use crate::planner_scope::_helpers::loaders; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Upstream loader dependency names. +#[inline] +pub fn upstream_loader_dependency_names<'py>( + loader_function: &Bound<'py, PyAny>, + loader_functions: &Bound<'py, PyAny>, +) -> PyResult> { + loaders::upstream_loader_dependency_names(loader_function, loader_functions) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs new file mode 100644 index 0000000000..d4eeff3d83 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs @@ -0,0 +1,11 @@ +//! `upstream_loader_dependency_names` as a function of `module`. + +use crate::planner_scope::_helpers::loaders; + +/// `upstream_loader_dependency_names` as a function of `module`. +#[inline] +pub fn upstream_loader_dependency_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + loaders::upstream_loader_dependency_names_function(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/mod.rs new file mode 100644 index 0000000000..6b362db2c5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/mod.rs @@ -0,0 +1,5 @@ +//! The planner scope: loaders, sources, retained values and scope values. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/planner_scope/models.rs b/crates/sqlbuild-python-project/src/planner_scope/models.rs new file mode 100644 index 0000000000..a58d239e0a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/models.rs @@ -0,0 +1,131 @@ +//! Public records of planner scope. + +use crate::graph::main::views::edge_dict::edge_dict; +use crate::graph::models::KeyCache; +use crate::native_host::models::ModelHandle; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyFrozenSet; +use pyo3::types::PyTuple; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::records::models::NativeRecord; +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +/// The graph and selection used throughout planning; model values remain compiled-resource views. +#[derive(Debug)] +pub struct ScopeData { + pub upstream: Vec<(ObjectKey, Vec)>, + pub downstream: Vec<(ObjectKey, Vec)>, + pub names: Vec<(String, ObjectKey)>, + pub models: HashMap, + pub model_order: Vec, + pub selected: HashSet, + pub order: Vec, + pub user_selected: HashSet, + pub python_reads: HashSet, + pub tests: Py, + pub load_sources: bool, + pub end_audit_sources: HashSet, +} + +/// Internal carrier, never returned in place of a supported public PlannerScope record. +#[pyclass(frozen)] +pub struct RetainedScope { + pub data: Arc, + pub(crate) projection: PyOnceLock>, +} + +impl std::fmt::Debug for RetainedScope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RetainedScope") + .finish_non_exhaustive() + } +} + +impl ScopeData { + pub fn object(self, py: Python<'_>) -> PyResult> { + Ok(Bound::new( + py, + RetainedScope { + data: Arc::new(self), + projection: PyOnceLock::new(), + }, + )? + .into_any()) + } + + pub fn model<'py>(&self, py: Python<'py>, name: &str) -> Option> { + self.models.get(name).map(|model| model.bind(py)) + } + + pub fn with_selected(&self, py: Python<'_>, selected: HashSet) -> Self { + Self { + upstream: self.upstream.clone(), + downstream: self.downstream.clone(), + names: self.names.clone(), + model_order: self.model_order.clone(), + models: self + .models + .iter() + .map(|(name, model)| (name.clone(), model.clone_ref(py))) + .collect(), + selected, + order: self.order.clone(), + user_selected: self.user_selected.clone(), + python_reads: self.python_reads.clone(), + tests: self.tests.clone_ref(py), + load_sources: self.load_sources, + end_audit_sources: self.end_audit_sources.clone(), + } + } + + /// Public field projection; internal graph consumers read the native fields above instead. + pub fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let cache = KeyCache::new(py)?; + let key_set = |keys: &HashSet| -> PyResult> { + Ok( + PyFrozenSet::new(py, cache.keys(&keys.iter().cloned().collect::>())?)? + .into_any(), + ) + }; + match name { + "upstream_deps" => Ok(edge_dict(&cache, &self.upstream)?.into_any()), + "downstream_deps" => Ok(edge_dict(&cache, &self.downstream)?.into_any()), + "selected_keys" => key_set(&self.selected), + "user_selected_keys" => key_set(&self.user_selected), + "end_audit_source_keys" => key_set(&self.end_audit_sources), + "execution_order" => Ok(PyTuple::new(py, cache.keys(&self.order)?)?.into_any()), + "all_keys" => { + let values = PyDict::new(py); + for (name, key) in &self.names { + values.set_item(name, cache.key(key)?)?; + } + Ok(values.into_any()) + } + "models_by_name" => { + let values = PyDict::new(py); + for name in &self.model_order { + if let Some(model) = self.models.get(name) { + values.set_item(name, model.bind(py))?; + } + } + Ok(values.into_any()) + } + "python_read_source_names" => Ok(PyFrozenSet::new(py, &self.python_reads)?.into_any()), + "sql_test_selection" => Ok(self.tests.bind(py).clone().into_any()), + "load_sources" => self.load_sources.into_bound_py_any(py), + _ => Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), + } + } +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs new file mode 100644 index 0000000000..c3209dac10 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs @@ -0,0 +1,28 @@ +//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; + +pub(crate) const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; + +/// Python strings for each item of an iterable of `str`. +pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { + values + .try_iter()? + .map(|value| Ok(value?.downcast_into::()?)) + .collect() +} + +pub(crate) fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { + let code: u32 = character + .py() + .import("builtins")? + .getattr("ord")? + .call1((character,))? + .extract()?; + Ok(code) +} + +pub(crate) fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { + py.import("builtins")?.getattr("chr")?.call1((code,)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs new file mode 100644 index 0000000000..f32364b5d9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod carrier; +pub(crate) mod model_selectors; +pub(crate) mod selectors; +pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs rename to crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs index 136cac00ac..e09b274fbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs @@ -12,8 +12,7 @@ use sqlbuild_analysis::planner_selection::models::{ ModelSelectorNames, SelectableModel, SelectorExpansion, }; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_selection::carrier::Carrier; +use crate::planner_selection::models::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -88,10 +87,11 @@ pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( term: &Bound<'py, PyString>, ) -> PyResult> { let py: Python<'py> = project.py(); - let models: Vec = project_reads::models(project)? - .try_iter()? - .map(|model| selectable_model(&model?)) - .collect::>()?; + let models: Vec = + crate::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { Ok(models .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs rename to crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs index 6e6cf885b8..4bf3f48784 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs @@ -10,12 +10,15 @@ use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_to use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; -use crate::bindings::_helpers::graph::algorithms::closure; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; -use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use crate::compiled_project::main::job_catalog::graph_with_jobs::graph_with_jobs; +use crate::graph::main::algorithms::closure::closure; +use crate::graph::main::views::graph_from_view_mappings::graph_from_view_mappings; +use crate::graph::models::CompiledProjectGraph; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::KeyCache; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::carrier::py_strings; +use crate::planner_selection::models::Carrier; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs rename to crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 7a8c3a69ab..117aa1d658 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -12,8 +12,9 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; -use crate::bindings::_helpers::compiled_project::job_catalog::job_names; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use crate::compiled_project::main::job_catalog::job_names::job_names; +use crate::planner_selection::_helpers::carrier::py_strings; +use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs new file mode 100644 index 0000000000..2487e94297 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs @@ -0,0 +1,16 @@ +//! The upstream key closure of one compiled graph key. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// The upstream key closure of one compiled graph key. +#[inline] +pub fn expand_project_upstream_keys<'py>( + key: Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + selectors::expand_project_upstream_keys(key, upstream_deps) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs new file mode 100644 index 0000000000..61df662932 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs @@ -0,0 +1,18 @@ +//! The keys one parsed selector matches before `+` graph expansion. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// The keys one parsed selector matches before `+` graph expansion. +#[inline] +pub fn match_project_selector<'py>( + parsed: &Bound<'py, PyAny>, + all_keys: &Bound<'py, PyAny>, + tag_index: &Bound<'py, PyAny>, + path_index: &Bound<'py, PyAny>, +) -> PyResult> { + selectors::match_project_selector(parsed, all_keys, tag_index, path_index) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs new file mode 100644 index 0000000000..5aada78366 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs @@ -0,0 +1,15 @@ +pub mod expand_project_upstream_keys; +pub mod match_project_selector; +pub mod parse_project_selector; +pub mod py_strings; +pub mod register_model_selectors; +pub mod reject_unit_test_selectors; +pub mod resolve_graph_keys; +pub mod resolve_graph_selection; +pub mod resolve_project_selector_tokens; +pub mod resolve_project_selectors; +pub mod resolve_sqlbuild_model_selector_names; +pub mod select_project_graph; +pub mod split_project_unit_test_selectors; +pub mod split_selector_expansion; +pub mod split_unit_test_selectors; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs new file mode 100644 index 0000000000..ff0c3b60e9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs @@ -0,0 +1,13 @@ +//! Parse one selector token into Python's `ParsedSelector` or `PathSelector`. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; + +/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. +#[inline] +pub fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + selectors::parse_project_selector(raw) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs new file mode 100644 index 0000000000..ec617699dc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs @@ -0,0 +1,13 @@ +//! Python strings for each item of an iterable of `str`. + +use crate::planner_selection::_helpers::carrier; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; + +/// Python strings for each item of an iterable of `str`. +#[inline] +pub fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { + carrier::py_strings(values) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs new file mode 100644 index 0000000000..de754ed49f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs @@ -0,0 +1,12 @@ +//! Register the model selectors functions and classes. + +use crate::planner_selection::_helpers::model_selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model selectors functions and classes. +#[inline] +pub fn register_model_selectors(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_selectors::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs new file mode 100644 index 0000000000..3e321e2401 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs @@ -0,0 +1,16 @@ +//! Raise when a selector names a unit test; return the unchanged resource selectors. + +use crate::planner_selection::_helpers::unit_tests; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Raise when a selector names a unit test; return the unchanged resource selectors. +#[inline] +pub fn reject_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + unit_tests::reject_unit_test_selectors(select, exclude, discovered_inputs) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs new file mode 100644 index 0000000000..f648dc9061 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs @@ -0,0 +1,17 @@ +//! Resolve graph keys for native planning without building a public frozenset. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::models::ProjectGraph; + +/// Resolve graph keys for native planning without building a public frozenset. +#[inline] +pub fn resolve_graph_keys<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + selectors::resolve_graph_keys(graph, (select, exclude)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs new file mode 100644 index 0000000000..95d13f3949 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs @@ -0,0 +1,19 @@ +//! `--select` minus `--exclude` on a native graph, adding the functions selected models need. + +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. +#[inline] +pub fn resolve_graph_selection<'py>( + graph: PyRef<'py, NativeProjectGraph>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + selectors::resolve_graph_selection(graph, select, exclude) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs new file mode 100644 index 0000000000..478a51e027 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs @@ -0,0 +1,17 @@ +//! The keys whitespace-separated selector tokens match, without required build resources. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyFrozenSet; + +/// The keys whitespace-separated selector tokens match, without required build resources. +#[inline] +pub fn resolve_project_selector_tokens<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + selectors::resolve_project_selector_tokens(py, options) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs new file mode 100644 index 0000000000..f49b926693 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs @@ -0,0 +1,17 @@ +//! Model, source, seed and function selectors resolved against compiled graph mappings. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyFrozenSet; + +/// Model, source, seed and function selectors resolved against compiled graph mappings. +#[inline] +pub fn resolve_project_selectors<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + selectors::resolve_project_selectors(py, options) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs new file mode 100644 index 0000000000..3bfa72b628 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs @@ -0,0 +1,17 @@ +//! One SQLBuild-owned model selector core's model names, and its path selector when translated. + +use crate::planner_selection::_helpers::model_selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; +use pyo3::types::PyTuple; + +/// One SQLBuild-owned model selector core's model names, and its path selector when translated. +#[inline] +pub fn resolve_sqlbuild_model_selector_names<'py>( + project: &Bound<'py, PyAny>, + term: &Bound<'py, PyString>, +) -> PyResult> { + model_selectors::resolve_sqlbuild_model_selector_names(project, term) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs new file mode 100644 index 0000000000..375d3757e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs @@ -0,0 +1,17 @@ +//! `--select` minus `--exclude` on a project graph, adding the functions selected models need. + +use crate::planner_selection::_helpers::selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. +#[inline] +pub fn select_project_graph<'py>( + graph: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + selectors::select_project_graph(graph, select, exclude) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs new file mode 100644 index 0000000000..fee54a6e46 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs @@ -0,0 +1,22 @@ +//! Move unit-test selectors out of resource selectors, or reject them for this command. + +use crate::planner_selection::_helpers::unit_tests; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Move unit-test selectors out of resource selectors, or reject them for this command. +#[inline] +pub fn split_project_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + unit_tests::split_project_unit_test_selectors( + select, + exclude, + discovered_inputs, + accepts_unit_tests, + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs new file mode 100644 index 0000000000..f33c7faeab --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs @@ -0,0 +1,13 @@ +//! Split `+selector+` marker syntax from the selector core without resolving the core. + +use crate::planner_selection::_helpers::model_selectors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyString; + +/// Split `+selector+` marker syntax from the selector core without resolving the core. +#[inline] +pub fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + model_selectors::split_selector_expansion(raw) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs new file mode 100644 index 0000000000..751849b1cc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs @@ -0,0 +1,16 @@ +//! Split unit-test selectors from resource selectors for a command that runs unit tests. + +use crate::planner_selection::_helpers::unit_tests; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Split unit-test selectors from resource selectors for a command that runs unit tests. +#[inline] +pub fn split_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + unit_tests::split_unit_test_selectors(select, exclude, discovered_inputs) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/mod.rs new file mode 100644 index 0000000000..4578773f1d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/mod.rs @@ -0,0 +1,5 @@ +//! Planner selection: selectors, model selectors, unit tests and their carrier. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python-project/src/planner_selection/models.rs similarity index 68% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs rename to crates/sqlbuild-python-project/src/planner_selection/models.rs index 4d788c3b31..72905eaf73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/models.rs @@ -1,25 +1,39 @@ -//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. - -use std::collections::{BTreeSet, HashSet}; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +//! Public records of planner selection. + +use crate::planner_selection::_helpers::carrier::PRIVATE_USE; +use crate::planner_selection::_helpers::carrier::ordinal; +use crate::planner_selection::_helpers::carrier::surrogate_text; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; use sqlbuild_analysis::graph::errors::SelectorError; - use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; - -const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; +use std::collections::BTreeSet; +use std::collections::HashSet; /// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. -pub(crate) struct Carrier<'py> { - py: Python<'py>, - tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +pub struct Carrier<'py> { + pub(crate) py: Python<'py>, + pub(crate) tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +} + +impl std::fmt::Debug for Carrier<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Carrier").finish_non_exhaustive() + } } impl<'py> Carrier<'py> { /// A carrier for `texts`, choosing carriers that appear in neither `texts` nor `reserved`. - pub(crate) fn new( + pub fn new( py: Python<'py>, texts: &[Bound<'py, PyString>], reserved: impl FnOnce() -> PyResult>, @@ -65,12 +79,12 @@ impl<'py> Carrier<'py> { } /// A carrier for one text with nothing reserved. - pub(crate) fn single(text: &Bound<'py, PyString>) -> PyResult { + pub fn single(text: &Bound<'py, PyString>) -> PyResult { Self::new(text.py(), std::slice::from_ref(text), || Ok(Vec::new())) } /// The text with surrogates replaced by their carriers. - pub(crate) fn send(&self, text: &Bound<'py, PyString>) -> PyResult { + pub fn send(&self, text: &Bound<'py, PyString>) -> PyResult { match &self.tables { None => Ok(text.to_str()?.to_owned()), Some((send, _)) => text.call_method1("translate", (send,))?.extract(), @@ -78,12 +92,12 @@ impl<'py> Carrier<'py> { } /// The texts with surrogates replaced by their carriers. - pub(crate) fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { + pub fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { texts.iter().map(|text| self.send(text)).collect() } /// The text with carriers restored to their surrogates. - pub(crate) fn back(&self, text: &str) -> PyResult> { + pub fn back(&self, text: &str) -> PyResult> { let restored: Bound<'py, PyString> = PyString::new(self.py, text); match &self.tables { None => Ok(restored), @@ -93,12 +107,12 @@ impl<'py> Carrier<'py> { } } - pub(crate) fn py(&self) -> Python<'py> { + pub fn py(&self) -> Python<'py> { self.py } /// The `PlannerInputError` instance for a selector failure, with carriers restored. - pub(crate) fn exception(&self, error: SelectorError) -> PyResult> { + pub fn exception(&self, error: SelectorError) -> PyResult> { let keywords: Bound<'py, PyDict> = PyDict::new(self.py); keywords.set_item("code", error.code)?; let help: Option> = error @@ -112,32 +126,10 @@ impl<'py> Carrier<'py> { } /// The `PlannerInputError` to raise for a selector failure, with carriers restored. - pub(crate) fn error(&self, error: SelectorError) -> PyErr { + pub fn error(&self, error: SelectorError) -> PyErr { match self.exception(error) { Ok(instance) => PyErr::from_value(instance), Err(failure) => failure, } } } - -/// Python strings for each item of an iterable of `str`. -pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { - values - .try_iter()? - .map(|value| Ok(value?.downcast_into::()?)) - .collect() -} - -fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { - let code: u32 = character - .py() - .import("builtins")? - .getattr("ord")? - .call1((character,))? - .extract()?; - Ok(code) -} - -fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { - py.import("builtins")?.getattr("chr")?.call1((code,)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs index 48c7fe6b83..91d994e8e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs @@ -8,29 +8,30 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context as project_planning_context; -use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; -use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; -use crate::bindings::_helpers::planner_strategy::fixture_selection::{ - fixture_keys_requiring_analysis, unknown_fixture_column_diagnostics, -}; -use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; -use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; -use crate::bindings::_helpers::planner_strategy::fixture_values::{ - columns, diagnostic, label, record, -}; -use crate::bindings::_helpers::planner_strategy::models::{ - ExpectedTypes, RelationFixturePlanningContext, -}; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::_helpers::fixture_context::planning_context as project_planning_context; +use crate::planner_strategy::_helpers::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixture_preparation::prepare; +use crate::planner_strategy::_helpers::fixture_reads::required_fixture_columns; +use crate::planner_strategy::_helpers::fixture_selection::fixture_keys_requiring_analysis; +use crate::planner_strategy::_helpers::fixture_selection::unknown_fixture_column_diagnostics; +use crate::planner_strategy::_helpers::fixture_sql::CompletionSql; +use crate::planner_strategy::_helpers::fixture_sql::render_completed; +use crate::planner_strategy::_helpers::fixture_types::expected_fixture_types; +use crate::planner_strategy::_helpers::fixture_values::columns; +use crate::planner_strategy::_helpers::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixture_values::label; +use crate::planner_strategy::_helpers::fixture_values::record; +use crate::planner_strategy::models::ExpectedTypes; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::records::types::RecordId; /// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. -pub(crate) struct FixtureCompletion<'py, 'a> { - pub(crate) sql_by_key: Bound<'py, PyDict>, +pub struct FixtureCompletion<'py, 'a> { + pub sql_by_key: Bound<'py, PyDict>, pub(crate) inferred_by_fixture: Bound<'py, PyDict>, pub(crate) expected_types: ExpectedTypes<'a>, - pub(crate) diagnostics: Vec>, + pub diagnostics: Vec>, } impl ExpectedTypes<'_> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs index 84e858ffdc..6191648c71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs @@ -10,24 +10,28 @@ use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_strategy::constants::{ - MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, -}; -use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_relation; -use crate::bindings::_helpers::planner_strategy::fixture_reads::lineage_reads; -use crate::bindings::_helpers::planner_strategy::fixture_types::projections; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; -use crate::bindings::_helpers::planner_strategy::models::{ - FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, -}; -use crate::bindings::_helpers::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::analysis_session::models::FixtureColumnInferences; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::project_reads::seeds::seeds; +use crate::compiled_views::main::project_reads::sources::sources; +use crate::compiled_views::main::resource_reads::name::name; +use crate::compiled_views::main::seed_reads::schema_entry::schema_entry; +use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::native_host::models::ModelHandle; +use crate::planner_strategy::_helpers::fixture_metadata::model_fixture_relation; +use crate::planner_strategy::_helpers::fixture_reads::lineage_reads; +use crate::planner_strategy::_helpers::fixture_types::projections; +use crate::planner_strategy::_helpers::fixture_values::columns; +use crate::planner_strategy::_helpers::fixture_values::folded_name; +use crate::planner_strategy::_helpers::types::FixtureRelationInput; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::constants::SEED_REFERENCE_KIND; +use crate::planner_strategy::constants::SOURCE_REFERENCE_KIND; +use crate::planner_strategy::models::FixtureReadCacheHost; +use crate::planner_strategy::models::FixtureRelation; +use crate::planner_strategy::models::ModelFixtureFacts; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::records::main::fields::field::field as native_field; static FIXTURE_READS: PyOnceLock> = PyOnceLock::new(); @@ -60,12 +64,11 @@ pub(crate) fn planning_context( .collect::>()?, None => { let mut models: HashMap = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + crate::compiled_views::main::project_reads::models::models(project)?.try_iter()? + { let model = model?; - models.insert( - model_reads::name_object(&model)?.extract()?, - ModelHandle::read(&model), - ); + models.insert(name_object(&model)?.extract()?, ModelHandle::read(&model)); } models } @@ -79,16 +82,16 @@ pub(crate) fn planning_context( } for (kind, source) in [(SOURCE_REFERENCE_KIND, true), (SEED_REFERENCE_KIND, false)] { let resources: Bound<'_, PyAny> = if source { - project_reads::sources(project)? + sources(project)? } else { - project_reads::seeds(project)? + seeds(project)? }; for resource in resources.try_iter()? { let resource = resource?; let entry: Bound<'_, PyAny> = if source { - source_reads::source_entry(&resource)? + source_entry(&resource)? } else { - seed_reads::schema_entry(&resource)? + schema_entry(&resource)? }; let columns: Vec = columns(&native_field(&entry, "columns")?, false)?; let authoritative = if source { @@ -97,7 +100,7 @@ pub(crate) fn planning_context( !columns.is_empty() }; relations.insert( - (kind.to_owned(), resource_reads::name(&resource)?.extract()?), + (kind.to_owned(), name(&resource)?.extract()?), FixtureRelation { columns, authoritative, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs index 2d15fabc57..86584191bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs @@ -4,11 +4,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; -use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; -use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; +use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use crate::compiled_views::main::model_reads::schema_entry::schema_entry; +use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::planner_strategy::_helpers::fixture_values::columns; +use crate::planner_strategy::_helpers::fixture_values::folded_name; +use crate::planner_strategy::constants::ENFORCED_CONTRACT; +use crate::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; @@ -19,7 +21,7 @@ const CONTRACT_KEY: &str = "contract"; /// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { let inferred: Option> = inferred_rows(model)?; - let schema = model_reads::schema_entry(model)?; + let schema = schema_entry(model)?; let declared: Vec = if schema.is_none() { Vec::new() } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs index 0f5f8af55b..5f8374c2dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs @@ -4,10 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; -use crate::bindings::_helpers::planner_strategy::fixture_sql::complete_empty_fixture_sql; -use crate::bindings::_helpers::planner_strategy::fixture_values::{diagnostic, label}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::_helpers::fixture_sql::complete_empty_fixture_sql; +use crate::planner_strategy::_helpers::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixture_values::label; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs index 8ac925a58a..256bffbe43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs @@ -13,14 +13,16 @@ use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::{ - LineageFactRow, lineage_facts, proven_inputs, star_flags, -}; -use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; -use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::compiled_views::main::model_reads::dep_rows::dep_rows; +use crate::compiled_views::main::model_reads::lineage_facts::lineage_facts; +use crate::compiled_views::main::model_reads::proven_inputs::proven_inputs; +use crate::compiled_views::main::model_reads::query_sql::query_sql; +use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::compiled_views::models::LineageFactRow; +use crate::planner_strategy::_helpers::fixture_context::work_counts; +use crate::planner_strategy::constants::HIGH_CONFIDENCE; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; @@ -62,7 +64,9 @@ pub(crate) fn required_fixture_columns( continue; }; let mut references: Vec = Vec::new(); - for reference in model_reads::references(&model)?.try_iter()? { + for reference in + crate::compiled_views::main::model_reads::references::references(&model)?.try_iter()? + { let reference = reference?; references.push(FixtureReadReference { kind: native_field(&reference, "ref_kind")?.extract()?, @@ -74,7 +78,7 @@ pub(crate) fn required_fixture_columns( models.push(FixtureReadModel { name: name.clone(), key: FixtureReadKey { - sql: model_reads::query_sql(&model)?.extract()?, + sql: query_sql(&model)?.extract()?, references, dialect: dialect.clone(), }, @@ -132,7 +136,7 @@ fn model_stars( return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for key in model_reads::dep_rows(model)? { + for key in dep_rows(model)? { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } @@ -159,7 +163,7 @@ fn model_stars( } else { None }; - let sql: String = model_reads::query_sql(model)?.extract()?; + let sql: String = query_sql(model)?.extract()?; model .py() .compiler_detach(|| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs index 3f25962c17..df94f2d543 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs @@ -9,10 +9,12 @@ use sqlbuild_analysis::planner_fixtures::main::{ unknown_columns::unknown_columns, }; -use crate::bindings::_helpers::planner_strategy::fixture_preparation::PreparedFixtures; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, diagnostic, label}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::_helpers::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixture_values::columns; +use crate::planner_strategy::_helpers::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixture_values::label; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; /// Fixtures whose inferred columns do not already match an authoritative relation shape. pub(crate) fn fixture_keys_requiring_analysis( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs index a3b237a8d3..13cdb549a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureSqlInput, SuppliedProjection, }; -use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; +use crate::planner_strategy::models::FixtureRelation; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs index 17109ed3ce..beb985c529 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs @@ -9,14 +9,13 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::compiled_views::model_reads::{LineageFactRow, lineage_facts}; -use crate::bindings::_helpers::planner_strategy::constants::{ - DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, -}; -use crate::bindings::_helpers::planner_strategy::models::{ - ExpectedTypes, RelationFixturePlanningContext, -}; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::compiled_views::main::model_reads::lineage_facts::lineage_facts; +use crate::compiled_views::models::LineageFactRow; +use crate::planner_strategy::constants::DIRECT_TRANSFORM; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::models::ExpectedTypes; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; /// Each chain model's known column types with direct lineage types propagated from its inputs. pub(crate) fn expected_fixture_types<'a>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_values.rs diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..c655c1598f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod fixture_completion; +pub(crate) mod fixture_context; +pub(crate) mod fixture_metadata; +pub(crate) mod fixture_preparation; +pub(crate) mod fixture_reads; +pub(crate) mod fixture_selection; +pub(crate) mod fixture_sql; +pub(crate) mod fixture_types; +pub(crate) mod fixture_values; +pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs similarity index 68% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs index 2ae899fbc1..e797065aaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs @@ -1,7 +1,4 @@ //! Fixture planning key and input shapes. -/// `(resource type value, name)` of one fixture relation. -pub(crate) type FixtureKey = (String, String); - /// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python-project/src/planner_strategy/constants.rs b/crates/sqlbuild-python-project/src/planner_strategy/constants.rs new file mode 100644 index 0000000000..10eeaf5f6f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/constants.rs @@ -0,0 +1,49 @@ +//! Public constants of planner strategy. + +pub const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; + +pub const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; + +pub const MODEL_ATTACHMENT: &str = "model"; + +pub const SOURCE_ATTACHMENT: &str = "source"; + +pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; + +pub const MICROBATCH_MODE: &str = "microbatch"; + +pub const CUSTOM_MATERIALIZATION: &str = "custom"; + +pub const ENFORCED_CONTRACT: &str = "enforced"; + +pub const TIMESTAMP_CURSOR: &str = "timestamp"; + +pub const MODEL_REFERENCE: &str = "ref"; + +pub const SEED_REFERENCE: &str = "seed"; + +pub const SOURCE_REFERENCE: &str = "source"; + +pub const DDL_SEPARATOR: &str = ";\n\n"; + +/// The lineage confidence whose reads fixtures must provide. +pub(crate) const HIGH_CONFIDENCE: &str = "high"; + +/// The lineage transform kind that passes an input column's type through. +pub(crate) const DIRECT_TRANSFORM: &str = "direct"; + +/// Resource type values keying fixture relations. +pub const MODEL_REFERENCE_KIND: &str = "model"; + +pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; + +pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; + +/// The SQL analysis dialect whose adapter lays tables out by the physical storage keys. +pub const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; + +/// The error code for physical storage keys on an adapter without them. +pub const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; + +/// Plan action values that create the model's table anew or leave it untouched. +pub const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs new file mode 100644 index 0000000000..490721a438 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs @@ -0,0 +1,17 @@ +//! An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. + +use crate::planner_strategy::_helpers::fixture_sql; +use crate::planner_strategy::models::FixtureRelation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. +#[inline] +pub fn complete_empty_fixture_sql( + sql: &str, + relation: Option<&FixtureRelation>, + adapter: &Bound<'_, PyAny>, +) -> PyResult<(String, Option)> { + fixture_sql::complete_empty_fixture_sql(sql, relation, adapter) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs new file mode 100644 index 0000000000..3afa4aa192 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs @@ -0,0 +1,18 @@ +//! Complete the fixture groups of one chain against the planning context. + +use crate::planner_strategy::_helpers::fixture_completion; +use crate::planner_strategy::_helpers::fixture_completion::FixtureCompletion; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Complete the fixture groups of one chain against the planning context. +#[inline] +pub fn complete_fixtures<'py, 'a>( + context: &'a RelationFixturePlanningContext, + adapter: &Bound<'py, PyAny>, + (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), +) -> PyResult> { + fixture_completion::complete_fixtures(context, adapter, (ordered_model_names, fixture_groups)) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs new file mode 100644 index 0000000000..bfb98df764 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs @@ -0,0 +1,17 @@ +//! Fixture SQL with its untyped null columns cast to the expected types. + +use crate::planner_strategy::_helpers::fixture_sql; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Fixture SQL with its untyped null columns cast to the expected types. +#[inline] +pub fn complete_typed_null_columns( + sql: &str, + inference: &Bound<'_, PyAny>, + expected_types: &[(String, String)], + adapter: &Bound<'_, PyAny>, +) -> PyResult { + fixture_sql::complete_typed_null_columns(sql, inference, expected_types, adapter) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs new file mode 100644 index 0000000000..bdac8debc7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs @@ -0,0 +1,11 @@ +//! Count one batch resolution of SQL-test model chains. + +use crate::planner_strategy::_helpers::fixture_context; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Count one batch resolution of SQL-test model chains. +#[inline] +pub fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { + fixture_context::count_chain_resolution(py) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..5cacd8332e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs @@ -0,0 +1,8 @@ +pub mod complete_empty_fixture_sql; +pub mod complete_fixtures; +pub mod complete_typed_null_columns; +pub mod count_chain_resolution; +pub mod planning_context; +pub mod register_fixture_completion; +pub mod register_fixture_context; +pub mod register_fixture_reads; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs new file mode 100644 index 0000000000..f3227f4772 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs @@ -0,0 +1,17 @@ +//! The planning context of a project, over its models or the given name-to-model map. + +use crate::planner_strategy::_helpers::fixture_context; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The planning context of a project, over its models or the given name-to-model map. +#[inline] +pub fn planning_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { + fixture_context::planning_context(project, models_by_name) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs new file mode 100644 index 0000000000..5b6fbb8c7d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs @@ -0,0 +1,12 @@ +//! Register the fixture completion functions and classes. + +use crate::planner_strategy::_helpers::fixture_completion; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fixture completion functions and classes. +#[inline] +pub fn register_fixture_completion(module: &Bound<'_, PyModule>) -> PyResult<()> { + fixture_completion::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs new file mode 100644 index 0000000000..1142f6ae24 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs @@ -0,0 +1,12 @@ +//! Register the fixture context functions and classes. + +use crate::planner_strategy::_helpers::fixture_context; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fixture context functions and classes. +#[inline] +pub fn register_fixture_context(module: &Bound<'_, PyModule>) -> PyResult<()> { + fixture_context::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs new file mode 100644 index 0000000000..01fac20a6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs @@ -0,0 +1,12 @@ +//! Register the fixture reads functions and classes. + +use crate::planner_strategy::_helpers::fixture_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fixture reads functions and classes. +#[inline] +pub fn register_fixture_reads(module: &Bound<'_, PyModule>) -> PyResult<()> { + fixture_reads::register(module) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..d4dfb24511 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/mod.rs @@ -0,0 +1,7 @@ +//! Planner strategy for fixtures: context, metadata, preparation, reads, selection, SQL and values. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/models.rs b/crates/sqlbuild-python-project/src/planner_strategy/models.rs new file mode 100644 index 0000000000..ac462fd145 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/models.rs @@ -0,0 +1,111 @@ +//! Public records of planner strategy. + +use crate::graph::models::NativeProjectGraph; +use crate::native_host::models::ModelHandle; +use crate::native_host::models::ModelLookup; +use crate::planner_strategy::types::FixtureKey; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::pyclass; +use pyo3::types::PyAny; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; +use sqlbuild_analysis::planner_fixtures::models::FixtureTypeProjection; +use sqlbuild_analysis::required_columns::models::ColumnRead; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; + +/// Per-extension cache replacing the Python fixture resolver's LRU. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureReadCacheHost { + pub cache: FixtureReadCache, + /// Fixture planning contexts built since the last reset. + pub context_builds: u64, + /// SQL-test model chain batches resolved since the last reset. + pub chain_resolutions: u64, +} + +/// One relation's known fixture columns and whether they form an authoritative shape. +#[derive(Clone, Debug)] +pub struct FixtureRelation { + pub columns: Vec, + pub authoritative: bool, +} + +/// One model's high-confidence lineage reads and direct type projections. +#[derive(Debug)] +pub(crate) struct ModelFixtureFacts { + pub reads: Vec, + pub projections: Vec, +} + +/// Project-wide fixture metadata shared across test and scenario planning. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct RelationFixturePlanningContext { + pub models_by_name: HashMap, + pub(crate) relations: HashMap, + pub authoritative_columns: HashMap>, + pub expected_types: HashMap>, + pub fixture_inferences: Py, + pub(crate) model_facts: Mutex>>, +} + +/// The expected column types of one completion: its chain's propagated types over the context's. +#[derive(Debug)] +pub(crate) struct ExpectedTypes<'a> { + pub overlay: Vec<(FixtureKey, Vec<(String, String)>)>, + pub base: &'a HashMap>, +} + +/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. +#[derive(Debug)] +pub struct ModelEntryDecisions<'py> { + pub model: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub change_result: Bound<'py, PyAny>, + pub action: Bound<'py, PyAny>, + pub reason: Bound<'py, PyAny>, + pub materialization_type: Bound<'py, PyAny>, + pub resolved_sql: Bound<'py, PyAny>, + pub sql_analysis_enabled: bool, + pub hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub cursor_column: Bound<'py, PyAny>, + pub cursor_input_relations: Bound<'py, PyAny>, + pub runtime_owned_cursor_bounds: bool, + pub cursor_bounds: Bound<'py, PyAny>, + pub microbatch_range: Bound<'py, PyAny>, + pub empty_cursor_inputs: Bound<'py, PyAny>, + pub waiting_on_empty_inputs: bool, + pub backfill: Bound<'py, PyAny>, + pub cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub invocation_time: Bound<'py, PyAny>, +} + +/// The project lookups that resolve one model's cursor inputs. +#[derive(Debug)] +pub struct CursorInputLookups<'py> { + pub models_by_name: ModelLookup, + pub functions_by_name: Bound<'py, PyAny>, + pub source_map: Bound<'py, PyAny>, + pub source_warehouse_columns: Bound<'py, PyAny>, + pub model_locations: Bound<'py, PyAny>, + pub seed_locations: Bound<'py, PyAny>, + pub runtime_cursor_producer_names: Bound<'py, PyAny>, +} + +/// The relations and lineage that render and schedule selected audits. +#[derive(Debug)] +pub struct AuditPlanning<'py> { + pub model_locations: Bound<'py, PyAny>, + pub seed_locations: Bound<'py, PyAny>, + pub source_map: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub model_materializations: Bound<'py, PyAny>, + pub lineage: Py, +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/types.rs b/crates/sqlbuild-python-project/src/planner_strategy/types.rs new file mode 100644 index 0000000000..f662e8ce6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of planner strategy. + +/// `(resource type value, name)` of one fixture relation. +pub type FixtureKey = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs rename to crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index 693a47d4e9..7cbc31e81a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -1,5 +1,7 @@ //! Compile's explicit-reference checks over plain project facts, run natively. +use crate::references::models::GateRow; +use crate::references::models::HookRow; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -24,8 +26,9 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; -use crate::bindings::_helpers::graph::execution_gates::model_python_hook_names; -use crate::bindings::_helpers::references::reference_facts::{ProjectFacts, project_facts}; +use crate::graph::main::execution_gates::model_python_hook_names::model_python_hook_names; +use crate::references::_helpers::reference_facts::ProjectFacts; +use crate::references::_helpers::reference_facts::project_facts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -426,23 +429,6 @@ pub(crate) fn old_name_reference_message( old_name_message(owner_label, written, method, model_name) } -/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. -pub(crate) struct GateRow { - pub(crate) audit_name: String, - pub(crate) target: ResourceKey, - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, -} - -/// One hook-read ordering edge: the hook, the gated and read resources, and its type. -pub(crate) struct HookRow { - pub(crate) hook_name: String, - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, - /// The `HookReadType` value. - pub(crate) hook_type: &'static str, -} - /// The attached-audit gate edges and hook-read edges of one project, in Python's order. pub(crate) fn native_execution_gate_edges( project: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-project/src/references/_helpers/mod.rs b/crates/sqlbuild-python-project/src/references/_helpers/mod.rs new file mode 100644 index 0000000000..2bc3b431b5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod explicit_references; +pub(crate) mod reference_facts; +pub(crate) mod relations; +pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs b/crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs rename to crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs rename to crates/sqlbuild-python-project/src/references/_helpers/relations.rs index 969f55318f..ed7fc1f63b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs @@ -6,9 +6,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::analysis::relation_names::{ - RelationRow, extract_sql_relation_names, -}; +use crate::analysis::main::extract_sql_relation_names::extract_sql_relation_names; +use crate::analysis::types::RelationRow; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs rename to crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs diff --git a/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs new file mode 100644 index 0000000000..e63e34c0ab --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs @@ -0,0 +1,18 @@ +//! Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. + +use crate::references::_helpers::explicit_references; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. +#[inline] +pub fn explicit_reference_report<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + enforce_explicit: bool, +) -> PyResult> { + explicit_references::explicit_reference_report(py, project, discovered_inputs, enforce_explicit) +} diff --git a/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs new file mode 100644 index 0000000000..8f4bb62207 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs @@ -0,0 +1,16 @@ +//! How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. + +use crate::references::_helpers::explicit_references; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. +#[inline] +pub fn hard_coded_relation_remedy( + owner_kind: &str, + r#ref: &Bound<'_, PyAny>, + upstream_loader_by_source: Option<&Bound<'_, PyAny>>, +) -> PyResult { + explicit_references::hard_coded_relation_remedy(owner_kind, r#ref, upstream_loader_by_source) +} diff --git a/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs new file mode 100644 index 0000000000..3c3d8277c5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs @@ -0,0 +1,17 @@ +//! The project relation `relation` names, resolving missing qualifiers from the defaults. + +use crate::references::_helpers::relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The project relation `relation` names, resolving missing qualifiers from the defaults. +#[inline] +pub fn match_project_relation<'py>( + index: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + default_database: Option, + default_schema: Option, +) -> PyResult>> { + relations::match_project_relation(index, relation, default_database, default_schema) +} diff --git a/crates/sqlbuild-python-project/src/references/main/mod.rs b/crates/sqlbuild-python-project/src/references/main/mod.rs new file mode 100644 index 0000000000..24512ac147 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/mod.rs @@ -0,0 +1,10 @@ +pub mod explicit_reference_report; +pub mod hard_coded_relation_remedy; +pub mod match_project_relation; +pub mod native_execution_gate_edges; +pub mod old_name_reference_message; +pub mod register_explicit_references; +pub mod register_relations; +pub mod register_source_relations; +pub mod render_source_read; +pub mod render_source_relation; diff --git a/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs new file mode 100644 index 0000000000..21f552da41 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs @@ -0,0 +1,16 @@ +//! The attached-audit gate edges and hook-read edges of one project, in Python's order. + +use crate::references::_helpers::explicit_references; +use crate::references::models::GateRow; +use crate::references::models::HookRow; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The attached-audit gate edges and hook-read edges of one project, in Python's order. +#[inline] +pub fn native_execution_gate_edges( + project: &Bound<'_, PyAny>, +) -> PyResult<(Vec, Vec)> { + explicit_references::native_execution_gate_edges(project) +} diff --git a/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs new file mode 100644 index 0000000000..196476fd04 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs @@ -0,0 +1,14 @@ +//! The P008 message for SQL naming a relation kept only as a compatibility view. + +use crate::references::_helpers::explicit_references; + +/// The P008 message for SQL naming a relation kept only as a compatibility view. +#[inline] +pub fn old_name_reference_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + explicit_references::old_name_reference_message(owner_label, written, method, model_name) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs new file mode 100644 index 0000000000..e880d11aa1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs @@ -0,0 +1,12 @@ +//! Register the explicit references functions and classes. + +use crate::references::_helpers::explicit_references; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the explicit references functions and classes. +#[inline] +pub fn register_explicit_references(module: &Bound<'_, PyModule>) -> PyResult<()> { + explicit_references::register(module) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_relations.rs new file mode 100644 index 0000000000..97bfedcc25 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_relations.rs @@ -0,0 +1,12 @@ +//! Register the relations functions and classes. + +use crate::references::_helpers::relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the relations functions and classes. +#[inline] +pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + relations::register(module) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs new file mode 100644 index 0000000000..a287a54c5a --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs @@ -0,0 +1,12 @@ +//! Register the source relations functions and classes. + +use crate::references::_helpers::source_relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the source relations functions and classes. +#[inline] +pub fn register_source_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + source_relations::register(module) +} diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_read.rs b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs new file mode 100644 index 0000000000..166eebbde5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs @@ -0,0 +1,15 @@ +//! A source as the table factor that reads it, deduplicated with `FINAL` when `final`. + +use crate::references::_helpers::source_relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. +#[inline] +pub fn render_source_read<'py>( + entry: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult { + source_relations::render_source_read(entry, adapter) +} diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs new file mode 100644 index 0000000000..98103c0c3f --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs @@ -0,0 +1,15 @@ +//! Render a source as a SQL table factor, through the adapter when one is given. + +use crate::references::_helpers::source_relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Render a source as a SQL table factor, through the adapter when one is given. +#[inline] +pub fn render_source_relation<'py>( + entry: &Bound<'py, PyAny>, + adapter: Option<&Bound<'py, PyAny>>, +) -> PyResult { + source_relations::render_source_relation(entry, adapter) +} diff --git a/crates/sqlbuild-python-project/src/references/mod.rs b/crates/sqlbuild-python-project/src/references/mod.rs new file mode 100644 index 0000000000..f89201734f --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/mod.rs @@ -0,0 +1,5 @@ +//! References between resources: explicit references, relations, source relations and their facts. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/references/models.rs b/crates/sqlbuild-python-project/src/references/models.rs new file mode 100644 index 0000000000..59b61e152b --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/models.rs @@ -0,0 +1,22 @@ +//! Public records of references. + +use sqlbuild_analysis::explicit_references::types::ResourceKey; + +/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. +#[derive(Debug)] +pub struct GateRow { + pub audit_name: String, + pub target: ResourceKey, + pub gated: ResourceKey, + pub read: ResourceKey, +} + +/// One hook-read ordering edge: the hook, the gated and read resources, and its type. +#[derive(Debug)] +pub struct HookRow { + pub hook_name: String, + pub gated: ResourceKey, + pub read: ResourceKey, + /// The `HookReadType` value. + pub hook_type: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs rename to crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs diff --git a/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs new file mode 100644 index 0000000000..79e24dc7c0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod fitting; diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs new file mode 100644 index 0000000000..0373160453 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs @@ -0,0 +1,16 @@ +//! `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. + +use crate::relation_naming::_helpers::fitting; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. +#[inline] +pub fn fitted_logical_name( + py: Python<'_>, + names: (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> PyResult { + fitting::fitted_logical_name(py, names, identifier_limit, artifact_label) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs new file mode 100644 index 0000000000..4c344f8415 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs @@ -0,0 +1,3 @@ +pub mod fitted_logical_name; +pub mod register_fitting; +pub mod shared_error; diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs new file mode 100644 index 0000000000..f029d8a2a7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs @@ -0,0 +1,12 @@ +//! Register the fitting functions and classes. + +use crate::relation_naming::_helpers::fitting; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fitting functions and classes. +#[inline] +pub fn register_fitting(module: &Bound<'_, PyModule>) -> PyResult<()> { + fitting::register(module) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs new file mode 100644 index 0000000000..a2c7f5b483 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs @@ -0,0 +1,11 @@ +//! Shared error. + +use crate::relation_naming::_helpers::fitting; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// Shared error. +#[inline] +pub fn shared_error(py: Python<'_>, message: String) -> PyErr { + fitting::shared_error(py, message) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/mod.rs new file mode 100644 index 0000000000..fdc0964b53 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/mod.rs @@ -0,0 +1,4 @@ +//! Fitting relation names to warehouse limits. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs index feeeae8e32..e06d5b178a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs @@ -6,9 +6,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; -use crate::bindings::_helpers::semantic_shapes::delta_shapes::semantic_reasons; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; +use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::semantic_shapes::_helpers::delta_shapes::semantic_reasons; +use crate::semantic_shapes::_helpers::shapes::project_shapes; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::compile_context::models::ModelCoverage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/delta_shapes.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/delta_shapes.rs diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs new file mode 100644 index 0000000000..5131e6afd5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod coverage; +pub(crate) mod declarations; +pub(crate) mod delta_shapes; +pub(crate) mod shapes; +pub(crate) mod source_bindings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs index 4063086085..dbe6672176 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs @@ -1,22 +1,25 @@ //! Closed relation shapes of a compiled project, as binding and semantic checks read them. +use crate::semantic_shapes::constants::VALID_FROM_DEFAULT; +use crate::semantic_shapes::constants::VALID_TO_DEFAULT; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, -}; -use crate::bindings::_helpers::analysis::normalization::normalize_analysis_sql; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_expression_source_shapes; -use crate::bindings::_helpers::compiled_project::rows::ReferenceTuple; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, NativeCompiledModel, -}; -use crate::bindings::_helpers::compiled_views::model_reads::{ - declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, -}; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; +use crate::analysis::models::BindingCatalog; +use crate::analysis::models::CatalogKey; +use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; +use crate::compiled_project::models::ReferenceTuple; +use crate::compiled_views::main::model_reads::declared_rows::declared_rows; +use crate::compiled_views::main::model_reads::dynamic_columns_declared::dynamic_columns_declared; +use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::compiled_views::main::model_reads::unchecked_columns::unchecked_columns; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::types::DeclaredRow; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -28,10 +31,7 @@ use sqlbuild_python_base::records::types::RecordId; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; -/// Snapshot validity start column when the model names none. -pub(crate) const VALID_FROM_DEFAULT: &str = "valid_from"; -/// Snapshot validity end column when the model names none. -pub(crate) const VALID_TO_DEFAULT: &str = "valid_to"; + const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const UDF_KIND: &str = "udf"; const TABLE_FUNCTION_KIND: &str = "table_fn"; @@ -327,9 +327,7 @@ fn model_shapes<'py>( let mut pending: Vec> = native_field(project, "models")? .try_iter()? .collect::>()?; - crate::bindings::_helpers::semantic_shapes::delta_shapes::inject_model_shapes( - &pending, shapes, - )?; + crate::semantic_shapes::_helpers::delta_shapes::inject_model_shapes(&pending, shapes)?; while !pending.is_empty() { let mut remaining: Vec> = Vec::new(); for model in &pending { @@ -375,7 +373,7 @@ fn model_shapes<'py>( } pending = remaining; } - crate::bindings::_helpers::semantic_shapes::delta_shapes::record_model_shapes(project, shapes) + crate::semantic_shapes::_helpers::delta_shapes::record_model_shapes(project, shapes) } fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 0cc5d322ae..1aeccb09a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -6,19 +6,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::analysis::binding_catalog::{ - self, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, schema_validations, -}; -use crate::bindings::_helpers::analysis::schema_request::{ - SchemaRequestParts, schema_validation_request, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; -use crate::bindings::_helpers::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; -use crate::bindings::_helpers::semantic_shapes::shapes::{ - model_inputs, project_shapes, published_shape, -}; -use crate::bindings::models::BindingPositions; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::main::schema_validation_request::schema_validation_request; +use crate::analysis::main::schema_validations::schema_validations; +use crate::analysis::models::BindingPositions; +use crate::analysis::models::SchemaRequestParts; +use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use crate::semantic_shapes::_helpers::declarations::declared_types; +use crate::semantic_shapes::_helpers::declarations::function_names; +use crate::semantic_shapes::_helpers::shapes::model_inputs; +use crate::semantic_shapes::_helpers::shapes::project_shapes; +use crate::semantic_shapes::_helpers::shapes::published_shape; +use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; @@ -281,7 +281,7 @@ fn validated_diagnostics<'py>( let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let ignores: bool = binding_catalog::ignores_quoted_case( + let ignores: bool = crate::analysis::main::ignores_quoted_case::ignores_quoted_case( &native_field(project, "effective_connection")?, dialect, )?; @@ -474,7 +474,7 @@ fn is_folded_identifier(name: &str) -> bool { } fn ignores_quoted_case<'py>(project: &Object<'py>, dialect: &Object<'py>) -> PyResult> { - let ignores: bool = binding_catalog::ignores_quoted_case( + let ignores: bool = crate::analysis::main::ignores_quoted_case::ignores_quoted_case( &native_field(project, "effective_connection")?, dialect, )?; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs b/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs new file mode 100644 index 0000000000..98eac192ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs @@ -0,0 +1,10 @@ +//! Public constants of semantic shapes. + +/// Model queries analysed again against inspected source columns. +pub(crate) const SOURCE_REBINDING_ANALYSES: &str = "source_rebinding_analyses"; + +/// Snapshot validity start column when the model names none. +pub const VALID_FROM_DEFAULT: &str = "valid_from"; + +/// Snapshot validity end column when the model names none. +pub const VALID_TO_DEFAULT: &str = "valid_to"; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs new file mode 100644 index 0000000000..115a374c2e --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs @@ -0,0 +1,15 @@ +//! Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. + +use crate::semantic_shapes::_helpers::declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. +#[inline] +pub fn declared_types( + functions: &Bound<'_, PyAny>, + column_types: &Bound<'_, PyAny>, +) -> PyResult> { + declarations::declared_types(functions, column_types) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs new file mode 100644 index 0000000000..4b3c5a012f --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs @@ -0,0 +1,12 @@ +//! Each declared function's name and its UDF and table-function analysis names, sorted. + +use crate::semantic_shapes::_helpers::declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each declared function's name and its UDF and table-function analysis names, sorted. +#[inline] +pub fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { + declarations::function_names(functions) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs new file mode 100644 index 0000000000..f55fb1fa45 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs @@ -0,0 +1,9 @@ +pub mod declared_types; +pub mod function_names; +pub mod project_shapes; +pub mod register_coverage; +pub mod register_shapes; +pub mod register_source_bindings; +pub mod semantic_coverage; +pub mod snapshot_validity_columns; +pub mod source_binding_diagnostics; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs new file mode 100644 index 0000000000..f854a3a172 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs @@ -0,0 +1,16 @@ +//! Closed shapes by relation name; absence means open, never an empty physical table. + +use crate::semantic_shapes::_helpers::shapes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Closed shapes by relation name; absence means open, never an empty physical table. +#[inline] +pub fn project_shapes<'py>( + project: &Bound<'py, PyAny>, + profile: Option>, +) -> PyResult> { + shapes::project_shapes(project, profile) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs new file mode 100644 index 0000000000..71208889db --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs @@ -0,0 +1,12 @@ +//! Register the coverage functions and classes. + +use crate::semantic_shapes::_helpers::coverage; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the coverage functions and classes. +#[inline] +pub fn register_coverage(module: &Bound<'_, PyModule>) -> PyResult<()> { + coverage::register(module) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs new file mode 100644 index 0000000000..eb6a14d4a0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs @@ -0,0 +1,12 @@ +//! Register the shapes functions and classes. + +use crate::semantic_shapes::_helpers::shapes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the shapes functions and classes. +#[inline] +pub fn register_shapes(module: &Bound<'_, PyModule>) -> PyResult<()> { + shapes::register(module) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs new file mode 100644 index 0000000000..18d09fd054 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs @@ -0,0 +1,12 @@ +//! Register the source bindings functions and classes. + +use crate::semantic_shapes::_helpers::source_bindings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the source bindings functions and classes. +#[inline] +pub fn register_source_bindings(module: &Bound<'_, PyModule>) -> PyResult<()> { + source_bindings::register(module) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs new file mode 100644 index 0000000000..73b112fa0e --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs @@ -0,0 +1,13 @@ +//! Partial-check reasons by model name; empty when SQL analysis is off. + +use crate::semantic_shapes::_helpers::coverage; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Partial-check reasons by model name; empty when SQL analysis is off. +#[inline] +pub fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + coverage::semantic_coverage(project) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs new file mode 100644 index 0000000000..bc4d7f1bd5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs @@ -0,0 +1,12 @@ +//! A snapshot model's validity column names, which the snapshot adds to its query's columns. + +use crate::semantic_shapes::_helpers::shapes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A snapshot model's validity column names, which the snapshot adds to its query's columns. +#[inline] +pub fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { + shapes::snapshot_validity_columns(model) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs new file mode 100644 index 0000000000..929ce461cb --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs @@ -0,0 +1,15 @@ +//! `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. + +use crate::semantic_shapes::_helpers::source_bindings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. +#[inline] +pub fn source_binding_diagnostics<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + source_bindings::source_binding_diagnostics(request) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs new file mode 100644 index 0000000000..227cc415d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs @@ -0,0 +1,5 @@ +//! Semantic shapes: declarations, coverage, delta shapes and source bindings. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs index b9ab6050b2..4604f2f372 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs @@ -8,12 +8,17 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::sql_test_glue::constants::{ - CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, - DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, - TIMESTAMP_CURSOR_TYPE, -}; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::model_reads::query_sql::query_sql; +use crate::sql_test_glue::constants::CURSOR_ALGEBRA_EXCEPTIONS; +use crate::sql_test_glue::constants::CURSOR_END_KEY; +use crate::sql_test_glue::constants::CURSOR_START_KEY; +use crate::sql_test_glue::constants::DEFAULT_INTEGER_END; +use crate::sql_test_glue::constants::DEFAULT_INTEGER_START; +use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; +use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; +use crate::sql_test_glue::constants::INTEGER_CURSOR_TYPE; +use crate::sql_test_glue::constants::TIMESTAMP_CURSOR_TYPE; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -44,19 +49,19 @@ struct BoundRequest<'a, 'py> { } /// Whether SQL calls `__cursor_start()` or `__cursor_end()`. -pub(super) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { +pub(crate) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { Ok(resolve_cursor_intrinsics(py, sql, None, None)?.1) } /// Whether a SQL test header declares its own cursor window. -pub(super) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { let block = native_field(test, "test_block")?; Ok(!native_field(&block, "cursor_start")?.is_none() || !native_field(&block, "cursor_end")?.is_none()) } /// SQL with intrinsics rendered from the test's declared window, or the wide default window. -pub(super) fn render_test_cursor_intrinsics<'py>( +pub(crate) fn render_test_cursor_intrinsics<'py>( sql: Bound<'py, PyAny>, model: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, @@ -85,14 +90,14 @@ pub(super) fn render_test_cursor_intrinsics<'py>( } /// Chain models using intrinsics, after validating the test's declared window against each. -pub(super) fn declared_window_cursor_models<'py>( +pub(crate) fn declared_window_cursor_models<'py>( test: &Bound<'py, PyAny>, chain_models: Vec>, ) -> PyResult>> { let py = test.py(); let mut cursor_models: Vec> = Vec::new(); for model in chain_models { - if uses_cursor_intrinsics(py, &model_reads::query_sql(&model)?.extract::()?)? { + if uses_cursor_intrinsics(py, &query_sql(&model)?.extract::()?)? { cursor_models.push(model); } } @@ -263,7 +268,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { _ => None, }; Ok(ModelCursor { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, cursor_type, type_name, grain, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs index a394bd21d6..d32a0a8fd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs @@ -8,14 +8,13 @@ use sqlbuild_analysis::planner_fixtures::main::{ collection_mismatch::collection_mismatch, fixture_line::fixture_line, }; -use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; -use crate::bindings::_helpers::planner_strategy::fixture_sql::{ - complete_empty_fixture_sql, complete_typed_null_columns, -}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -use crate::bindings::_helpers::sql_test_glue::constants::FIXTURE_PREFIXES; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::main::complete_empty_fixture_sql::complete_empty_fixture_sql; +use crate::planner_strategy::main::complete_fixtures::complete_fixtures; +use crate::planner_strategy::main::complete_typed_null_columns::complete_typed_null_columns; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; +use crate::sql_test_glue::constants::FIXTURE_PREFIXES; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -24,7 +23,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; /// Report every statically provable fixture defect or return the completed fixture SQL groups. -pub(super) fn validated_fixtures<'py>( +pub(crate) fn validated_fixtures<'py>( test: &Bound<'py, PyAny>, request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -181,7 +180,7 @@ pub(super) fn validated_fixtures<'py>( } /// Return the test file and line that defines one fixture or expected-output CTE. -pub(super) fn fixture_location( +pub(crate) fn fixture_location( test: &Bound<'_, PyAny>, resource_type: &Bound<'_, PyAny>, name: &str, diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs new file mode 100644 index 0000000000..593b57eae6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs @@ -0,0 +1,12 @@ +pub(crate) mod cursor_window; +pub(crate) mod fixture_validation; +pub(crate) mod plan_assembly; +pub(crate) mod plan_driver; +pub(crate) mod plan_entries; +pub(crate) mod plan_errors; +pub(crate) mod plan_fixtures; +pub(crate) mod plan_rendering; +pub(crate) mod plan_rows; +pub(crate) mod plan_values; +pub(crate) mod planning; +pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs index df3e85d5e2..f9808ee903 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; -use crate::bindings::_helpers::sql_test_glue::plan_driver::planned as planned_tests; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::planner_strategy::main::planning_context::planning_context; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::plan_driver::planned as planned_tests; +use crate::sql_test_glue::models::SqlTestAssembly; +use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -15,7 +16,7 @@ use sqlbuild_python_base::records::types::RecordId; #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(super) fn plan_sql_tests<'py>( +pub(crate) fn plan_sql_tests<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -52,21 +53,24 @@ pub(crate) fn assemble_sql_test_plans<'py>( ) -> PyResult> { let py = project.py(); let mut context: Option> = None; - let (planned, diagnostics) = if assembly.sql_analysis_enabled.is_truthy()? - && assembly.validate_fixtures - { - let given = assembly.fixture_planning_context.clone(); - let shared = if given.is_none() { - Bound::new(py, planning_context(project, None)?)? + let (planned, diagnostics) = + if assembly.sql_analysis_enabled.is_truthy()? && assembly.validate_fixtures { + let given = assembly.fixture_planning_context.clone(); + let shared = if given.is_none() { + Bound::new(py, planning_context(project, None)?)? + } else { + given.cast_into::()? + }; + let validated = crate::sql_test_glue::_helpers::plan_fixtures::validate( + tests, + (project, adapter), + shared.get(), + )?; + context = Some(shared); + validated } else { - given.cast_into::()? + (tests.to_vec(), Default::default()) }; - let validated = super::plan_fixtures::validate(tests, (project, adapter), shared.get())?; - context = Some(shared); - validated - } else { - (tests.to_vec(), Default::default()) - }; let plannable: Vec<_> = planned .into_iter() .enumerate() @@ -92,7 +96,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( let models = PyDict::new(py); for model in native_field(project, "models")?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } let mut results: Vec> = Vec::new(); for (index, test) in tests.iter().enumerate() { @@ -113,11 +117,13 @@ pub(crate) fn assemble_sql_test_plans<'py>( )); }; let mut errors: Vec = native_field(&plan, "error_messages")?.extract()?; - errors.extend(super::plan_entries::expected_diagnostics( - test, - &plan, - context.as_ref().map(Bound::get), - )?); + errors.extend( + crate::sql_test_glue::_helpers::plan_entries::expected_diagnostics( + test, + &plan, + context.as_ref().map(Bound::get), + )?, + ); if !errors.is_empty() { results.push(new_record_with( py, @@ -135,7 +141,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( &[ ( "entry", - super::plan_entries::entry( + crate::sql_test_glue::_helpers::plan_entries::entry( test, &plan, (project, &models), @@ -148,3 +154,10 @@ pub(crate) fn assemble_sql_test_plans<'py>( } PyTuple::new(py, results) } + +/// `plan_sql_tests` as a function of `module`. +pub(crate) fn plan_sql_tests_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_sql_tests, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs index 7cf82dd6ba..82ab4f3613 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs @@ -4,14 +4,15 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunct use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; -use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use crate::bindings::_helpers::sql_test_glue::planning::{ - ChainRequest, PlanRow, PlanningRequest, plan_compiled_sql_tests, - resolve_compiled_sql_test_chains, -}; +use crate::planner_phases::main::object::object; +use crate::planner_strategy::main::count_chain_resolution::count_chain_resolution; +use crate::sql_test_glue::_helpers::plan_values::tuple; +use crate::sql_test_glue::_helpers::planning::ChainRequest; +use crate::sql_test_glue::_helpers::planning::PlanningRequest; +use crate::sql_test_glue::_helpers::planning::plan_compiled_sql_tests; +use crate::sql_test_glue::_helpers::planning::resolve_compiled_sql_test_chains; +use crate::sql_test_glue::models::PlanRow; +use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -22,7 +23,7 @@ use sqlbuild_python_base::records::types::RecordId; impl PlanRow { /// Rendered SQL for an artifact, or the planning error shared by native and Python consumers. - pub(crate) fn rendered_sql(&self, py: Python<'_>) -> PyResult<&str> { + pub fn rendered_sql(&self, py: Python<'_>) -> PyResult<&str> { let Some(sql) = self.plan.sql.as_deref() else { return Err(PyErr::from_value( error_class(py, ErrorId::NativeSqlTestPlanning)? @@ -33,7 +34,7 @@ impl PlanRow { } } -pub(super) fn chains<'py>( +pub(crate) fn chains<'py>( project: &Bound<'py, PyAny>, tests: &Bound<'py, PyAny>, ) -> PyResult> { @@ -55,7 +56,9 @@ pub(super) fn chains<'py>( let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, tests)?); + return Err(crate::sql_test_glue::_helpers::plan_errors::planning_error( + py, error, tests, + )?); } Err(error) => return Err(error), }; @@ -89,9 +92,10 @@ pub(crate) fn planned_rows<'py>( let start: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; - let model_sql = super::plan_rendering::models(project, adapter)?; - let functions = super::plan_rendering::functions(project, adapter)?; - let overrides = super::plan_rendering::overrides(project, tests, adapter)?; + let model_sql = crate::sql_test_glue::_helpers::plan_rendering::models(project, adapter)?; + let functions = crate::sql_test_glue::_helpers::plan_rendering::functions(project, adapter)?; + let overrides = + crate::sql_test_glue::_helpers::plan_rendering::overrides(project, tests, adapter)?; let request = PlanningRequest { models: native_field(project, "models")?.extract()?, rendered_model_sql: model_sql.extract()?, @@ -122,7 +126,9 @@ pub(crate) fn planned_rows<'py>( let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, tests)?); + return Err(crate::sql_test_glue::_helpers::plan_errors::planning_error( + py, error, tests, + )?); } Err(error) => return Err(error), }; @@ -160,7 +166,7 @@ pub(crate) fn planned<'py>( let rows = planned_rows((project, adapter), tests, options)?; let mut plans: Vec> = Vec::new(); for row in rows { - let plan = super::plan_rows::plan(py, row)?; + let plan = crate::sql_test_glue::_helpers::plan_rows::plan(py, row)?; if options.artifact { let sql = native_field(&plan, "sql")?; plans.push(new_record_with( @@ -181,7 +187,7 @@ pub(crate) fn planned<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(super) fn plan_sql_tests_natively<'py>( +pub(crate) fn plan_sql_tests_natively<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -250,7 +256,7 @@ pub(crate) fn sql_test_plan_error_messages<'py>( let warning = warning?; let message: String = native_field(&warning, "message")?.extract()?; if native_field(&native_field(&warning, "severity")?, "value")?.extract::()? - == super::constants::ERROR_SEVERITY + == crate::sql_test_glue::constants::ERROR_SEVERITY && seen.insert(message.clone()) { messages.push(message); @@ -274,3 +280,31 @@ pub(crate) fn sql_test_model_chain_names_by_key<'py>( } Ok(by_key) } + +/// `plan_compiled_sql_test_artifacts` as a function of `module`. +pub(crate) fn plan_compiled_sql_test_artifacts_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_compiled_sql_test_artifacts, module) +} + +/// `plan_sql_tests_natively` as a function of `module`. +pub(crate) fn plan_sql_tests_natively_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_sql_tests_natively, module) +} + +/// `sql_test_model_chain_names_by_key` as a function of `module`. +pub(crate) fn sql_test_model_chain_names_by_key_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(sql_test_model_chain_names_by_key, module) +} + +/// `sql_test_plan_error_messages` as a function of `module`. +pub(crate) fn sql_test_plan_error_messages_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(sql_test_plan_error_messages, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs index 01f3019e20..d4235c81c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs @@ -6,10 +6,11 @@ use sqlbuild_model_config::execution_limits::constants::TIMEOUT_KEY as TIMEOUT_H use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use sqlbuild_model_config::execution_limits::models::ExecutionLimit; -use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; -use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::plan_values::model_payload; +use crate::sql_test_glue::constants::TABLE_FN; +use crate::sql_test_glue::constants::UDF; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -91,7 +92,7 @@ fn function_deps<'py>( Ok(PyTuple::new(py, deps.keys().iter())?.into_any()) } -pub(super) fn entry<'py>( +pub(crate) fn entry<'py>( test: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), @@ -154,7 +155,7 @@ pub(super) fn entry<'py>( new_record(py, RecordId::SqlTestPlanEntry, (), Some(&fields)) } -pub(super) fn expected_diagnostics( +pub(crate) fn expected_diagnostics( test: &Bound<'_, PyAny>, plan: &Bound<'_, PyAny>, context: Option<&RelationFixturePlanningContext>, @@ -186,7 +187,7 @@ pub(super) fn expected_diagnostics( else { continue; }; - let location = super::fixture_validation::fixture_location( + let location = crate::sql_test_glue::_helpers::fixture_validation::fixture_location( test, &native_field(&resource_types, "SQL_TEST")?, &name.extract::()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs index ceda5bea27..8ab51cb46b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs @@ -1,6 +1,6 @@ //! Project native planning failures into existing located diagnostic exceptions. -use crate::bindings::_helpers::sql_test_glue::plan_values::string; +use crate::sql_test_glue::_helpers::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; @@ -35,7 +35,7 @@ fn cte_call_location<'py>( new_record(py, RecordId::SourceLocation, (), Some(&arguments)) } -pub(super) fn planning_error( +pub(crate) fn planning_error( py: Python<'_>, error: PyErr, tests: &Bound<'_, PyAny>, @@ -73,7 +73,10 @@ pub(super) fn planning_error( kwargs.set_item("location", cte_call_location(py, &test, (&cte, &call))?)?; ErrorId::SqlTestReference } else { - kwargs.set_item("code", super::constants::SQL_TEST_HELPER_REFERENCE_CODE)?; + kwargs.set_item( + "code", + crate::sql_test_glue::constants::SQL_TEST_HELPER_REFERENCE_CODE, + )?; ErrorId::CompileInput }; return Ok(PyErr::from_value( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs index 4e5953ee77..42a6a51a27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs @@ -3,16 +3,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; -use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; -use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::fixture_validation::validated_fixtures; +use crate::sql_test_glue::_helpers::plan_driver::resolve_sql_test_model_chains; +use crate::sql_test_glue::_helpers::plan_values::model_payload; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_front::sql_test_glue::main::prefixed_ctes::prefixed_ctes; -pub(super) fn validated_test<'py>( +pub(crate) fn validated_test<'py>( test: &Bound<'py, PyAny>, chain: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -76,11 +76,11 @@ pub(super) fn validated_test<'py>( replaced_keywords(test, &kwargs).map(Ok) } -pub(super) fn validate<'py>( +pub(crate) fn validate<'py>( tests: &[Bound<'py, PyAny>], inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), context: &RelationFixturePlanningContext, -) -> PyResult> { +) -> PyResult> { let py = inputs.0.py(); let mut model_tests: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs index 4aad8b40c1..2b89f128b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs @@ -1,26 +1,28 @@ //! Adapter rendering and declared cursor-window boundaries for native test planning. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::model_reads::query_sql::query_sql; +use crate::sql_test_glue::_helpers::plan_values::model_payload; +use crate::sql_test_glue::_helpers::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::bindings::_helpers::sql_test_glue::constants::SENTINEL; -use crate::bindings::_helpers::sql_test_glue::cursor_window::{ - declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, -}; +use crate::sql_test_glue::_helpers::cursor_window::declared_window_cursor_models; +use crate::sql_test_glue::_helpers::cursor_window::declares_cursor_window; +use crate::sql_test_glue::_helpers::cursor_window::render_test_cursor_intrinsics; +use crate::sql_test_glue::constants::SENTINEL; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -pub(super) fn mentions(sql: &str) -> bool { +pub(crate) fn mentions(sql: &str) -> bool { ["__cursor_start", "__cursor_end"] .iter() .any(|name| sql.contains(name)) } -pub(super) fn models<'py>( +pub(crate) fn models<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, ) -> PyResult> { @@ -28,7 +30,7 @@ pub(super) fn models<'py>( let result = PyDict::new(py); for (index, model) in native_field(project, "models")?.try_iter()?.enumerate() { let model = model?; - let sql = model_reads::query_sql(&model)?; + let sql = query_sql(&model)?; if mentions(&sql.extract::()?) { result.set_item( index, @@ -39,7 +41,7 @@ pub(super) fn models<'py>( Ok(result) } -pub(super) fn overrides<'py>( +pub(crate) fn overrides<'py>( project: &Bound<'py, PyAny>, tests: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, @@ -48,7 +50,7 @@ pub(super) fn overrides<'py>( let models = PyDict::new(py); for model in native_field(project, "models")?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } let tests: Vec> = tests.extract()?; let mut indexes: Vec = Vec::new(); @@ -62,8 +64,11 @@ pub(super) fn overrides<'py>( let chains = if indexes.is_empty() { Vec::new() } else { - super::plan_driver::chains(project, &PyTuple::new(py, windowed)?.into_any())? - .extract::>>()? + crate::sql_test_glue::_helpers::plan_driver::chains( + project, + &PyTuple::new(py, windowed)?.into_any(), + )? + .extract::>>()? }; if indexes.len() != chains.len() { return Err(pyo3::exceptions::PyValueError::new_err( @@ -116,12 +121,12 @@ pub(super) fn overrides<'py>( } } for model in declared_window_cursor_models(test, chain_models)? { - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; if !rendered.contains(&name)? { rendered.set_item( name, render_test_cursor_intrinsics( - model_reads::query_sql(&model)?, + query_sql(&model)?, &model, adapter, Some(test), @@ -135,7 +140,7 @@ pub(super) fn overrides<'py>( Ok(result) } -pub(super) fn functions<'py>( +pub(crate) fn functions<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rows.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rows.rs index 2e521f1627..bc8ac59e81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rows.rs @@ -1,6 +1,6 @@ //! Materialize native plan rows as the existing executor-facing dataclasses. -use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; +use crate::sql_test_glue::models::PlanRow; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; @@ -64,7 +64,7 @@ fn assertion_step<'py>(py: Python<'py>, step: SqlTestAssertionStep) -> PyResult< ) } -pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { +pub(crate) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { let PlanRow { plan: SqlTestPlan { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs index fad31f5a37..0b09a2b44b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs @@ -8,17 +8,17 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -pub(super) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { native_field(test, "payload")?.is_instance(&record_class( test.py(), RecordId::CompiledModelSqlTestPayload, )?) } -pub(super) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { imported(value.py(), "builtins", "tuple")?.call1((value,)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs index 45a0cfcced..4f9fa1f2af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs @@ -1,5 +1,6 @@ //! Plan compiled SQL tests from the compiled project's own objects, without a JSON request. +use crate::sql_test_glue::models::PlanRow; use sqlbuild_python_base::positional_record; use std::collections::{BTreeMap, HashMap}; @@ -12,7 +13,8 @@ use sqlbuild_analysis::compiler::models::{ SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; -use crate::bindings::_helpers::compiled_views::model_reads; +use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::model_reads::query_sql::query_sql; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; @@ -21,12 +23,6 @@ use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -/// A native SQL-test plan with its distinct error messages. -pub(crate) struct PlanRow { - pub(crate) plan: SqlTestPlan, - pub(crate) error_messages: Vec, -} - positional_record! { extract; /// An SQL function's name with its rendered UDF and table-function call affixes. @@ -76,7 +72,7 @@ pub(crate) fn resolve_compiled_sql_test_chains( .iter() .map(|model| { Ok(SqlTestChainModel { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, model_dependencies: model_dependencies(model)?, }) }) @@ -142,10 +138,10 @@ fn plan_model( rendered_sql: Option, ) -> PyResult { Ok(SqlTestPlanModel { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, query_sql: match rendered_sql { Some(sql) => sql, - None => model_reads::query_sql(model)?.extract()?, + None => query_sql(model)?.extract()?, }, model_dependencies: model_dependencies(model)?, }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs index 707f3f2975..e230e76bf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs @@ -3,4 +3,4 @@ use pyo3::prelude::{Bound, PyAny}; use std::collections::BTreeMap; -pub(super) type ValidatedTests<'py> = (Vec>, BTreeMap>); +pub(crate) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs b/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs new file mode 100644 index 0000000000..2983e7a114 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs @@ -0,0 +1,43 @@ +//! Public constants of sql test glue. + +pub(crate) const UDF: &str = "udf"; + +pub(crate) const TABLE_FN: &str = "table_fn"; + +pub(crate) const ERROR_SEVERITY: &str = "error"; + +pub(crate) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ + ("model", "__ref__"), + ("source", "__source__"), + ("seed", "__seed__"), + ("sql_test", "__expected__"), +]; + +pub(crate) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; + +pub(crate) const INTEGER_CURSOR_TYPE: &str = "integer"; + +pub(crate) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; + +pub const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; + +pub const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; + +pub const DEFAULT_INTEGER_START: &str = "-1000000000000000"; + +pub const DEFAULT_INTEGER_END: &str = "1000000000000000"; + +pub(crate) const CURSOR_START_KEY: &str = "cursor_start"; + +pub(crate) const CURSOR_END_KEY: &str = "cursor_end"; + +pub(crate) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; + +/// The diagnostic code of a SQL test that references a helper it may not read. +pub const SQL_TEST_HELPER_REFERENCE_CODE: &str = "P013"; + +/// The scan store namespace of expanded SQL-test CTEs. +pub const SQL_TEST_CTE_SCAN_ALGORITHM: &str = "expanded-sql-test-ctes-v1"; + +/// The scan store namespace of the models SQL tests expect. +pub const SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: &str = "sql-test-expected-models-v1"; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs new file mode 100644 index 0000000000..ecd602f9e0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs @@ -0,0 +1,18 @@ +//! Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. + +use crate::sql_test_glue::_helpers::plan_assembly; +use crate::sql_test_glue::models::SqlTestAssembly; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. +#[inline] +pub fn assemble_sql_test_plans<'py>( + tests: &[Bound<'py, PyAny>], + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + assembly: &SqlTestAssembly<'py>, +) -> PyResult> { + plan_assembly::assemble_sql_test_plans(tests, (project, adapter), assembly) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs new file mode 100644 index 0000000000..5d3bad2924 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs @@ -0,0 +1,8 @@ +pub mod assemble_sql_test_plans; +pub mod plan_compiled_sql_test_artifacts_function; +pub mod plan_sql_tests_function; +pub mod plan_sql_tests_natively_function; +pub mod planned_rows; +pub mod sql_test_model_chain_names_by_key; +pub mod sql_test_model_chain_names_by_key_function; +pub mod sql_test_plan_error_messages_function; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs new file mode 100644 index 0000000000..06dfdd9335 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs @@ -0,0 +1,11 @@ +//! `plan_compiled_sql_test_artifacts` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan_driver; + +/// `plan_compiled_sql_test_artifacts` as a function of `module`. +#[inline] +pub fn plan_compiled_sql_test_artifacts_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + plan_driver::plan_compiled_sql_test_artifacts_function(module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs new file mode 100644 index 0000000000..31d15fcd73 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs @@ -0,0 +1,11 @@ +//! `plan_sql_tests` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan_assembly; + +/// `plan_sql_tests` as a function of `module`. +#[inline] +pub fn plan_sql_tests_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + plan_assembly::plan_sql_tests_function(module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs new file mode 100644 index 0000000000..6e6e3cf4f5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs @@ -0,0 +1,11 @@ +//! `plan_sql_tests_natively` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan_driver; + +/// `plan_sql_tests_natively` as a function of `module`. +#[inline] +pub fn plan_sql_tests_natively_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + plan_driver::plan_sql_tests_natively_function(module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs new file mode 100644 index 0000000000..8a52300595 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs @@ -0,0 +1,18 @@ +//! Native plans of one SQL-test batch, with adapter rendering and timing accounted for. + +use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::models::PlanRow; +use crate::sql_test_glue::models::SqlTestPlanOptions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Native plans of one SQL-test batch, with adapter rendering and timing accounted for. +#[inline] +pub fn planned_rows<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + tests: &Bound<'py, PyAny>, + options: &SqlTestPlanOptions<'py>, +) -> PyResult> { + plan_driver::planned_rows((project, adapter), tests, options) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs new file mode 100644 index 0000000000..4226cd70ea --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs @@ -0,0 +1,16 @@ +//! Return model closures resolved natively in one project batch, keyed by SQL test key. + +use crate::sql_test_glue::_helpers::plan_driver; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Return model closures resolved natively in one project batch, keyed by SQL test key. +#[inline] +pub fn sql_test_model_chain_names_by_key<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + plan_driver::sql_test_model_chain_names_by_key(project, tests) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs new file mode 100644 index 0000000000..c905c4b594 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs @@ -0,0 +1,11 @@ +//! `sql_test_model_chain_names_by_key` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan_driver; + +/// `sql_test_model_chain_names_by_key` as a function of `module`. +#[inline] +pub fn sql_test_model_chain_names_by_key_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + plan_driver::sql_test_model_chain_names_by_key_function(module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs new file mode 100644 index 0000000000..44e9dccf4e --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs @@ -0,0 +1,11 @@ +//! `sql_test_plan_error_messages` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan_driver; + +/// `sql_test_plan_error_messages` as a function of `module`. +#[inline] +pub fn sql_test_plan_error_messages_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + plan_driver::sql_test_plan_error_messages_function(module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs new file mode 100644 index 0000000000..4cff23cef2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs @@ -0,0 +1,6 @@ +//! SQL test planning: fixtures, entries, rows, rendering, errors and cursor windows. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/models.rs b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs new file mode 100644 index 0000000000..06681b9e6b --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs @@ -0,0 +1,29 @@ +//! Public records of sql test glue. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use sqlbuild_analysis::compiler::models::SqlTestPlan; + +/// How one batch of SQL tests is planned. +#[derive(Debug)] +pub struct SqlTestPlanOptions<'py> { + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub render_sql: bool, + pub include_plan: bool, + pub artifact: bool, +} + +/// How assembling SQL-test plan results validates fixtures. +#[derive(Debug)] +pub struct SqlTestAssembly<'py> { + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub validate_fixtures: bool, + pub fixture_planning_context: Bound<'py, PyAny>, +} + +/// A native SQL-test plan with its distinct error messages. +#[derive(Debug)] +pub struct PlanRow { + pub plan: SqlTestPlan, + pub error_messages: Vec, +} diff --git a/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs b/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs new file mode 100644 index 0000000000..c27ad92817 --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod nullability; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs b/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs rename to crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs diff --git a/crates/sqlbuild-python-project/src/type_system/main/mod.rs b/crates/sqlbuild-python-project/src/type_system/main/mod.rs new file mode 100644 index 0000000000..20e7f1fc56 --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/main/mod.rs @@ -0,0 +1 @@ +pub mod register_nullability; diff --git a/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs new file mode 100644 index 0000000000..83c5332bda --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs @@ -0,0 +1,12 @@ +//! Register the nullability functions and classes. + +use crate::type_system::_helpers::nullability; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the nullability functions and classes. +#[inline] +pub fn register_nullability(module: &Bound<'_, PyModule>) -> PyResult<()> { + nullability::register(module) +} diff --git a/crates/sqlbuild-python-project/src/type_system/mod.rs b/crates/sqlbuild-python-project/src/type_system/mod.rs new file mode 100644 index 0000000000..3b579f414a --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/mod.rs @@ -0,0 +1,4 @@ +//! Nullability of SQL types. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index e31e9d0f07..43a1fc0144 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -17,7 +17,6 @@ extension-module = ["pyo3/extension-module"] [dependencies] chrono.workspace = true -chrono-tz.workspace = true fensu-policy.workspace = true pyo3.workspace = true rayon.workspace = true @@ -25,7 +24,6 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true -sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -42,6 +40,7 @@ sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true @@ -50,7 +49,6 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true -tokio.workspace = true uuid.workspace = true [lints] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs index 4a2c0e4e79..e60427317e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::binding_locations::models::{ use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use sqlbuild_python_base::positional_record; -use crate::bindings::models::Positional; +use sqlbuild_python_project::analysis::models::Positional; positional_record! { into; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs index 8e6a305bf8..445fe009fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis::models::BindingCatalog; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 39fd6f893b..f896a0e29c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -1,14 +1,6 @@ -pub(crate) mod binding_catalog; pub(crate) mod binding_locations; -mod catalog; -pub(crate) mod conversions; pub(crate) mod getters; -pub(crate) mod normalization; -pub(crate) mod normalization_results; -mod positions; pub(crate) mod queries; -pub(crate) mod relation_names; mod required_columns; -pub(crate) mod schema_request; pub(crate) mod sql_test_comparisons; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index 249917466f..853a047f26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -27,9 +27,11 @@ fn parse_warehouse_type(py: Python<'_>, spelling: &str, dialect: &str) -> PyResu } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::analysis::binding_catalog::register(module)?; + sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog( + module, + )?; crate::bindings::_helpers::analysis::required_columns::register(module)?; - crate::bindings::_helpers::references::explicit_references::register(module)?; + sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs deleted file mode 100644 index e676df32f8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Fixture column inference computed once per fixture body and inference profile. - -use crate::bindings::_helpers::analysis_session::fixture_columns::infer_fixture_column_facts; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use std::sync::Mutex; - -/// Reuse inference for identical fixture SQL under an equal, catalog-free profile. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct FixtureColumnInferences { - by_profile: Mutex, Py)>>, -} - -#[pymethods] -impl FixtureColumnInferences { - #[new] - pub(crate) fn new() -> Self { - Self { - by_profile: Mutex::new(Vec::new()), - } - } - - /// The inference for one fixture query, computing each distinct query once. - #[pyo3(signature = (*, query_sql, inference_profile))] - fn infer<'py>( - &self, - query_sql: &Bound<'py, PyAny>, - inference_profile: &Bound<'py, PyAny>, - ) -> PyResult> { - let py = query_sql.py(); - let compute = || { - infer_fixture_column_facts(Some(&keywords( - py, - &[ - ("query_sql", query_sql.clone()), - ("inference_profile", inference_profile.clone()), - ], - )?)) - }; - if !native_field(inference_profile, "binding_catalog")?.is_none() { - return compute(); - } - let inferred = self.profile_results(py, inference_profile)?; - if let Some(found) = inferred.get_item(query_sql)? { - return Ok(found); - } - let computed = compute()?; - inferred.set_item(query_sql, &computed)?; - Ok(computed) - } -} - -impl FixtureColumnInferences { - fn profile_results<'py>( - &self, - py: Python<'py>, - inference_profile: &Bound<'py, PyAny>, - ) -> PyResult> { - let known: Vec<(Py, Py)> = self - .by_profile - .lock() - .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? - .iter() - .map(|(profile, results)| (profile.clone_ref(py), results.clone_ref(py))) - .collect(); - for (profile, results) in known { - if profile.bind(py).eq(inference_profile)? { - return Ok(results.into_bound(py)); - } - } - let results = PyDict::new(py); - self.by_profile - .lock() - .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? - .push((inference_profile.clone().unbind(), results.clone().unbind())); - Ok(results) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs index 2e8ae0d008..f2f6b837c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index fae540de91..1131fb370c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,13 +1,5 @@ -pub(crate) mod fixture_columns; -pub(crate) mod fixture_inferences; pub(crate) mod getters; pub(crate) mod model_analyses; -pub(crate) mod models; -pub(crate) mod nullability_rules; pub(crate) mod oracle; pub(crate) mod outcome_objects; -pub(crate) mod profile_inputs; -pub(crate) mod query_columns; pub(crate) mod registration; -pub(crate) mod session; -pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 730a898710..2b9554924d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -7,24 +7,12 @@ use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; use crate::bindings::_helpers::project_assembly::declared_relations::DeclaredShapes; +use sqlbuild_python_project::analysis::models::BindingCatalog; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, -}; -use crate::bindings::_helpers::analysis_session::session::{ - CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, - start_model_analysis_session, -}; use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - AttachedSchema, NativeModelInput, attached_schema, input_analysis_sql, input_model_name, - input_reference_rows, input_sql_macro_checks, -}; -use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::cache::main::content_digest::content_digest; @@ -36,6 +24,26 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_python_project::analysis_session::main::case_sensitive_shapes::case_sensitive_shapes; +use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; +use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; +use sqlbuild_python_project::analysis_session::main::return_types::return_types; +use sqlbuild_python_project::analysis_session::main::shape_rows::shape_rows; +use sqlbuild_python_project::analysis_session::main::start_model_analysis_session::start_model_analysis_session; +use sqlbuild_python_project::analysis_session::models::CacheStatsRow; +use sqlbuild_python_project::analysis_session::models::FamilyRow; +use sqlbuild_python_project::analysis_session::models::ModelRow; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::analysis_session::models::RequestRow; +use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_model_name::input_model_name; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_sql_macro_checks::input_sql_macro_checks; +use sqlbuild_python_project::compiled_views::models::AttachedSchema; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; +use sqlbuild_python_project::semantic_shapes::constants::VALID_FROM_DEFAULT; +use sqlbuild_python_project::semantic_shapes::constants::VALID_TO_DEFAULT; const LOGGING_MODULE: &str = "logging"; const DEBUG_LOGGER: &str = "sqlbuild.compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs deleted file mode 100644 index 8fd2e82dbf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Native state kept by the analysis-session bindings. - -use pyo3::pyclass; - -/// Fixture column inferences computed since the last reset. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FixtureInferenceCount { - pub(crate) computed: u64, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index a0a1b9e965..bb49591f29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -6,12 +6,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; +use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; const LINEAGE_NATIVE: &str = "native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index aa62b2be6a..54c1e99891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -2,17 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::analysis_session::{ - fixture_columns, fixture_inferences, oracle, outcome_objects, query_columns, session, - stage_functions, -}; +use crate::bindings::_helpers::analysis_session::{oracle, outcome_objects}; +use sqlbuild_python_project::analysis_session::main::register_fixture_columns::register_fixture_columns; +use sqlbuild_python_project::analysis_session::main::register_query_columns::register_query_columns; +use sqlbuild_python_project::analysis_session::main::register_session::register_session; +use sqlbuild_python_project::analysis_session::main::register_stage_functions::register_stage_functions; +use sqlbuild_python_project::analysis_session::models::FixtureColumnInferences; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - session::register(module)?; - fixture_columns::register(module)?; - module.add_class::()?; + register_session(module)?; + register_fixture_columns(module)?; + module.add_class::()?; outcome_objects::register(module)?; - query_columns::register(module)?; - stage_functions::register(module)?; + register_query_columns(module)?; + register_stage_functions(module)?; oracle::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 1fb668c0c9..61b81b210c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::build_progress::progress_text::write_execution_he use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; @@ -43,6 +42,8 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::compiled_project::types::VerifiedRun; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs index 5b11d2bbb2..ab6f2de2b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs @@ -1,5 +1,4 @@ mod identity; mod measurement; pub(crate) mod registration; -pub(crate) mod render; mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs index 3128dee0f4..8a06cf2b74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs @@ -3,6 +3,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::auditing::identity::register(module)?; crate::bindings::_helpers::auditing::measurement::register(module)?; - crate::bindings::_helpers::auditing::render::register(module)?; + sqlbuild_python_project::auditing::main::register_render::register_render(module)?; crate::bindings::_helpers::auditing::schema_audits::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8e3408df42..423da7418d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,8 +19,6 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, }; @@ -36,6 +34,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index dd9b0fb38d..4239e54c9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -22,9 +22,6 @@ use crate::bindings::_helpers::build_command::policies::enforce_model_execution_ use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; -use crate::bindings::_helpers::planner_selection::unit_tests::split_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -34,6 +31,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::ScopeData; +use sqlbuild_python_project::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; const VIEW_MATERIALIZATION: &str = "view"; const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 59374d9439..a50b17398d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -33,7 +33,6 @@ use crate::bindings::_helpers::consumption::recorder::{ }; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::publication::append_freshness_records; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; @@ -60,6 +59,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; const EXECUTION_COMMAND: &str = "sqb build"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index d5ae29884b..5de64046c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -32,6 +31,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 8ab3d82fbd..b5d40e9b4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -14,9 +14,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::planner_scope::loaders::build_intermediate_source_map; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -32,6 +29,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; const SUCCESS_STATUS: &str = "success"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index 2fda521d2e..8f14f874f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -25,7 +25,6 @@ use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, buil use crate::bindings::_helpers::build_progress::result_text::{ function_text, load_text, model_text, optional, seed_text, test_text, text, }; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -48,6 +47,7 @@ use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; const PASS_OUTCOME: &str = "pass"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 34b76fdc58..3245d10cc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -21,13 +21,16 @@ use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefi use crate::bindings::_helpers::change_selection::fingerprints::{ Fingerprinting, directory_digest, file_digest, text_of, }; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; +use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; /// Header values that rename rather than redefine a model. const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; @@ -237,10 +240,14 @@ fn model_node( fingerprints: &Fingerprinting, uses: &BTreeMap>, ) -> PyResult<(String, NodeDefinition)> { - let name: String = model_reads::model_name(model)?; - let config: Bound<'_, PyAny> = model_reads::config(model)?; + let name: String = model_name(model)?; + let config: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?; let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; - let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let schema: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; let contract: String = match config_values.cast::() { Ok(values) => match values.get_item(CONTRACT_KEY)? { Some(value) => fingerprints.canonical(&value, &[])?.to_string(), @@ -296,8 +303,10 @@ fn source_node( source: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { - let name: String = source_reads::name(source)?.extract()?; - let mut encoded: Value = fingerprints.canonical(&source_reads::source_entry(source)?, &[])?; + let name: String = + sqlbuild_python_project::compiled_views::main::source_reads::name::name(source)? + .extract()?; + let mut encoded: Value = fingerprints.canonical(&source_entry(source)?, &[])?; let mut shape: Map = Map::new(); let mut reads: Option = None; if let Value::Object(fields) = &mut encoded { @@ -333,9 +342,13 @@ fn seed_node( seed: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { - let name: String = seed_reads::name(seed)?.extract()?; + let name: String = + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?.extract()?; let (version_hash, _) = seed_identity(seed)?; - let schema: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let schema: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry( + seed, + )?; let columns: String = if schema.is_none() { definition_digest(&[]) } else { @@ -450,7 +463,10 @@ fn function_nodes( fingerprints: &Fingerprinting, ) -> PyResult> { let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); - let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let functions: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )?; let hashes: Bound<'_, PyDict> = function_local_hashes( &functions, None, @@ -458,17 +474,20 @@ fn function_nodes( )?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; - let key: Bound<'_, PyAny> = function_reads::key(&function)?; + let key: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?; let kind: String = text_of(&native_field( &native_field(&key, "resource_type")?, "value", )?)?; - let name: String = function_reads::name(&function)?.extract()?; + let name: String = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + .extract()?; let Some(hash) = hashes.get_item(&name)? else { continue; }; let mut digest: String = hash.extract()?; - let source_file: Bound<'_, PyAny> = function_reads::source_file_path(&function)?; + let source_file: Bound<'_, PyAny> = source_file_path(&function)?; if !source_file.is_none() { let path: String = text_of(&source_file)?; if path.ends_with(PYTHON_SUFFIX) { @@ -479,7 +498,7 @@ fn function_nodes( } } let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); - for dependency in function_reads::deps(&function)?.try_iter()? { + for dependency in deps(&function)?.try_iter()? { let dependency: Bound<'_, PyAny> = dependency?; definition.uses.push(node_key( &text_of(&native_field( @@ -501,7 +520,7 @@ fn test_file_nodes( fingerprints: &Fingerprinting, ) -> PyResult> { let mut nodes: BTreeMap = BTreeMap::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; let path: String = text_of(&native_field(&file, "relative_path")?)?; let key: String = node_key(TEST_FILE_KIND, &path); @@ -590,15 +609,15 @@ pub(crate) fn definition_snapshot( ); let (connection_key, connection) = connection_node(project, discovered, &fingerprints)?; nodes.insert(connection_key, connection); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let (key, definition) = model_node(&model?, &fingerprints, &uses)?; nodes.insert(key, definition); } - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let (key, definition) = source_node(&source?, &fingerprints)?; nodes.insert(key, definition); } - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let (key, definition) = seed_node(&seed?, &fingerprints)?; nodes.insert(key, definition); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 84f252a614..af780e7b0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -26,12 +26,12 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::{ declaration_uses, definition_snapshot, project_wide_keys, }; use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; const MACRO_TEST_MODE: &str = "macro"; /// The JSON document key of the changes and selection of a `--since` run. @@ -107,7 +107,7 @@ fn relation_keys((refs, sources, seeds): (&[String], &[String], &[String])) -> V /// The sources `model` reads, other than `mocked` ones, as node keys. fn read_sources(model: &Bound<'_, PyAny>, mocked: &[String]) -> PyResult> { let mut keys: Vec = Vec::new(); - for dependency in model_reads::deps(model)?.try_iter()? { + for dependency in deps(model)?.try_iter()? { let (kind, name) = native_key(&dependency?)?; if kind == SOURCE_KIND && !mocked.contains(&name) { keys.push(node_key(SOURCE_KIND, &name)); @@ -239,7 +239,7 @@ impl ChangedProject { continue; }; item.executed.extend(read_sources(model, &mocked_sources)?); - for dependency in model_reads::deps(model)?.try_iter()? { + for dependency in deps(model)?.try_iter()? { let (kind, dependency_name) = native_key(&dependency?)?; if kind == SEED_KIND && !mocked_seeds.contains(&dependency_name) { item.executed.push(node_key(SEED_KIND, &dependency_name)); @@ -328,14 +328,22 @@ pub(crate) fn tests_and_models<'py>( project: &Bound<'py, PyAny>, ) -> PyResult<(ByName<'py>, ByName<'py>)> { let mut tests: ByName<'py> = HashMap::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; tests.insert(native_field(&test, "name")?.extract()?, test); } let mut models: ByName<'py> = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model: Bound<'py, PyAny> = model?; - models.insert(model_reads::model_name(&model)?, model); + models.insert( + sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( + &model, + )?, + model, + ); } Ok((tests, models)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 0982412eeb..37b62b99c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -8,13 +8,7 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; -use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -36,6 +30,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_project::graph::main::algorithms::transitive_closure_many::transitive_closure_many; +use sqlbuild_python_project::graph::main::execution_gates::sql_ref_key::sql_ref_key; +use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use crate::bindings::_helpers::check::check_core::{ ReadSideInputs, load_results_by_loader_name, record_python_run_state_results, @@ -46,8 +46,6 @@ use crate::bindings::_helpers::check::check_results::{ }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::graph::python_selection::python_node_names; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -55,6 +53,8 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_selection::python_node_names::python_node_names; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 15d6903129..1301432fc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -3,8 +3,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::graph::python_selection::resolve_python_check_excludes; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -12,6 +10,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_project::graph::main::python_selection::resolve_python_check_excludes::resolve_python_check_excludes; +use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_core::build_command::errors::BuildCommandError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 00ea711da8..5ae1faf784 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; use crate::bindings::_helpers::compile_command::reuse_attempt::run_compile; -use crate::bindings::_helpers::compiled_project::job_catalog::with_job_scope; use crate::bindings::_helpers::contract_command::command::run_contract_command; use crate::bindings::_helpers::cost_command::command::run_cost_command; use crate::bindings::_helpers::dbt_command::command::run_dbt_command; @@ -57,6 +56,7 @@ use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_rec use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::models::MissingSecretsScope; +use sqlbuild_python_project::compiled_project::main::job_catalog::with_job_scope::with_job_scope; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index f4844e130a..4fd56ca394 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -29,7 +29,6 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_target_conne use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -48,6 +47,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const CLONE_LOGGER: &str = "sqlbuild.clone"; /// The clone phase that copies relations into the destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index c6399918c0..5c3ed796fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyTuple; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::models::SqlTestScanCache; @@ -22,6 +21,7 @@ use sqlbuild_python_front::scopes::main::index::build_scope_lookup::build_scope_ use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_native_scope_index; use sqlbuild_python_front::sql_test_glue::main::validate_authored_sql_test_ctes::validate_authored_sql_test_ctes; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index a5d3007cac..61d9404905 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod cursor_roles; pub(crate) mod declaration_scope; pub(crate) mod model_config; pub(crate) mod model_inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 0d2924508e..884976fc17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -11,8 +11,6 @@ use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_a use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; use crate::bindings::_helpers::input_loops::loop_classes::model_loop_classes; use crate::bindings::_helpers::input_loops::model_loop::{LoopSwitches, NativeModelLoop}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -29,6 +27,8 @@ use sqlbuild_python_front::model_config::main::model_nodes::unicode_environment: use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::render::main::expansion::macro_expansion_classes::macro_expansion_classes; use sqlbuild_python_front::render::models::ConstantRendering; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index a6aa6e901d..7770b938b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_attachment::{ - cursor_roles, declaration_scope, model_config, model_inputs, sql_expansion_context, target, + declaration_scope, model_config, model_inputs, sql_expansion_context, target, }; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config_function::build_effective_connection_config_function; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_function::build_effective_connection_function; @@ -13,6 +13,7 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_settings_co use sqlbuild_python_front::compile_attachment::main::build_effective_settings_function::build_effective_settings_function; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config_function::build_effective_target_config_function; use sqlbuild_python_front::compile_attachment::main::build_effective_vars_function::build_effective_vars_function; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles_function::resolve_cursor_input_roles_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -47,9 +48,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { declaration_scope::rebind_declaration_scope, module )?)?; - module.add_function(wrap_pyfunction!( - cursor_roles::resolve_cursor_input_roles, - module - )?)?; + module.add_function(resolve_cursor_input_roles_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 8ce8ee5894..5fdcc2c834 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -14,7 +14,6 @@ use sqlbuild_core::cli_text::models::{ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; -use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::emit_json_report::emit_json_report; @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; +use sqlbuild_python_project::semantic_shapes::main::semantic_coverage::semantic_coverage; const METADATA_MODULE: &str = "importlib.metadata"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 27c9fe1a28..ddd0e533d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -33,11 +33,6 @@ use crate::bindings::_helpers::compile_outputs::artifacts::{ use crate::bindings::_helpers::compile_outputs::runtime_target::{ compiled_sql_test_path, planned_sql_test_path, }; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; -use crate::bindings::_helpers::sql_test_glue::plan_driver::planned_rows; -use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; -use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -56,6 +51,11 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::path_posix_text::path_posix_text; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::sql_test_glue::main::planned_rows::planned_rows; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key::sql_test_model_chain_names_by_key; +use sqlbuild_python_project::sql_test_glue::models::PlanRow; +use sqlbuild_python_project::sql_test_glue::models::SqlTestPlanOptions; const COMPILED_DIR: &str = "compiled"; const MODELS_DIR: &str = "models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index 125c22a667..d98c4c3381 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -7,7 +7,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethod use sqlbuild_core::cli_text::models::ModelLine; use crate::bindings::_helpers::compile_command::report::diagnostic_item; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; @@ -17,6 +16,7 @@ use sqlbuild_python_base::compile_context::models::CapturedModel; use sqlbuild_python_base::compile_context::models::CapturedTest; use sqlbuild_python_base::compile_outputs::main::plain_json::plain_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::models::NativeProjectGraph; /// The graph key kind of a model. const MODEL_KIND: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 8f5b47607d..384f5a0b41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -10,10 +10,7 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; -use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, EarlyRulesSettings}; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; @@ -36,6 +33,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::contracts::main::evaluate_model_contracts::evaluate_model_contracts; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 621adc4833..18f39627af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -11,9 +11,9 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::compiled_views::records::record_model_analyses; -use crate::bindings::_helpers::references::explicit_references::explicit_reference_report; +use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::records::record_model_analyses::record_model_analyses; +use sqlbuild_python_project::references::main::explicit_reference_report::explicit_reference_report; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d74df6da1c..2cff136e89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -13,24 +13,14 @@ use sqlbuild_host::compile_context::main::record_compile_cpu_timing::record_comp use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CatalogKey, ignores_quoted_case, -}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - ADAPTER_RULE_ID, nullability_rules, text_pairs, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; -use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; -use crate::bindings::_helpers::compiled_views::source_views::read_source_view; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, DeclaredShapes, declared_relations, }; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::compile_context::models::PythonProcessClock; @@ -48,6 +38,17 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; +use sqlbuild_python_project::analysis::main::ignores_quoted_case::ignores_quoted_case; +use sqlbuild_python_project::analysis::models::BindingCatalog; +use sqlbuild_python_project::analysis::models::CatalogKey; +use sqlbuild_python_project::analysis_session::constants::ADAPTER_RULE_ID; +use sqlbuild_python_project::analysis_session::main::expression_source_shapes::expression_source_shapes; +use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; +use sqlbuild_python_project::analysis_session::main::text_pairs::text_pairs; +use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::semantic_shapes::main::declared_types::declared_types; +use sqlbuild_python_project::semantic_shapes::main::function_names::function_names; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::HashMap; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 60ef674765..465c19a371 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -4,10 +4,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use crate::bindings::_helpers::analysis_session::profile_inputs::dialect_or_generic; -use crate::bindings::_helpers::analysis_session::session::{ - FamilyRow, NativeModelAnalysisSession, PivotRequestRow, prove_dynamic_column_contracts, -}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; @@ -22,6 +18,11 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::models::HeaderFamilies; +use sqlbuild_python_project::analysis_session::main::dialect_or_generic::dialect_or_generic; +use sqlbuild_python_project::analysis_session::main::prove_dynamic_column_contracts::prove_dynamic_column_contracts; +use sqlbuild_python_project::analysis_session::models::FamilyRow; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::analysis_session::models::PivotRequestRow; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 1008452f1f..1a632256b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -8,14 +8,9 @@ use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; -use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; -use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; -use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::model_input_view::input_analysis_sql; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::project_assembly::syntax_validation::{ @@ -28,6 +23,11 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; use sqlbuild_python_front::render::main::calls::top_level_macro_call_names::top_level_macro_call_names; +use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; +use sqlbuild_python_project::analysis_session::main::query_column_rows::query_column_rows; +use sqlbuild_python_project::analysis_session::types::ColumnsRow; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 089b7861b2..c5811e1da4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -24,7 +24,6 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::semantic_checks::stage::{ SemanticCompletion, completed_semantic_project, }; @@ -43,6 +42,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::r use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; use sqlbuild_python_front::scopes::main::checks::scope_index_with_compile_usages::scope_index_with_compile_usages; use sqlbuild_python_front::scopes::main::reports::report_scope_index_errors::report_scope_index_errors; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index e2fe06bb1b..2675a9007a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -13,8 +13,6 @@ use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_dif use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::graph::views::project_lineage_views; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -23,6 +21,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::graph::main::views::project_lineage_views::project_lineage_views; /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. struct KeyIndexes<'py> { @@ -166,13 +167,13 @@ fn diff_model_names<'py>( let mut names: Vec> = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; + let key: Bound<'py, PyAny> = model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; if chosen && native_field(&key, "resource_type")?.eq(&model_type)? { - names.push(model_reads::name_object(&model)?); + names.push(name_object(&model)?); } } PyTuple::new(py, names) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 6bad877a8f..624b1c1be8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -11,8 +11,9 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_settings::b use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::records::{compile_inputs_view, inputs_with}; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::records::compile_inputs_view::compile_inputs_view; +use sqlbuild_python_project::compiled_views::main::records::inputs_with::inputs_with; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::input_loops::attachment::{attach_checks, attach_resources}; use sqlbuild_python_front::model_config::main::config::run_id::run_id; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index e2f94f9638..3b2138ba47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, write_static_target, }; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::manifest::builder::build_manifest; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index fccf7712f3..08a549b104 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -5,13 +5,14 @@ use pyo3::types::{PyDict, PyFloat}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; +use sqlbuild_python_project::graph::models::NativeProjectGraph; /// Discovered Python node functions whose selectors need the Python graph. const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 6ab72cb494..bc6f15c382 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,13 +9,14 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_g use crate::bindings::_helpers::compile_driver::python_plan_entries::{ build_python_run_plan_outputs, python_read_source_names_for, }; -use crate::bindings::_helpers::compiled_views::{project_reads, source_reads}; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; -use crate::bindings::_helpers::planner_phases::objects::object; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_phases::main::object::object; use std::collections::HashSet; use pyo3::exceptions::PyException; @@ -24,12 +25,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; -use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; const SINGLE_CONNECTION: i64 = 1; @@ -67,12 +68,18 @@ fn without_loaders<'py>( } } let mut sources: Vec<(String, String)> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { let source = source?; - let loader: Option = - native_field(&source_reads::source_entry(&source)?, "loader")?.extract()?; + let loader: Option = native_field(&source_entry(&source)?, "loader")?.extract()?; if loader.is_some_and(|loader| dropped.contains(&loader)) { - sources.push((SOURCE.to_owned(), source_reads::name(&source)?.extract()?)); + sources.push(( + SOURCE.to_owned(), + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + .extract()?, + )); } } let mut selected: Vec<(String, String)> = sql_keys diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 488081bd09..a2adff7b02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -30,8 +30,6 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; @@ -43,6 +41,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; +use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Optional keywords of the project graph builders. const GRAPH_OPTIONAL: [&str; 7] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index e58b34c785..f31b5b77ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -2,11 +2,11 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -16,7 +16,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::ordered_python_names::ordered_python_names; use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -24,6 +23,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; const SKIPPED_UPSTREAM_CODE: &str = "P501"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index 49b36814fd..b137970c1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -7,13 +7,13 @@ use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::{AdapterHost, UserCodeHost}; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 85d6a569ae..005212247d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -9,14 +9,15 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; -use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; -use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::ScopeData; +use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 6999a8412b..fa23b192b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::compile_driver::deferral::{ use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; -use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::planner_entries::command_plans::{ resolve_static_command_scope, resolve_static_relation_context, }; @@ -19,6 +18,7 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; /// Locations of the deferred target's relations, when the options defer to one. fn deferred_locations<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 06535472a5..1a1455efae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -3,16 +3,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::compiled_views::models::NativeCompiledFunction; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::compiled_views::models::NativeCompiledSeed; /// The adapter defaults every compiled location falls back to. pub(crate) struct TargetDefaults<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs deleted file mode 100644 index 4f548c9a9f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ /dev/null @@ -1,298 +0,0 @@ -//! The resource rows and settings the compile attaches into the project, and the installed runs. - -use sqlbuild_python_base::positional_record; -use std::collections::HashMap; -use std::sync::OnceLock; - -use pyo3::FromPyObject; -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; -use crate::bindings::_helpers::compiled_views::function_views::NativeFunctionInput; -use crate::bindings::_helpers::compiled_views::source_views::NativeSourceInput; -use sqlbuild_python_base::cache::models::SqlTestScanStore; - -const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; - -static INSTALLED: OnceLock = OnceLock::new(); - -/// One discovered source: its file's path, the discovered file and the entry. -pub(crate) type SourceRow = (String, Py, Py); - -/// `(default contract, whether expressions are syntax-checked, source kind)`. -pub(crate) type SourceSettings = (Option, bool, Py); - -positional_record! { - extract; - /// A function's paths, stem, discovered file, header values, body and SQL file contents. - pub(crate) struct FunctionRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) name: String, - pub(crate) function_file: Py, - pub(crate) header_values: Py, - pub(crate) body: String, - pub(crate) contents: Option, - } -} - -positional_record! { - extract; - /// Adapter, validation, raw default and target namespaces, inheritance and resource kind. - pub(crate) struct FunctionSettings { - pub(crate) adapter_name: String, - pub(crate) validate: bool, - pub(crate) defaults: (Option, Option), - pub(crate) target: Option<(Option, Option)>, - pub(crate) inherit: bool, - pub(crate) kind: Py, - } -} - -/// A test case's discovered case, name and parameter values in authored order. -pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); - -positional_record! { - extract; - /// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. - pub(crate) struct TestBlockRow { - pub(crate) block: Py, - pub(crate) name: Option, - pub(crate) sql_body: String, - pub(crate) mode: String, - pub(crate) parameters: Vec, - pub(crate) cases: Vec, - pub(crate) sql_body_span: Option<(usize, usize)>, - } -} - -positional_record! { - extract; - /// A test file's paths, stem, discovered file, contents and blocks. - pub(crate) struct TestFileRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) stem: String, - pub(crate) file: Py, - pub(crate) contents: String, - pub(crate) blocks: Vec, - } -} - -/// What every SQL test and scenario of one compile resolves and records with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestSettings { - /// `CompileSqlTestInput`, its model and direct payloads, and `CompileSqlTestCte`. - pub(crate) test_input: Py, - pub(crate) model_payload: Py, - pub(crate) direct_payload: Py, - pub(crate) test_cte: Py, - /// `CompileSqlScenarioInput` and `CompileSqlScenarioCte`. - pub(crate) scenario_input: Py, - pub(crate) scenario_cte: Py, - /// `SqlTestMode` by value. - pub(crate) modes: Py, - /// The `ResourceKind`s of tests and scenarios, and the runtime `UsageKind`. - pub(crate) test_kind: Py, - pub(crate) scenario_kind: Py, - pub(crate) runtime_usage: Py, - /// The adapter's typed scalar renderer, returning `(text, None)` or `(None, error)`. - pub(crate) render_parameter: Py, - pub(crate) adapter_name: String, - /// `SourceLocation`, for helper problems located natively. - pub(crate) source_location: Py, - /// The shared scan store with the scan algorithm and lexical rules keying it, if any. - pub(crate) scan_store: Option>, - pub(crate) scan_key: (String, String), -} - -/// The attached SQL test inputs, then scenario inputs. -pub(crate) type AttachedSqlTests = (Vec>, Vec>); - -positional_record! { - extract; - /// A scenario file's paths, discovered file, contents, SQL body and name. - pub(crate) struct ScenarioRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) scenario_file: Py, - pub(crate) contents: String, - pub(crate) sql_body: String, - pub(crate) name: String, - } -} - -positional_record! { - extract, into; - /// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. - pub(crate) struct AuditBlockRow { - pub(crate) block: Py, - pub(crate) name: Option, - pub(crate) sql_body: String, - pub(crate) evidence_sql: Option, - pub(crate) measurement: bool, - pub(crate) header: Py, - } -} - -positional_record! { - extract, into; - /// An audit file's paths, stem, discovered file, generic flag, contents and blocks. - pub(crate) struct AuditFileRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) stem: String, - pub(crate) file: Py, - pub(crate) generic: bool, - pub(crate) contents: String, - pub(crate) blocks: Vec, - } -} - -/// What every audit in one compile resolves against, and the Python values it builds with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct AuditSettings { - pub(crate) default_severity: Option, - pub(crate) default_run_scope: Option, - /// Built-in generic definitions as audit file rows, overridden by project definitions. - pub(crate) builtins: Vec, - /// `DeclarationIdentity`, and the singular and generic audit `DeclarationKind`s. - pub(crate) declaration_identity: Py, - pub(crate) singular_kind: Py, - pub(crate) audit_kind: Py, - /// `ResourceKind` by value, for the resources singular audits read and attach to. - pub(crate) resource_kinds: Py, -} - -/// A loader's description, relative path and whether that path is a YAML file. -pub(crate) type LoaderRow = (Option, String, bool); - -positional_record! { - extract; - /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. - pub(crate) struct PythonResourceRow { - pub(crate) kind: String, - pub(crate) name: String, - pub(crate) description: Option>, - pub(crate) location: Py, - pub(crate) help: String, - } -} - -positional_record! { - extract; - /// A scenario's name, relative path, contents and authored header description. - pub(crate) struct ScenarioDescriptionRow { - pub(crate) name: String, - pub(crate) path: String, - pub(crate) contents: String, - pub(crate) raw: Py, - } -} - -positional_record! { - extract; - /// A SQL hook file's name, relative path, contents and description. - pub(crate) struct HookDescriptionRow { - pub(crate) name: String, - pub(crate) path: String, - pub(crate) contents: String, - pub(crate) description: Option, - } -} - -/// What the resource diagnostics read beyond the owner, and the Python values they build. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct DiagnosticSettings { - /// `CompilerDiagnostic`, `SourceLocation`, `Path` and `CompileInputError`. - pub(crate) compiler_diagnostic: Py, - pub(crate) compile_error: Py, - pub(crate) source_location: Py, - pub(crate) path_class: Py, - /// Every discovered model file, whose headers attach audit factories. - pub(crate) model_files: Vec>, - /// Each discovered audit factory's name, relative path and line. - pub(crate) audit_factories: Vec<(String, String, usize)>, - /// Each scenario's name, relative path, contents and authored header description. - pub(crate) scenarios: Vec, - /// Each SQL hook file's name, relative path, contents and description. - pub(crate) sql_hooks: Vec, - /// Each loader by name. - pub(crate) loaders: HashMap, - /// Python-defined resources: kind, name, description, location and help, read by Python. - pub(crate) python_resources: Vec, - /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. - pub(crate) description_limit: (usize, i64), -} - -/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. -pub(crate) type ResourceDiagnostics<'py> = ( - Vec>, - Vec>, - Vec>, -); - -/// Attaches the project's sources, given their rows and settings. -pub(crate) type AttachSources = fn( - &Bound<'_, NativeCompiledProject>, - Vec, - SourceSettings, -) -> PyResult>; - -/// Attaches the project's SQL, then Python, functions. -pub(crate) type AttachFunctions = fn( - &Bound<'_, NativeCompiledProject>, - (Vec, Vec), - FunctionSettings, -) -> PyResult>; - -/// Reads the project's resource diagnostics. -pub(crate) type ResourceDiagnosticsRun = for<'py> fn( - &NativeCompiledProject, - Python<'py>, - &DiagnosticSettings, -) -> PyResult>; - -/// Attaches the project's SQL test files, then its scenarios. -pub(crate) type AttachSqlTests = fn( - &Bound<'_, NativeCompiledProject>, - Vec, - Vec, - SqlTestSettings, -) -> PyResult; - -/// Attaches the project's audits; returns their views and the built-in shadow warnings. -pub(crate) type AttachAudits = - for<'py> fn( - &Bound<'py, NativeCompiledProject>, - Vec, - AuditSettings, - ) -> PyResult<(Vec, Vec>)>; - -/// The compile's runs behind the project's attachment methods. -#[derive(Clone, Copy, Debug)] -pub(crate) struct ResourceAttachment { - pub(crate) sources: AttachSources, - pub(crate) functions: AttachFunctions, - pub(crate) diagnostics: ResourceDiagnosticsRun, - pub(crate) sql_tests: AttachSqlTests, - pub(crate) audits: AttachAudits, -} - -/// Install the compile's attachment runs once, at registration. -pub(crate) fn install(attachment: ResourceAttachment) { - let _ = INSTALLED.get_or_init(|| attachment); -} - -/// The installed attachment runs. -pub(crate) fn resource_attachment() -> PyResult<&'static ResourceAttachment> { - INSTALLED - .get() - .ok_or_else(|| PyRuntimeError::new_err(ATTACHMENT_UNINSTALLED)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 8f4b12ec83..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,16 +1 @@ -pub(crate) mod attachment_inputs; -pub(crate) mod constants; -pub(crate) mod job_catalog; -pub(crate) mod project; pub(crate) mod registration; -pub(crate) mod rows; -pub(crate) mod schema_facts; -pub(crate) mod snapshot; -pub(crate) mod snapshot_boundary; -pub(crate) mod snapshot_declarations; -pub(crate) mod snapshot_execution; -pub(crate) mod snapshot_models; -pub(crate) mod snapshot_resources; -pub(crate) mod snapshot_roles; -pub(crate) mod snapshot_worker; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs deleted file mode 100644 index 6f644d93cc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ /dev/null @@ -1,664 +0,0 @@ -//! The compile's natively owned project, which producers record into and consumers read. - -use sqlbuild_python_base::positional_record; -use std::collections::{BTreeMap, HashMap}; -use std::sync::Mutex; - -use pyo3::exceptions::{PyIndexError, PyRuntimeError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{FromPyObject, pyclass, pymethods}; -use serde_json::Value; -use sqlbuild_analysis::compiled_project::main::project_models::project_models; -use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; -use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; -use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicProofFact, ModelAnalysisFact, - ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, -}; - -use sqlbuild_project::models::CompiledProject; - -use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::DiagnosticSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ResourceDiagnostics; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SqlTestSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::resource_attachment; -use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; -use crate::bindings::_helpers::compiled_views::audit_views::{ - AttachedAudit, AuditRecord, NativeAuditInput, NativeCompiledAudit, audit_record, -}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledViewClasses, NativeCompiledModel, -}; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, FunctionRecord, NativeCompiledFunction, NativeFunctionInput, function_record, -}; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputRecord, NativeModelInput, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, SeedViews, paired_seeds, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - NativeCompiledSource, NativeSourceInput, SourceRecord, -}; -use sqlbuild_project::types::InferredRow; -use sqlbuild_project::types::ObjectKeyRow; -use sqlbuild_project::types::SourceNamespace; -use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; -use sqlbuild_python_front::render::models::ConstantRendering; - -pub(crate) const CONTRACT_KEY: &str = "contract"; -const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; -const ATTACHMENT_LOOP_UNSET: &str = "no model loop was kept to attach this project's resources"; -pub(crate) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; -const MATERIALIZED_KEY: &str = "materialized"; -const TABLE_MATERIALIZATION: &str = "table"; - -/// `(name, type)`. -pub(crate) type NamedType = (String, Option); -positional_record! { - extract; - /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. - pub(crate) struct ProofPy { - pub(crate) output_proven: bool, - pub(crate) bare_dynamic_pivot: bool, - pub(crate) families: Vec, - pub(crate) failure_reason: Option, - pub(crate) named_families: Vec, - } -} - -positional_record! { - extract; - /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. - pub(crate) struct AnalysisPy { - pub(crate) inferred: bool, - pub(crate) inferred_columns: Vec, - pub(crate) nullable_columns: Vec, - pub(crate) fast_lineage_has_star: bool, - pub(crate) unchecked: Vec, - } -} - -positional_record! { - extract; - /// A model's name, relative path, compiled and authored queries, and authored SQL file. - pub(crate) struct ModelTextsPy { - name: String, - relative_path: String, - query_sql: String, - authored_query_sql: String, - authored_sql: String, - } -} - -/// Encoded config values and the encoder's error, if any. -pub(crate) type ConfigValues = (BTreeMap, Option); - -/// One model as the compile-input stage produced it. -#[derive(FromPyObject)] -pub(crate) struct ModelInputPy<'py>( - ModelTextsPy, - (Bound<'py, PyDict>, Vec, Option), - Vec<(String, String, Option)>, - Vec, - (Vec>, Vec>), -); - -/// Name, relative path, members, value, value type and rendering of one declaration. -#[derive(FromPyObject)] -pub(crate) struct DeclarationPy<'py>( - String, - String, - Vec<(String, Bound<'py, PyAny>)>, - Bound<'py, PyAny>, - Option, - Option, -); - -/// One compile's project, recorded by the stages that produce it, with its Python projections. -#[pyclass(module = "sqlbuild._native")] -#[derive(Default)] -pub(crate) struct NativeCompiledProject { - project: CompiledProject, - /// The Python classes model input views project their attributes with. - input_classes: Option>, - /// Shared immutable column projections, allocated only when Python consumers request them. - inferred_column_projections: Mutex>>, - /// The model loop and its constant rendering, which attach the project's other resources. - attachment: Option<(Py, Py)>, -} - -impl NativeCompiledProject { - /// The facts recorded so far, for native consumers such as the rules request. - pub(crate) fn facts(&self) -> &CompiledProjectFacts { - &self.project.facts - } - - /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. - pub(crate) fn with_facts(facts: CompiledProjectFacts) -> Self { - Self { - project: CompiledProject::with_facts(facts), - ..Self::default() - } - } - - /// Retain facts a producer finished natively. - pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { - self.project.retain(facts); - } - - /// Retain a finished model's input record; return its index for the model's view. - pub(crate) fn retain_input(&mut self, record: ModelInputRecord) -> usize { - self.project.retain_input(record) - } - - /// Keep the classes model input views project their attributes with. - pub(crate) fn set_input_classes(&mut self, classes: Py) { - self.input_classes = Some(classes); - } - - /// Count one lineage graph a project view built. - pub(crate) fn record_graph_build(&mut self) { - self.project.graph_builds += 1; - } - - /// The kept model loop and its constant rendering. - pub(crate) fn attachment_loop( - &self, - py: Python<'_>, - ) -> PyResult<(Py, Py)> { - match &self.attachment { - Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), - None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), - } - } - - /// Keep the model loop and its constant rendering for attaching the project's other resources. - pub(crate) fn keep_attachment_loop( - &mut self, - model_loop: Py, - host: Py, - ) { - self.attachment = Some((model_loop, host)); - } - - /// Retain attached functions; return the index of the first. - pub(crate) fn extend_functions(&mut self, attached: Vec) -> usize { - let first: usize = self.project.functions.len(); - self.project - .functions - .extend(attached.into_iter().map(function_record)); - first - } - - /// Retain attached audits; return the index of the first. - pub(crate) fn extend_audits(&mut self, attached: Vec) -> usize { - let first: usize = self.project.audits.len(); - self.project - .audits - .extend(attached.into_iter().map(audit_record)); - first - } - - /// Every attached audit, in attachment order. - pub(crate) fn audits(&self) -> &[AuditRecord] { - self.project.audits() - } - - /// Every attached function, in attachment order. - pub(crate) fn functions(&self) -> &[FunctionRecord] { - self.project.functions() - } - - /// Every attached source, in attachment order. - pub(crate) fn sources(&self) -> &[SourceRecord] { - self.project.sources() - } - - /// Keep one attached source. - pub(crate) fn retain_source(&mut self, source: SourceRecord) { - self.project.sources.push(source); - } - - /// Every attached seed, in attachment order. - pub(crate) fn seeds(&self) -> &[SeedRecord] { - self.project.seeds() - } - - /// The schema fact recorded for an input record's model. - pub(crate) fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { - self.project.input_schema_fact(record) - } - - /// Every model's input record, in loop order. - pub(crate) fn inputs(&self) -> &[ModelInputRecord] { - &self.project.inputs - } - - /// The input record at `index`. - pub(crate) fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { - self.project - .input(index) - .ok_or_else(|| PyIndexError::new_err(index)) - } - - /// The input record at `index`, for recording what later stages finish. - pub(crate) fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { - self.project - .inputs - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index)) - } - - /// Record that the native parser skipped the model at `index`: it is not analysed. - pub(crate) fn mark_analysis_skipped(&mut self, index: usize) -> PyResult<()> { - let record: &mut ModelInputRecord = self.input_mut(index)?; - record.sql_validation_enabled = false; - let name: String = record.name.clone(); - record_skipped_analysis(&mut self.project.facts, &name); - Ok(()) - } - - /// The classes model input views project their attributes with. - pub(crate) fn input_classes<'py>(&self, py: Python<'py>) -> PyResult> { - self.input_classes - .as_ref() - .map(|classes| classes.bind(py).clone()) - .ok_or_else(|| PyIndexError::new_err(INPUT_CLASSES_UNSET)) - } - - pub(crate) fn inferred_column<'py>( - &self, - py: Python<'py>, - classes: &CompiledViewClasses<'py>, - row: &InferredRow, - ) -> PyResult> { - let (name, column_type, nullability) = row; - if let Some(column) = self - .inferred_column_projections - .lock() - .map_err(|error| native_compiler_error(error.to_string()))? - .get(row) - { - return Ok(column.bind(py).clone()); - } - let fields = PyDict::new(py); - fields.set_item("name", name)?; - fields.set_item("type", column_type.as_deref())?; - fields.set_item("nullability", classes.nullability.call1((nullability,))?)?; - let column = classes.inferred_column.call((), Some(&fields))?; - self.inferred_column_projections - .lock() - .map_err(|error| native_compiler_error(error.to_string()))? - .insert(row.clone(), column.clone().unbind()); - Ok(column) - } -} - -#[pymethods] -impl NativeCompiledProject { - #[new] - fn new() -> Self { - Self::default() - } - - /// Attach a recorded model's typed analysis; `False` when that model was never recorded. - pub(crate) fn record_model_analysis( - &mut self, - name: &str, - columns: AnalysisPy, - dynamic_proof: Option, - ) -> bool { - let AnalysisPy { - inferred, - inferred_columns, - nullable_columns, - fast_lineage_has_star, - unchecked, - } = columns; - let analysis: ModelAnalysisFact = ModelAnalysisFact { - inferred_columns, - dynamic_proof: dynamic_proof.map( - |ProofPy { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - }| DynamicProofFact { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - }, - ), - inferred, - nullable_columns, - fast_lineage_has_star, - unchecked_output_columns: unchecked, - }; - record_model_analysis(&mut self.project.facts, name, analysis) - } - - /// Record each assembled native model view's final typed analysis; return how many. - pub(crate) fn record_view_analyses( - slf: &Bound<'_, Self>, - views: Vec>, - ) -> PyResult { - let py: Python<'_> = slf.py(); - let mut facts: Vec<(String, ModelAnalysisFact)> = Vec::with_capacity(views.len()); - for view in &views { - facts.push(view.analysis_fact(py)?); - } - let mut owner = slf.borrow_mut(); - let mut recorded: usize = 0; - for (name, analysis) in facts { - recorded += usize::from(record_model_analysis( - &mut owner.project.facts, - &name, - analysis, - )); - } - Ok(recorded) - } - - /// The compile inputs view of this owner's model inputs with these input fields. - pub(crate) fn inputs_view( - slf: Bound<'_, Self>, - fields: Bound<'_, PyDict>, - ) -> PyResult { - NativeInputsView::new(slf.unbind(), &fields) - } - - /// The compiled project view of this owner's models with these project fields. - pub(crate) fn project_view( - slf: Bound<'_, Self>, - fields: Bound<'_, PyDict>, - ) -> PyResult { - NativeProjectView::new(slf.unbind(), &fields) - } - - /// Pair seed declarations with their CSV files and keep them; return each seed's view. - pub(crate) fn attach_seeds( - slf: &Bound<'_, Self>, - declarations: Vec<(Py, Py)>, - seed_files: Vec>, - compile_error: &Bound<'_, PyAny>, - ) -> PyResult> { - let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; - let count: usize = seeds.len(); - slf.borrow_mut().project.seeds = seeds; - Ok((0..count) - .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) - .collect()) - } - - /// Keep the destinations native project assembly resolved, in seed order. - fn record_seed_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { - for (index, namespace) in namespaces.into_iter().enumerate() { - self.project - .seeds - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_namespace(namespace); - } - Ok(()) - } - - /// Attach every discovered source in order with the kept loop; return each source's view. - pub(crate) fn attach_sources( - slf: &Bound<'_, Self>, - rows: Vec, - settings: SourceSettings, - ) -> PyResult> { - (resource_attachment()?.sources)(slf, rows, settings) - } - - /// Attach every SQL, then Python, function with the kept loop; return each one's view. - pub(crate) fn attach_functions( - slf: &Bound<'_, Self>, - rows: (Vec, Vec), - settings: FunctionSettings, - ) -> PyResult> { - (resource_attachment()?.functions)(slf, rows, settings) - } - - /// The orphaned audit factory warnings, missing descriptions and cursor-input errors. - fn resource_diagnostics<'py>( - &self, - py: Python<'py>, - settings: DiagnosticSettings, - ) -> PyResult> { - (resource_attachment()?.diagnostics)(self, py, &settings) - } - - /// Attach every SQL test case, then every scenario, with the kept loop. - pub(crate) fn attach_sql_tests( - slf: &Bound<'_, Self>, - files: Vec, - scenarios: Vec, - settings: SqlTestSettings, - ) -> PyResult { - (resource_attachment()?.sql_tests)(slf, files, scenarios, settings) - } - - /// Attach singular, then resource-attached, audits; return views and override warnings. - pub(crate) fn attach_audits<'py>( - slf: &Bound<'py, Self>, - files: Vec, - settings: AuditSettings, - ) -> PyResult<(Vec, Vec>)> { - (resource_attachment()?.audits)(slf, files, settings) - } - /// Keep the scope dependency keys native project assembly resolved, in audit order. - fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { - for (index, keys) in deps.into_iter().enumerate() { - self.project - .audits - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_scope_deps(keys); - } - Ok(()) - } - - /// The assembled audit at `index`. - fn compiled_audit(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.audits.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) - } - - /// Keep the dependency keys native project assembly resolved, in function order. - fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { - for (index, keys) in deps.into_iter().enumerate() { - self.project - .functions - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_deps(keys); - } - Ok(()) - } - - /// The assembled function at `index`. - fn compiled_function(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.functions.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) - } - - /// Keep the namespaces native project assembly resolved, in source order. - fn record_source_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { - for (index, namespace) in namespaces.into_iter().enumerate() { - self.project - .sources - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_namespace(namespace); - } - Ok(()) - } - - /// The assembled source at `index`. - fn compiled_source(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.sources.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) - } - - /// The assembled seed at `index`. - fn compiled_seed(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.seeds.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) - } - - /// How many lineage graphs this owner's project views built. - fn graph_builds(&self) -> usize { - self.project.graph_builds - } - - /// Whether both projects retained the same facts. - fn __eq__(&self, other: &Self) -> bool { - self.project.facts == other.project.facts - } - - /// The recorded models' names, in production order. - fn model_names(&self) -> Vec { - project_models(&self.project.facts) - .iter() - .map(|model| model.name.clone()) - .collect() - } -} - -pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { - let ModelInputPy( - ModelTextsPy { - name, - relative_path, - query_sql, - authored_query_sql, - authored_sql, - }, - (values, header_keys, layer_schema), - references, - enum_columns, - (enum_declarations, constant_declarations), - ) = model; - let config_values: ConfigValues = config_values(&values); - Ok(CompiledModelFacts { - name, - relative_path, - query_sql: query_sql.into(), - authored_query_sql: authored_query_sql.into(), - authored_sql: authored_sql.into(), - config: ModelConfigFacts { - values: config_values.0, - encoding_error: config_values.1, - header_keys, - layer_schema, - contract: text_value(&values, CONTRACT_KEY)?, - materialized: materialization(&values)?, - incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, - }, - references: references - .into_iter() - .map(|(kind, name, package)| ModelReferenceFact { - kind, - name, - package, - }) - .collect(), - schema: None, - enum_columns, - enum_declarations: declarations(enum_declarations)?, - constant_declarations: declarations(constant_declarations)?, - analysis: None, - typed_calls: Vec::new(), - expansion_passes: Vec::new(), - sql_analysis_skipped: false, - }) -} - -/// The config values, or the encoder's error, kept for the consumer that reads them. -pub(crate) fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { - match object_value(values) { - Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), - Ok(_) => (BTreeMap::new(), None), - Err(error) => (BTreeMap::new(), Some(error.to_string())), - } -} - -/// The `materialized` config value when it is text, `table` when it is absent. -pub(crate) fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { - if values.get_item(MATERIALIZED_KEY)?.is_none() { - return Ok(Some(TABLE_MATERIALIZATION.to_owned())); - } - text_value(values, MATERIALIZED_KEY) -} - -/// A config value when it is text. -pub(crate) fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match values.get_item(key)? { - Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), - _ => Ok(None), - } -} - -pub(crate) fn declarations(rows: Vec>) -> PyResult> { - rows.into_iter().map(declaration).collect() -} - -fn declaration(row: DeclarationPy<'_>) -> PyResult { - let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; - Ok(DeclarationFact { - name, - relative_path, - members: members - .into_iter() - .map(|(name, value)| Ok((name, plain_value(&value)?))) - .collect::>()?, - value: Some(plain_value(&value)?).filter(|value| !value.is_null()), - value_type, - render_as, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 2c8daa3464..fd0b8bc90c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -4,9 +4,13 @@ use crate::bindings::_helpers::{compiled_views, input_loops}; /// Register the compiled project, then its input loops, then its views and records. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::compiled_project::project::register(module)?; + sqlbuild_python_project::compiled_project::main::project::register_project::register_project( + module, + )?; input_loops::registration::register(module)?; compiled_views::registration::register(module)?; crate::bindings::_helpers::location_command::command::register(module)?; - crate::bindings::_helpers::compiled_project::snapshot::register(module) + sqlbuild_python_project::compiled_project::main::snapshot::register_snapshot::register_snapshot( + module, + ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs deleted file mode 100644 index 089dfedbfc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! Project metadata stays native until its immutable snapshot is sent over HTTP. - -use std::collections::BTreeMap; - -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyclass, pymethods}; -use serde_json::Value; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; - -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; - -/// A data-only publication; this object never contains adapters, connections or user callables. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct NativeOrchestrationSnapshot { - pub(crate) input: SnapshotInput, -} - -impl NativeOrchestrationSnapshot { - /// Native callers retain transport errors so they can report an uncertain acknowledgement. - pub(crate) fn publication( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - ) -> PyResult> { - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Ok(result) - }) - } -} - -#[pymethods] -impl NativeOrchestrationSnapshot { - /// Serialize only at the public boundary; native transports can use the retained input. - fn to_json(&self, py: Python<'_>) -> PyResult { - py.detach(|| serde_json::to_string(&self.input)) - .map_err(|error| PyValueError::new_err(error.to_string())) - } - - /// Publish retained data without projecting it through Python, releasing the GIL during IO. - fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { - let record = self - .publication(py, server_url, bearer)? - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - py.detach(|| { - serde_json::to_string(&record) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Retain the verified claim and its independent native heartbeat in the worker interpreter. - fn start_worker( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - identity: (String, String), - ) -> PyResult { - NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) - } -} - -/// Duplicate resource identities must fail, rather than silently replacing an earlier node. -pub(crate) fn insert_node( - mut nodes: BTreeMap, - key: String, - node: SnapshotNode, -) -> PyResult> { - if nodes.contains_key(&key) { - return Err(PyValueError::new_err(format!( - "duplicate snapshot node identity {key}" - ))); - } - nodes.insert(key, node); - Ok(nodes) -} - -pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { - deps.iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect() -} - -pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { - serde_json::json!({ - "database": value.database, - "schema": value.schema, - "name": value.name, - "qualified_name": value.qualified_name, - "logical_database": value.logical_database, - "logical_schema": value.logical_schema, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs deleted file mode 100644 index e1731445c6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ /dev/null @@ -1,750 +0,0 @@ -//! Frozen selection, executor callbacks and completion belong to one verified worker build. - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; - -use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyStringMethods; -use sqlbuild_core::models::{CompiledResourceType, ResourceKind}; -use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; -use sqlbuild_host::models::SqlTestOutcome; -use sqlbuild_host::types::ResultEnum; -use sqlbuild_orchestration::runs::models::{ - NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, -}; - -use crate::bindings::_helpers::compiled_project::constants::{ - NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, -}; -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; -use sqlbuild_python_base::records::main::fields::field::field; - -/// Node keys by snapshot node key, such as the models each test checks. -pub(crate) type NodeKeys = BTreeMap>; - -/// A finished node's outcome, rows affected, error code and error message. -type Completion = (NodeOutcome, Option, (Option, Option)); - -/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. -#[derive(Debug)] -pub(crate) struct WorkerBuild { - pub(crate) worker: Py, - execution: RunExecution, - completed: Mutex>, - checks: Mutex>>, - /// The snapshot's audits and SQL tests, each with the node keys it checks. - attached: BTreeMap>, - /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. - blocked: BTreeMap>, - /// How many of each function's planned tests have still to report; a function is built before its tests run. - untested: Mutex>, - /// A built function's result, held until its tests report so a failing one can fail it instead. - held: Mutex>, - /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. - upstream: BTreeMap>, - /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. - causes: Mutex>, - /// The worst attached outcome of each node, for commands that only run tests or audits. - checked: Mutex>, - /// Whether the command returned, having run every check it planned, rather than being refused or raising. - returned: AtomicBool, - /// Why the worker refused the run before executing anything, reported as the run's status reason. - refusal: Mutex>, -} - -impl WorkerBuild { - pub(crate) fn new( - worker: &Bound<'_, NativeOrchestrationWorker>, - (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), - ) -> PyResult> { - let state = Arc::new(Self { - execution: worker.get().execution(worker.py())?, - worker: worker.clone().unbind(), - completed: Mutex::default(), - checks: Mutex::default(), - attached, - blocked, - untested: Mutex::default(), - held: Mutex::default(), - upstream, - causes: Mutex::default(), - checked: Mutex::default(), - returned: AtomicBool::new(false), - refusal: Mutex::default(), - }); - for key in state.execution.skipped_nodes.keys() { - state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; - } - Ok(state) - } - - pub(crate) fn command(&self) -> RunCommand { - self.execution.claim.request.command - } - - /// The server run this build executes. - pub(crate) fn run_id(&self) -> uuid::Uuid { - self.execution.claim.request.run_id - } - - /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. - fn checks_only(&self) -> bool { - matches!(self.command(), RunCommand::Test | RunCommand::Audit) - } - - /// Record that the command returned instead of being refused or raising. - pub(crate) fn command_returned(&self) { - self.returned.store(true, Ordering::SeqCst); - } - - pub(crate) fn has_work(&self) -> bool { - self.execution - .claim - .request - .selection - .iter() - .any(|key| !self.execution.skipped_nodes.contains_key(key)) - } - - /// Record why the run is refused; the terminal report carries it as the run's status reason. - pub(crate) fn refuse(&self, reason: &str) -> PyResult<()> { - *self.refusal.lock().map_err(|_| poisoned())? = Some(reason.to_owned()); - Ok(()) - } - - pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { - if self.worker.bind(py).get().cancel_requested(py) { - return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); - } - Ok(()) - } - - /// Reject writers outside the frozen write set before any executor connection. - pub(crate) fn enforce_writers( - &self, - py: Python<'_>, - keys: &[String], - ) -> Result<(), pyo3::PyErr> { - self.check_cancelled(py)?; - match keys - .iter() - .find(|key| !self.execution.claim.request.write_set.contains(*key)) - { - Some(key) => Err(PyValueError::new_err(format!( - "planned writer {key} is outside the frozen run write set" - ))), - None => Ok(()), - } - } - - /// Reject an implicitly expanded write set before any executor connection or schema change. - pub(crate) fn enforce_plan_writers( - &self, - pipeline: &Bound<'_, PyAny>, - ) -> Result<(), pyo3::PyErr> { - self.check_cancelled(pipeline.py())?; - let plan = field(pipeline, "plan_output")?; - self.count_planned_tests(&plan)?; - for collection in [ - "model_entries", - "seed_entries", - "function_entries", - "source_load_entries", - ] { - for entry in field(&plan, collection)?.try_iter()? { - let entry = entry?; - let identity = field(&entry, "key")?; - let kind = field(&identity, "resource_type")? - .str()? - .to_str()? - .to_owned(); - let key = format!("{kind}:{}", field(&identity, "name")?.str()?); - if !self.execution.claim.request.write_set.contains(&key) { - return Err(PyValueError::new_err(format!( - "planned writer {key} is outside the frozen run write set" - ))); - } - if kind == ResourceKind::Model.as_str() - && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION - { - let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { - NodeOutcome::Success - } else { - NodeOutcome::Skipped - }; - self.complete(pipeline.py(), &key, (outcome, None, (None, None)))?; - } - } - } - Ok(()) - } - - /// Count each function's tests this build plans, so a built function waits only for tests that will report. - fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { - let mut untested = self.untested.lock().map_err(|_| poisoned())?; - for entry in field(plan, "test_entries")?.try_iter()? { - let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); - for node in self.blocked.get(&test).into_iter().flatten() { - if is_function(node) { - *untested.entry(node.clone()).or_default() += 1; - } - } - } - Ok(()) - } - - /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. - pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { - for key in keys { - self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; - } - Ok(()) - } - - /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. - pub(crate) fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { - for (key, why) in nodes { - self.complete( - py, - key, - ( - NodeOutcome::Skipped, - None, - (None, Some(format!("held back: {why}"))), - ), - )?; - } - Ok(()) - } - - /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. - fn complete( - &self, - py: Python<'_>, - key: &str, - (outcome, rows_affected, (error_code, error_message)): ( - NodeOutcome, - Option, - (Option, Option), - ), - ) -> PyResult<()> { - if !self.execution.claim.request.selection.contains(key) { - return Ok(()); - } - if self - .completed - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - return Ok(()); - } - let skipped_because_of = match outcome { - NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { - Some(cause) => Some(cause.clone()), - None if !self.checks_only() => self.failed_upstream(key)?, - None => None, - }, - _ => None, - }; - self.worker.bind(py).get().change( - py, - &format!("node-finished/{key}"), - RunChange::NodeFinished { - node_key: key.into(), - outcome, - rows_affected, - error_code, - skipped_because_of: skipped_because_of.clone(), - error_message: error_message.as_deref().map(redact_error_text), - }, - )?; - let cause = match outcome { - NodeOutcome::Failed => Some(key.to_owned()), - _ => skipped_because_of, - }; - if let Some(cause) = cause { - self.causes - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), cause); - } - self.completed - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), outcome); - Ok(()) - } - - pub(crate) fn start( - &self, - py: Python<'_>, - name: &str, - resource_kind: &Bound<'_, PyAny>, - ) -> PyResult<()> { - self.check_cancelled(py)?; - let kind = field(resource_kind, "value")?.extract::()?; - let kind = match kind.as_str() { - "table" | "view" | "snapshot" | "custom" => "model", - kind => kind, - }; - self.start_key(py, &format!("{kind}:{name}")) - } - - /// Report a selected node as started unless its outcome is final, such as a planner skip. - pub(crate) fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { - self.check_cancelled(py)?; - if self.execution.claim.request.selection.contains(key) - && !self - .completed - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - self.worker.bind(py).get().change( - py, - &format!("node-started/{key}"), - RunChange::NodeStarted { - node_key: key.into(), - }, - )?; - } - Ok(()) - } - - /// Report a selected node's outcome when the command reports no executor result for it. - pub(crate) fn finish_key( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { - self.complete(py, key, (outcome, None, (None, None))) - } - - /// A test or audit the snapshot records has started; a selected SQL test reports as a node. - pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { - self.check_cancelled(py)?; - let Some(nodes) = self.attached.get(key) else { - return Ok(()); - }; - if self.execution.claim.request.selection.contains(key) { - return self.start_key(py, key); - } - if self - .checks - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - return Ok(()); - } - self.worker.bind(py).get().change( - py, - &format!("check-started/{key}"), - RunChange::CheckStarted { - node_key: key.into(), - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), None); - if self.checks_only() { - for node in nodes { - self.start_key(py, node)?; - } - } - Ok(()) - } - - /// A test or audit finished; a check the snapshot does not record is not reported. - pub(crate) fn finish_attached( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { - let Some(nodes) = self.attached.get(key) else { - return Ok(()); - }; - if outcome != NodeOutcome::Skipped { - self.start_attached(py, key)?; - } - let outcome = if outcome == NodeOutcome::Failed - && self.worker.bind(py).get().refresh_cancellation(py)? - { - NodeOutcome::Cancelled - } else { - outcome - }; - if self.execution.claim.request.selection.contains(key) { - self.complete(py, key, (outcome, None, (None, None)))?; - } else { - self.finish_check(py, key, outcome)?; - } - if outcome == NodeOutcome::Skipped { - return Ok(()); - } - let mut checked = self.checked.lock().map_err(|_| poisoned())?; - for node in nodes { - let worst = checked.get(node).map_or(outcome, |previous| { - if severity(outcome) > severity(*previous) { - outcome - } else { - *previous - } - }); - checked.insert(node.clone(), worst); - } - Ok(()) - } - - /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. - fn failed_upstream(&self, key: &str) -> PyResult> { - let causes = self.causes.lock().map_err(|_| poisoned())?; - let mut seen: BTreeSet<&str> = BTreeSet::from([key]); - let mut pending: VecDeque<&str> = VecDeque::from([key]); - while let Some(node) = pending.pop_front() { - for upstream in self.upstream.get(node).into_iter().flatten() { - if let Some(cause) = causes.get(upstream) { - return Ok(Some(cause.clone())); - } - if seen.insert(upstream) { - pending.push_back(upstream); - } - } - } - Ok(None) - } - - /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. - fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { - let py = result.py(); - let outcome: String = field(result, "outcome")?.str()?.extract()?; - let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); - let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; - let name: String = field(result, "test_name")?.extract()?; - let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); - let error: Option = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message: Option = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - let node = match (passed, cancelled) { - (true, _) => NodeOutcome::Success, - (false, true) => NodeOutcome::Cancelled, - (false, false) => NodeOutcome::Failed, - }; - self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; - if !passed && !cancelled { - for model in self.blocked.get(&key).into_iter().flatten() { - self.complete( - py, - model, - (NodeOutcome::Failed, None, (error.clone(), message.clone())), - )?; - } - } - self.release_tested(py, &key) - } - - /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. - fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { - let mut released: Vec<(String, Completion)> = Vec::new(); - { - let mut untested = self.untested.lock().map_err(|_| poisoned())?; - let mut held = self.held.lock().map_err(|_| poisoned())?; - for node in self.blocked.get(test).into_iter().flatten() { - let Some(remaining) = untested.get_mut(node) else { - continue; - }; - *remaining = remaining.saturating_sub(1); - if *remaining == 0 - && let Some(completion) = held.remove(node) - { - released.push((node.clone(), completion)); - } - } - } - for (node, completion) in released { - self.complete(py, &node, completion)?; - } - Ok(()) - } - - /// Publish a node's result, or hold a built function's while its tests have still to report. - fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> { - let waiting = !self.checks_only() - && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) - && self - .untested - .lock() - .map_err(|_| poisoned())? - .get(key) - .is_some_and(|remaining| *remaining > 0); - if waiting { - self.held - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), completion); - return Ok(()); - } - self.complete(py, key, completion) - } - - /// Existing native progress callbacks publish selected-node evidence without Python relays. - pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { - if key.starts_with(SQL_TEST_KEY_PREFIX) { - return self.finished_test(result); - } - if self.execution.claim.request.selection.contains(key) { - let status: String = field(result, "status")?.str()?.extract()?; - let outcome = match status.as_str() { - "success" => { - if result.hasattr("warning_messages")? - && field(result, "warning_messages")?.is_truthy()? - { - NodeOutcome::Warn - } else { - NodeOutcome::Success - } - } - "failed" => { - if self - .worker - .bind(result.py()) - .get() - .refresh_cancellation(result.py())? - { - NodeOutcome::Cancelled - } else { - NodeOutcome::Failed - } - } - "skipped" => NodeOutcome::Skipped, - _ => { - return Err(PyValueError::new_err(format!( - "unsupported worker node outcome {status}" - ))); - } - }; - let rows = match result.getattr_opt("rows_affected")? { - Some(value) => value.extract()?, - None => None, - }; - let error = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; - } - Ok(()) - } - - pub(crate) fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { - self.check_cancelled(py)?; - let key = format!("check:{name}"); - self.worker.bind(py).get().change( - py, - &format!("check-started/{key}"), - RunChange::CheckStarted { - node_key: key.clone(), - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key, None); - Ok(()) - } - - /// A completion without our actual-execution start callback is a blocked check, not a failure. - pub(crate) fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { - let name: String = field(result, "node_name")?.extract()?; - let key = format!("check:{name}"); - let started = self - .checks - .lock() - .map_err(|_| poisoned())? - .contains_key(&key); - let outcome = if !started { - NodeOutcome::Skipped - } else if field(result, "passed")?.is_truthy()? { - NodeOutcome::Success - } else if field(result, "warned")?.is_truthy()? { - NodeOutcome::Warn - } else if self - .worker - .bind(result.py()) - .get() - .refresh_cancellation(result.py())? - { - NodeOutcome::Cancelled - } else { - NodeOutcome::Failed - }; - self.finish_check(result.py(), &key, outcome) - } - - fn finish_check(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { - self.worker.bind(py).get().change( - py, - &format!("check-finished/{key}"), - RunChange::CheckFinished { - node_key: key.into(), - outcome, - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), Some(outcome)); - Ok(()) - } - - /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. - pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { - let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; - let checks_only = self.checks_only(); - if checks_only && !cancelled { - let checked = self.checked.lock().map_err(|_| poisoned())?.clone(); - let returned = self.returned.load(Ordering::SeqCst); - for key in &self.execution.claim.request.selection { - let seen = checked.get(key).copied(); - let outcome = if returned { - Some(seen.unwrap_or(NodeOutcome::Success)) - } else { - seen.filter(|outcome| { - matches!(outcome, NodeOutcome::Failed | NodeOutcome::Cancelled) - }) - }; - if let Some(outcome) = outcome { - self.complete(py, key, (outcome, None, (None, None)))?; - } - } - } - let held: Vec<(String, Completion)> = - std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) - .into_iter() - .collect(); - for (key, completion) in held { - self.complete(py, &key, completion)?; - } - let returned = self.returned.load(Ordering::SeqCst); - let build = self.command().is_build(); - let source_needs_no_run = |key: &str| build && key.starts_with("source:"); - for key in &self.execution.claim.request.selection { - let outcome = if cancelled { - NodeOutcome::Cancelled - } else if returned && source_needs_no_run(key) { - NodeOutcome::Success - } else { - NodeOutcome::Skipped - }; - self.complete(py, key, (outcome, None, (None, None)))?; - } - let outcome = { - let completed = self.completed.lock().map_err(|_| poisoned())?; - let outcomes: Vec = completed.values().copied().collect(); - let good = completed.iter().any(|(key, value)| { - matches!(value, NodeOutcome::Success | NodeOutcome::Warn) - && !source_needs_no_run(key) - }); - let bad = outcomes.iter().any(|value| { - matches!( - value, - NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled - ) - }); - if cancelled { - RunOutcome::Cancelled - } else if bad && good { - RunOutcome::Partial - } else if bad || exit_code != 0 { - RunOutcome::Failed - } else if outcomes.contains(&NodeOutcome::Warn) { - RunOutcome::Warn - } else { - RunOutcome::Success - } - }; - Ok(outcome) - } - - /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. - pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { - let outcome = self.outcome(py, exit_code)?; - let unfinished: Vec = self - .checks - .lock() - .map_err(|_| poisoned())? - .iter() - .filter(|(_, outcome)| outcome.is_none()) - .map(|(key, _)| key.clone()) - .collect(); - for key in unfinished { - self.finish_check( - py, - &key, - if outcome == RunOutcome::Cancelled { - NodeOutcome::Cancelled - } else { - NodeOutcome::Skipped - }, - )?; - } - let reason: String = self - .refusal - .lock() - .map_err(|_| poisoned())? - .clone() - .unwrap_or_else(|| "worker executor completed and closed its connections".into()); - self.worker.bind(py).get().change( - py, - "worker-finished", - RunChange::Finished { - outcome, - reason, - exit_code: Some(i64::from(exit_code)), - }, - )?; - Ok(outcome) - } -} - -/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. -pub(crate) type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); - -/// Failure outranks cancellation, then warnings, then success. -fn severity(outcome: NodeOutcome) -> u8 { - match outcome { - NodeOutcome::Skipped => 0, - NodeOutcome::Success => 1, - NodeOutcome::Warn => 2, - NodeOutcome::Cancelled => 3, - NodeOutcome::Failed => 4, - } -} - -/// Whether a snapshot node key names a scalar or table function. -fn is_function(key: &str) -> bool { - key.split_once(':').is_some_and(|(kind, _)| { - [CompiledResourceType::Udf, CompiledResourceType::TableFn] - .iter() - .any(|function| function.as_str() == kind) - }) -} - -fn poisoned() -> pyo3::PyErr { - PyRuntimeError::new_err("worker build outcome lock is unavailable") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs deleted file mode 100644 index aa2342912c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs +++ /dev/null @@ -1,217 +0,0 @@ -//! A worker interpreter retains the native claim owner; it never opens the control database. - -use std::sync::Mutex; - -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{PyRef, PyResult, Python}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_orchestration::runs::models::RunChange; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; -use sqlbuild_orchestration::worker::main::start_worker::start_worker; -use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; -use uuid::Uuid; - -use crate::bindings::_helpers::compiled_project::constants::{REPORT_ATTEMPTS, STORE_BUSY_STATUS}; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; - -struct WorkerOwner { - runtime: tokio::runtime::Runtime, - client: WorkerClient, - session: WorkerSession, -} - -/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct NativeOrchestrationWorker { - owner: Mutex>, -} - -impl NativeOrchestrationWorker { - /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. - fn with_owner( - &self, - py: Python<'_>, - work: impl FnOnce(&WorkerOwner) -> PyResult + Send, - ) -> PyResult { - py.detach(|| { - let owner = self.owner.lock().map_err(|_| unavailable())?; - work(owner.as_ref().ok_or_else(unavailable)?) - }) - } - - /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. - pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - if owner.session.cancel_requested() { - return Ok(true); - } - owner - .runtime - .block_on(owner.client.heartbeat()) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Native execution reads the retained request, never caller-provided selection or settings. - pub(crate) fn execution( - &self, - py: Python<'_>, - ) -> PyResult { - self.with_owner(py, |owner| Ok(owner.session.execution.clone())) - } - - /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. - pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { - self.with_owner(py, |owner| { - let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); - owner.runtime.block_on(async { - let mut attempts = 0; - loop { - attempts += 1; - match owner.client.report(event_id, change.clone()).await { - Ok(_) => return Ok(()), - Err(error) - if attempts < REPORT_ATTEMPTS - && match &error { - WorkerError::Transport(_) => true, - WorkerError::Rejected(status) => { - status.as_u16() == STORE_BUSY_STATUS - } - _ => false, - } => - { - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - } - Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), - } - } - }) - }) - } - - /// Verify this retained baseline before the caller applies run settings or opens a warehouse. - pub(crate) fn start( - snapshot: &NativeOrchestrationSnapshot, - py: Python<'_>, - (server_url, bearer): (&str, &str), - (run_id, claim_id): (String, String), - ) -> PyResult { - let identity = ( - Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - ); - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let connected = runtime.block_on(async { - let client = connect_worker(server_url, bearer, identity)?; - let session = start_worker(&client, &snapshot.input).await?; - Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) - }); - match connected { - Ok((client, session)) => Ok(NativeOrchestrationWorker { - owner: Mutex::new(Some(WorkerOwner { - runtime, - client, - session, - })), - }), - Err(error) => { - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Err(PyRuntimeError::new_err(error.to_string())) - } - } - }) - } -} - -#[pymethods] -impl NativeOrchestrationWorker { - /// The frozen request supplies selection and settings only after baseline verification. - fn execution_json(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - serde_json::to_string(&owner.session.execution) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Cancellation remains sticky; a closed worker cannot authorize further work. - pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { - py.detach(|| { - self.owner.lock().map_or(true, |owner| { - owner - .as_ref() - .is_none_or(|owner| owner.session.cancel_requested()) - }) - }) - } - - /// Retry an uncertain report with the identical event ID and change, never a new identity. - fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { - let event_id = - Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; - let change: RunChange = serde_json::from_str(change_json) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - self.with_owner(py, |owner| { - owner - .runtime - .block_on(owner.client.report(event_id, change)) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { - if slf.owner.lock().map_err(|_| unavailable())?.is_none() { - return Err(unavailable()); - } - Ok(slf) - } - - fn __exit__( - &self, - py: Python<'_>, - _kind: &pyo3::Bound<'_, pyo3::PyAny>, - _value: &pyo3::Bound<'_, pyo3::PyAny>, - _traceback: &pyo3::Bound<'_, pyo3::PyAny>, - ) -> PyResult<()> { - self.close(py) - } - - /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. - fn close(&self, py: Python<'_>) -> PyResult<()> { - py.detach(|| { - let mut owner = self.owner.lock().map_err(|_| unavailable())?; - if let Some(active) = owner.take() { - active.close(); - } - Ok(()) - }) - } -} - -fn unavailable() -> pyo3::PyErr { - PyRuntimeError::new_err("native worker session is closed or unavailable") -} - -impl WorkerOwner { - fn close(self) { - drop(self.session); - drop(self.client); - self.runtime - .shutdown_timeout(std::time::Duration::from_secs(1)); - } -} - -impl Drop for NativeOrchestrationWorker { - /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. - fn drop(&mut self) { - if let Ok(owner) = self.owner.get_mut() - && let Some(owner) = owner.take() - { - owner.close(); - } - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs deleted file mode 100644 index 24e937b063..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ /dev/null @@ -1,942 +0,0 @@ -//! A compiled model, read from the native compiled project that owns it. - -use sqlbuild_project::models as project_models; -use sqlbuild_project::models::{DestinationFact, ModelMetadataTexts}; -use sqlbuild_project::types::InferredRow; -use sqlbuild_python_front::model_config::models::HeaderAudits; -use std::sync::Arc; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{FromPyObject, pyclass, pymethods}; - -use sqlbuild_analysis::assembly::analysis_session::main::contract_proof_json::contract_proof_json; -use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; -use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; -use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, DynamicProofFact, ModelAnalysisFact, -}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; -use crate::bindings::_helpers::compiled_views::compiled_model_analysis::{ - AnalysisRecord, AnalysisViews, ProofArgument, SessionArgument, analysis_record, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, ModelInputRecord, model_relation_key_row, query_sql_text, read_input, - read_input_with_owner, references, rejected_opt_out, schema_entry, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::RelationKeyRow; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::models::NativeModelFile; -use sqlbuild_python_front::model_config::models::HeaderColumns; - -const DATABASE_KEY: &str = "database"; -const SCHEMA_KEY: &str = "schema"; -const ALIAS_KEY: &str = "alias"; -const VALUES_ATTRIBUTE: &str = "values"; -const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; -const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; -const MODEL_RESOURCE: &str = "model"; -const SQL_ANALYSIS_KEY: &str = "sql_analysis"; -const TAGS_KEY: &str = "tags"; -const NULLABLE: &str = "nullable"; - -/// A model's SQL rule inputs; `excluded` is `None` when its config values could not be read natively. -pub(crate) struct SqlRuleInputs { - pub(crate) posix_path: String, - pub(crate) excluded: Option, - pub(crate) key: Option, -} - -/// What a model's SQL rule cache key reads. -pub(crate) struct SqlRuleKey { - pub(crate) authored_sql: String, - pub(crate) query_sql: String, - /// The canonical JSON text of the model's dynamic pivot proof. - pub(crate) contract: Option, -} - -/// A schema's entry audits, column specs and each audited column's index and name. -pub(crate) type AuditedSchema = (Arc, Arc, Vec<(usize, String)>); - -/// `(name, type)` of a declared column. -pub(crate) type DeclaredRow = (String, Option); - -/// `(database, schema, name)` of a relation. -pub(crate) type DestinationRow = (Option, Option, String); - -/// `(output column, [(resource name, column name)])` of one lineage output. -pub(crate) type LineageInput = (String, Vec<(String, String)>); - -/// The Python classes a compiled model's projected attributes are built with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct CompiledViewClasses<'py> { - object_key: Bound<'py, PyAny>, - relation_location: Bound<'py, PyAny>, - pub(crate) inferred_column: Bound<'py, PyAny>, - pub(crate) nullability: Bound<'py, PyAny>, - pub(crate) compiler_diagnostic: Bound<'py, PyAny>, - pub(crate) source_location: Bound<'py, PyAny>, - pub(crate) compact_lineage_facts: Bound<'py, PyAny>, - pub(crate) lineage_column_fact: Bound<'py, PyAny>, - pub(crate) lineage_source_fact: Bound<'py, PyAny>, - pub(crate) contract_proof: Bound<'py, PyAny>, - pub(crate) family_proof: Bound<'py, PyAny>, -} - -/// What assembly and the semantic checks finished for one compiled model. -pub(crate) type CompiledRecord = project_models::CompiledRecord; - -/// A compiled model's Python attribute projections, each built once on first access. -pub(crate) struct CompiledProjections { - key: PyOnceLock>, - deps: PyOnceLock>, -} - -impl Default for CompiledProjections { - fn default() -> Self { - Self { - key: PyOnceLock::new(), - deps: PyOnceLock::new(), - } - } -} - -/// What later stages changed for one view of a model; the owner's record stays as assembled. -#[derive(Default)] -struct ModelOverlay { - binding_diagnostics: Option>, - /// Output columns whose types semantic recovery forgot, left unchecked by the contract. - poisoned: Vec, - destination: Option, -} - -impl ModelOverlay { - fn clone_ref(&self, py: Python<'_>) -> Self { - Self { - binding_diagnostics: self - .binding_diagnostics - .as_ref() - .map(|diagnostics| diagnostics.clone_ref(py)), - poisoned: self.poisoned.clone(), - destination: self.destination.clone(), - } - } -} - -/// One view's projections of attributes its overlay can change, each built once. -struct ViewProjections { - destination: PyOnceLock>, - inferred_columns: PyOnceLock>, -} - -impl Default for ViewProjections { - fn default() -> Self { - Self { - destination: PyOnceLock::new(), - inferred_columns: PyOnceLock::new(), - } - } -} - -/// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. -#[derive(FromPyObject)] -pub(crate) struct CompiledArguments( - Vec<(String, String)>, - Option, - Option>, - Option, - Vec, -); - -/// The compiled record of one model, with its configured destination. -pub(crate) fn compiled_record( - py: Python<'_>, - input: &ModelInputRecord, - name: &str, - arguments: CompiledArguments, -) -> PyResult { - let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; - let analysis: AnalysisRecord = - analysis_record(py, input, session, (fallback, standalone_proof))?; - let config: &Bound<'_, PyAny> = input.host.config.bind(py); - let values: Bound<'_, PyDict> = native_field(config, VALUES_ATTRIBUTE)?.downcast_into()?; - Ok(CompiledRecord { - deps, - destination: DestinationFact { - database: text_value(&values, DATABASE_KEY)?, - schema: text_value(&values, SCHEMA_KEY)?, - name: text_value(&values, ALIAS_KEY)?.unwrap_or_else(|| name.to_owned()), - qualified_name: None, - logical_schema: native_field(config, LOGICAL_SCHEMA_ATTRIBUTE)?.extract()?, - logical_database: native_field(config, LOGICAL_DATABASE_ATTRIBUTE)?.extract()?, - }, - analysis, - macro_deps, - host: CompiledProjections::default(), - }) -} - -fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match values.get_item(key)? { - Some(value) if value.is_instance_of::() => { - Ok(Some(value.extract()?)) - } - _ => Ok(None), - } -} - -/// The destination with adapter defaults filled in; `render` renders its qualified name. -pub(crate) fn defaulted_destination( - current: &DestinationFact, - defaults: (Option, Option), - render: &Bound<'_, PyAny>, -) -> PyResult { - let (default_schema, default_database) = defaults; - let schema: Option = current.schema.clone().or(default_schema); - let database: Option = current.database.clone().or(default_database); - let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); - keywords.set_item(DATABASE_KEY, &database)?; - keywords.set_item(SCHEMA_KEY, &schema)?; - keywords.set_item("name", ¤t.name)?; - let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; - Ok(DestinationFact { - database, - schema, - name: current.name.clone(), - qualified_name, - logical_schema: current.logical_schema.clone(), - logical_database: current.logical_database.clone(), - }) -} - -/// One compiled model, projected from the owner on demand. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledModel { - owner: Py, - index: usize, - overlay: ModelOverlay, - projections: ViewProjections, -} - -impl NativeCompiledModel { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self::with_overlay(owner, index, ModelOverlay::default()) - } - - fn with_overlay(owner: Py, index: usize, overlay: ModelOverlay) -> Self { - Self { - owner, - index, - overlay, - projections: ViewProjections::default(), - } - } - - fn changed(&self, py: Python<'_>, change: impl FnOnce(&mut ModelOverlay)) -> Self { - let mut overlay: ModelOverlay = self.overlay.clone_ref(py); - change(&mut overlay); - Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) - } - - pub(crate) fn read<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, - ) -> PyResult { - read_input(&self.owner, self.index, py, read) - } - - fn compiled<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&CompiledRecord, Python<'py>) -> PyResult, - ) -> PyResult { - self.read(py, |record, py| read(compiled_of(record)?, py)) - } - - /// Read the effective destination, including namespace overlays, without a Python projection. - pub(crate) fn read_destination( - &self, - py: Python<'_>, - read: impl FnOnce(&DestinationFact) -> PyResult, - ) -> PyResult { - self.compiled(py, |compiled, _| { - read( - self.overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination), - ) - }) - } - - /// This view's typed analysis as the owner's model facts read it, and the model's name. - pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { - self.read(py, |record, _| { - let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; - let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); - let mut inferred_columns: Vec<(String, Option)> = - Vec::with_capacity(rows.len()); - for (name, column_type, _) in rows { - let poisoned: bool = self.overlay.poisoned.contains(name); - inferred_columns.push((name.clone(), column_type.clone().filter(|_| !poisoned))); - } - let dynamic_proof: Option = - analysis.proof.as_ref().map(|proof| DynamicProofFact { - output_proven: proof.output_proven, - bare_dynamic_pivot: proof.bare_dynamic_pivot, - families: casefolded_families(&proof.families), - failure_reason: proof.failure_reason.clone(), - named_families: proof.families.clone(), - }); - let fact: ModelAnalysisFact = ModelAnalysisFact { - inferred_columns, - dynamic_proof, - inferred: analysis.inferred.is_some(), - nullable_columns: rows - .iter() - .map(|(_, _, nullability)| nullability == NULLABLE) - .collect(), - fast_lineage_has_star: analysis.has_star, - unchecked_output_columns: self.overlay.poisoned.clone(), - }; - Ok((model_name(record), fact)) - }) - } - - /// Each output column of the kept lineage with its `(resource name, column)` sources. - pub(crate) fn lineage_inputs(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .lineage_rows() - .map(|rows| rows.iter().map(lineage_input).collect())) - }) - } - - /// Whether the model's dynamic pivot proof proved its output columns. - pub(crate) fn output_proven(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .proof - .as_ref() - .is_some_and(|proof| proof.output_proven)) - }) - } - - /// Output columns whose types semantic recovery forgot. - pub(crate) fn unchecked_columns(&self) -> &[String] { - &self.overlay.poisoned - } - - /// The kept lineage rows with their transform and confidence codes, or `None`. - pub(crate) fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) - }) - } - - /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. - pub(crate) fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .proof - .as_ref() - .filter(|proof| proof.output_proven) - .map(|proof| proof.input_relations.clone())) - }) - } - - /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. - pub(crate) fn declared_columns(&self, py: Python<'_>) -> PyResult>> { - self.read(py, |record, py| { - record - .schema - .as_ref() - .map(|schema| schema.host.columns.project(py).map(Bound::unbind)) - .transpose() - }) - } - - /// The attached schema's entry audits, its column specs and each audited column's index and name. - pub(crate) fn schema_audits_and_columns( - &self, - py: Python<'_>, - ) -> PyResult> { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - let (Some(schema), Some(fact)) = (&record.schema, owner.input_schema_fact(record)) - else { - return Ok(None); - }; - let audited: Vec<(usize, String)> = fact - .columns - .iter() - .enumerate() - .filter(|(_, column)| !column.audit_definitions.is_empty()) - .map(|(index, column)| (index, column.name.clone())) - .collect(); - Ok(Some(( - schema.host.audits.clone(), - schema.host.columns.clone(), - audited, - ))) - }) - } - - /// What SQL rules select the model by, plus its cache key inputs when `keyed`. - pub(crate) fn sql_rule_inputs(&self, py: Python<'_>, keyed: bool) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - let facts: &CompiledModelFacts = - model_facts(owner.facts(), &record.name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - let excluded: Option = match &facts.config.encoding_error { - Some(_) => None, - None => Some( - facts.config.values.get(SQL_ANALYSIS_KEY) - == Some(&serde_json::Value::Bool(false)), - ), - }; - let key = keyed - .then(|| -> PyResult { - Ok(SqlRuleKey { - authored_sql: facts.authored_sql.to_string(), - query_sql: facts.query_sql.to_string(), - contract: compiled_of(record)? - .analysis - .proof - .as_ref() - .map(contract_proof_json), - }) - }) - .transpose()?; - Ok(SqlRuleInputs { - posix_path: facts.relative_path.clone(), - excluded, - key, - }) - }) - } - - /// The model's name, `str(relative_path)` and authored file text. - pub(crate) fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { - self.read(py, |record, py| { - let model_file: &NativeModelFile = record.host.model_file.get(); - Ok(( - model_name(record), - model_file.relative_text(py)?, - Arc::clone(model_file.contents_text()), - )) - }) - } - - /// The name, compiled query and authored file of the model, and whether it calls a function. - pub(crate) fn metadata_texts( - &self, - py: Python<'_>, - function_kinds: &[&str], - ) -> PyResult { - self.read(py, |record, _| { - Ok(ModelMetadataTexts { - name: model_name(record), - query_sql: record.query_sql.to_string(), - authored_sql: record.host.model_file.get().contents_text().to_string(), - calls_functions: record - .references - .iter() - .any(|reference| function_kinds.contains(&reference.kind.as_str())), - }) - }) - } - - /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. - pub(crate) fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { - self.read(py, |record, py| { - Ok(( - record.host.model_file.get().posix_path(py)?, - Arc::clone(&record.query_sql), - )) - }) - } - - /// The model's authored file text and expanded query, from its recorded facts. - pub(crate) fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { - let name: String = self.name(py)?; - let owner = self.owner.bind(py).borrow(); - let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) - } - - fn model_file( - &self, - py: Python<'_>, - read: impl FnOnce(&NativeModelFile) -> PyResult, - ) -> PyResult { - self.read(py, |record, _| read(record.host.model_file.get())) - } -} - -/// A lineage row's output column with its `(resource name, column)` sources. -fn lineage_input(row: &LineageRow) -> LineageInput { - let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); - for (_, resource, column) in &row.sources { - sources.push((resource.clone(), column.clone())); - } - (row.output_column.clone(), sources) -} - -/// Each family's name casefolded as Python's, with its inferred type. -fn casefolded_families(families: &[(String, Option)]) -> Vec<(String, Option)> { - families - .iter() - .map(|(name, kind)| (str::to_lowercase(name), kind.clone())) - .collect() -} - -/// The compiled record of an input, which assembly records before any view is handed out. -fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { - record - .compiled - .as_ref() - .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) -} - -#[pymethods] -impl NativeCompiledModel { - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - let compiled: &CompiledRecord = compiled_of(record)?; - compiled - .host - .key - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record)) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { - compiled - .host - .deps - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let mut keys: Vec> = Vec::with_capacity(compiled.deps.len()); - for (kind, name) in &compiled.deps { - keys.push(object_key(&classes.compiled, kind, name)?); - } - Ok::<_, pyo3::PyErr>(PyTuple::new(py, keys)?.unbind()) - }) - .map(|deps| deps.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _| Ok(model_name(record))) - } - - #[getter] - pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { - self.model_file(py, |file| file.relative_path(py)) - } - - #[getter] - pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(query_sql_text(py, record))) - } - - #[getter] - pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) - } - - #[getter] - pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.read_destination(py, |destination| { - self.projections - .destination - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - relation_location(&classes.compiled, destination) - }) - .map(|destination| destination.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn references(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| references(&classes, record)) - } - - /// Each reference as `(kind, name, package, call argument count)`. - #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| { - Ok(record - .references - .iter() - .cloned() - .map(reference_row_tuple) - .collect()) - }) - } - - /// The model's declared key facts, for deriving its unique keys. - #[getter] - pub(crate) fn relation_key_row(&self, py: Python<'_>) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - model_relation_key_row(owner, record, py) - }) - } - - /// `(resource type, name)` of the model's key. - #[getter] - pub(crate) fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { - self.read(py, |record, _| { - Ok((MODEL_RESOURCE.to_owned(), model_name(record))) - }) - } - - /// `(resource type, name)` of each dependency. - #[getter] - pub(crate) fn dep_rows(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) - } - - /// `(database, schema, name)` of the model's destination. - #[getter] - pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.read_destination(py, |destination| { - Ok(( - destination.database.clone(), - destination.schema.clone(), - destination.name.clone(), - )) - }) - } - - /// Each inferred column as `(name, type, nullability)`, or `None` when none were inferred. - #[getter] - pub(crate) fn inferred_rows(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - let Some(rows) = &compiled.analysis.inferred else { - return Ok(None); - }; - let mut shown: Vec = Vec::with_capacity(rows.len()); - for (name, column_type, nullability) in rows { - let poisoned: bool = self.overlay.poisoned.contains(name); - let column_type: Option = if poisoned { None } else { column_type.clone() }; - shown.push((name.clone(), column_type, nullability.clone())); - } - Ok(Some(shown)) - }) - } - - /// Each declared column of the attached schema as `(name, type)`, or `None` without one. - #[getter] - pub(crate) fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { - let name: String = self.name(py)?; - let owner = self.owner.bind(py).borrow(); - let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - let Some(schema) = &facts.schema else { - return Ok(None); - }; - Ok(Some( - schema - .columns - .iter() - .map(|column| (column.name.clone(), column.data_type.clone())) - .collect(), - )) - } - - /// Whether the attached schema declares dynamic columns. - #[getter(has_dynamic_columns)] - pub(crate) fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _py| { - Ok(record - .schema - .as_ref() - .is_some_and(|schema| !schema.host.dynamic_columns.specs.is_empty())) - }) - } - - /// Inferred columns, else the attached schema's declared columns, else zero. - #[getter] - pub(crate) fn output_column_count(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _py| { - if let Some(rows) = &compiled_of(record)?.analysis.inferred { - return Ok(rows.len()); - } - Ok(record - .schema - .as_ref() - .map_or(0, |schema| schema.host.columns.specs.len())) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| schema_entry(&classes, record)) - } - - #[getter] - pub(crate) fn inferred_columns(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { - self.projections - .inferred_columns - .get_or_try_init(py, || { - let Some(rows) = &compiled.analysis.inferred else { - return Ok::<_, pyo3::PyErr>(py.None()); - }; - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let mut columns: Vec> = Vec::with_capacity(rows.len()); - for row in rows { - let poisoned_row: InferredRow; - let row = if self.overlay.poisoned.contains(&row.0) { - poisoned_row = (row.0.clone(), None, row.2.clone()); - &poisoned_row - } else { - row - }; - columns.push(self.owner.bind(py).borrow().inferred_column( - py, - &classes.compiled, - row, - )?); - } - Ok(PyTuple::new(py, columns)?.into_any().unbind()) - }) - .map(|columns| columns.clone_ref(py)) - }) - } - - /// The upstream source count of the model's lineage facts, `None` without them. - pub(crate) fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) - } - - #[getter] - pub(crate) fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.compiled(py, |compiled, py| { - compiled.analysis.lineage_object(py, &classes.compiled) - }) - } - - #[getter] - pub(crate) fn fast_lineage_has_star(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.has_star)) - } - - #[getter] - pub(crate) fn fast_lineage_star_resolved(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.star_resolved)) - } - - #[getter] - pub(crate) fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.model_file(py, |file| Ok(file.contents(py))) - } - - #[getter] - pub(crate) fn authored_query_sql<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.model_file(py, |file| Ok(file.query_sql(py))) - } - - #[getter] - pub(crate) fn output_column_locations<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.model_file(py, |file| file.output_column_locations(py)) - } - - #[getter] - pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { - self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) - } - - #[getter] - pub(crate) fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { - self.compiled(py, |compiled, py| PyTuple::new(py, &compiled.macro_deps)) - } - - #[getter] - pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.enum_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.constant_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) - } - - #[getter] - pub(crate) fn binding_diagnostics(&self, py: Python<'_>) -> PyResult> { - if let Some(diagnostics) = &self.overlay.binding_diagnostics { - return Ok(diagnostics.clone_ref(py)); - } - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, py| { - let name: String = model_name(record); - let path: Bound<'_, PyAny> = record.host.model_file.get().path_objects(py)?.1; - compiled_of(record)? - .analysis - .binding_tuple(py, &classes.compiled, (&name, &path)) - }) - } - - #[getter] - pub(crate) fn binding_validated(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.binding_validated)) - } - - #[getter] - pub(crate) fn dynamic_column_contract(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.compiled(py, |compiled, py| { - compiled.analysis.proof_object(py, &classes.compiled) - }) - } - - #[getter] - pub(crate) fn unchecked_output_columns<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - PyFrozenSet::new(py, &self.overlay.poisoned) - } - - #[getter] - pub(crate) fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(rejected_opt_out(record, py))) - } - - /// This model with its final binding diagnostics. - fn with_binding_diagnostics(&self, py: Python<'_>, diagnostics: Py) -> Self { - self.changed(py, |overlay| { - overlay.binding_diagnostics = Some(diagnostics) - }) - } - - /// This model after semantic type recovery: its bindings and the outputs left unchecked. - fn with_type_recovery( - &self, - py: Python<'_>, - diagnostics: Py, - poisoned: Vec, - ) -> Self { - self.changed(py, |overlay| { - overlay.binding_diagnostics = Some(diagnostics); - overlay.poisoned = poisoned; - }) - } - - /// This model with the adapter's default schema and database filled into its destination. - pub(crate) fn with_target_defaults( - &self, - py: Python<'_>, - defaults: (Option, Option), - render: Bound<'_, PyAny>, - ) -> PyResult { - let destination: DestinationFact = self.compiled(py, |compiled, _| { - let current: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); - defaulted_destination(current, defaults, &render) - })?; - Ok(self.changed(py, |overlay| overlay.destination = Some(destination))) - } -} - -/// The model's project graph row: its key, dependencies, tags and folder. -pub(crate) fn model_graph_row( - model: &NativeCompiledModel, - py: Python<'_>, -) -> PyResult { - model.read(py, |record, py| { - let compiled: &CompiledRecord = compiled_of(record)?; - let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; - let mut tags: Vec = Vec::new(); - if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? - && (value.is_instance_of::() - || value.is_instance_of::()) - { - for tag in value.try_iter()? { - tags.push(tag?.str()?.extract()?); - } - } - let folder: String = record.host.model_file.get().folder(py)?; - Ok(ResourceRow { - key: (MODEL_RESOURCE.to_owned(), model_name(record)), - deps: compiled.deps.clone(), - tags, - folder: Some(folder), - }) - }) -} - -fn model_name(record: &ModelInputRecord) -> String { - record.name.clone() -} - -/// The `CompiledRelationLocation` of a destination. -pub(crate) fn relation_location( - classes: &CompiledViewClasses<'_>, - destination: &DestinationFact, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); - fields.set_item(DATABASE_KEY, &destination.database)?; - fields.set_item(SCHEMA_KEY, &destination.schema)?; - fields.set_item("name", &destination.name)?; - fields.set_item("qualified_name", &destination.qualified_name)?; - fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; - fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; - Ok(classes.relation_location.call((), Some(&fields))?.unbind()) -} - -pub(crate) fn object_key( - classes: &CompiledViewClasses<'_>, - kind: &str, - name: &str, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); - fields.set_item("resource_type", kind)?; - fields.set_item("name", name)?; - Ok(classes.object_key.call((), Some(&fields))?.unbind()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs deleted file mode 100644 index 9cfb2af044..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! A compile's attached inputs, read from the native compiled project that owns its models. - -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyclass, pymethods}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; - -const MODEL_INPUTS_FIELD: &str = "model_inputs"; -const RUN_ID_FIELD: &str = "run_id"; -const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; - -/// Compile inputs held as Python objects: user code, discovery or an integration owns them. -struct BoundaryInputs { - /// The discovered project config. - project_config: Py, - /// The discovered local config. - local_config: Py, - /// The discovered project inputs. - discovered_inputs: Py, - /// The adapter's lexical syntax. - sql_lexical_syntax: Py, - /// The selected target config, or `None`. - effective_target: Py, - /// The compile cache directory path. - compile_cache_dir: Py, - /// The resolved connection, adapter integration values. - effective_connection: Py, - /// Project variables, user values. - effective_vars: Py, - /// The macro context user macro bodies run with. - macro_context: Py, - /// Loaded macro modules, user code. - loaded_macros: Py, - /// Public enums, user declaration values. - public_enums: Py, - /// Public constants, user declaration values. - public_constants: Py, - /// The external reference resolver integration, or `None`. - external_sql_reference_resolver: Py, -} - -/// Compiler-derived inputs still held as Python objects until they are owner facts. -struct CompilerHandles { - /// Effective settings. - effective_settings: Py, - /// Attached seed inputs. - seed_inputs: Py, - /// Attached source inputs. - source_inputs: Py, - /// Attached SQL function inputs. - sql_function_inputs: Py, - /// Attached SQL test inputs. - test_inputs: Py, - /// Attached SQL scenario inputs. - scenario_inputs: Py, - /// Attached audit inputs. - audit_inputs: Py, - /// Compiler diagnostics. - diagnostics: Py, - /// The declaration scope index. - scope_index: Py, - /// The declaration scope build. - declaration_scope: Py, -} - -/// The values of one compile inputs view. -struct InputsFields { - run_id: String, - effective_target_name: Option, - no_sql_validation: bool, - model_inputs: Py, - boundary: BoundaryInputs, - compiler: CompilerHandles, -} - -impl InputsFields { - /// The fields named in `fields`, each required. - fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { - Ok(Self { - run_id: required(fields, RUN_ID_FIELD)?.extract()?, - effective_target_name: required(fields, "effective_target_name")?.extract()?, - no_sql_validation: required(fields, NO_SQL_VALIDATION_FIELD)?.extract()?, - model_inputs: required(fields, MODEL_INPUTS_FIELD)? - .downcast_into::()? - .unbind(), - boundary: BoundaryInputs { - project_config: required(fields, "project_config")?.unbind(), - local_config: required(fields, "local_config")?.unbind(), - discovered_inputs: required(fields, "discovered_inputs")?.unbind(), - sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), - effective_target: required(fields, "effective_target")?.unbind(), - compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), - effective_connection: required(fields, "effective_connection")?.unbind(), - effective_vars: required(fields, "effective_vars")?.unbind(), - macro_context: required(fields, "macro_context")?.unbind(), - loaded_macros: required(fields, "loaded_macros")?.unbind(), - public_enums: required(fields, "public_enums")?.unbind(), - public_constants: required(fields, "public_constants")?.unbind(), - external_sql_reference_resolver: required( - fields, - "external_sql_reference_resolver", - )? - .unbind(), - }, - compiler: CompilerHandles { - effective_settings: required(fields, "effective_settings")?.unbind(), - seed_inputs: required(fields, "seed_inputs")?.unbind(), - source_inputs: required(fields, "source_inputs")?.unbind(), - sql_function_inputs: required(fields, "sql_function_inputs")?.unbind(), - test_inputs: required(fields, "test_inputs")?.unbind(), - scenario_inputs: required(fields, "scenario_inputs")?.unbind(), - audit_inputs: required(fields, "audit_inputs")?.unbind(), - diagnostics: required(fields, "diagnostics")?.unbind(), - scope_index: required(fields, "scope_index")?.unbind(), - declaration_scope: required(fields, "declaration_scope")?.unbind(), - }, - }) - } - - /// Every field by name. - fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item(RUN_ID_FIELD, &self.run_id)?; - fields.set_item("effective_target_name", &self.effective_target_name)?; - fields.set_item(NO_SQL_VALIDATION_FIELD, self.no_sql_validation)?; - fields.set_item(MODEL_INPUTS_FIELD, self.model_inputs.bind(py))?; - fields.set_item("project_config", self.boundary.project_config.bind(py))?; - fields.set_item("local_config", self.boundary.local_config.bind(py))?; - fields.set_item( - "discovered_inputs", - self.boundary.discovered_inputs.bind(py), - )?; - fields.set_item( - "sql_lexical_syntax", - self.boundary.sql_lexical_syntax.bind(py), - )?; - fields.set_item("effective_target", self.boundary.effective_target.bind(py))?; - fields.set_item( - "compile_cache_dir", - self.boundary.compile_cache_dir.bind(py), - )?; - fields.set_item( - "effective_connection", - self.boundary.effective_connection.bind(py), - )?; - fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; - fields.set_item("macro_context", self.boundary.macro_context.bind(py))?; - fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; - fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; - fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; - fields.set_item( - "external_sql_reference_resolver", - self.boundary.external_sql_reference_resolver.bind(py), - )?; - fields.set_item( - "effective_settings", - self.compiler.effective_settings.bind(py), - )?; - fields.set_item("seed_inputs", self.compiler.seed_inputs.bind(py))?; - fields.set_item("source_inputs", self.compiler.source_inputs.bind(py))?; - fields.set_item( - "sql_function_inputs", - self.compiler.sql_function_inputs.bind(py), - )?; - fields.set_item("test_inputs", self.compiler.test_inputs.bind(py))?; - fields.set_item("scenario_inputs", self.compiler.scenario_inputs.bind(py))?; - fields.set_item("audit_inputs", self.compiler.audit_inputs.bind(py))?; - fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; - fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; - fields.set_item( - "declaration_scope", - self.compiler.declaration_scope.bind(py), - )?; - Ok(fields) - } -} - -fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { - fields - .get_item(name)? - .ok_or_else(|| PyTypeError::new_err(format!("missing compile input field {name}"))) -} - -/// One compile's attached inputs, its model inputs projected from the owner. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeInputsView { - owner: Py, - fields: InputsFields, -} - -impl NativeInputsView { - /// The view over `owner` with these input fields. - pub(crate) fn new( - owner: Py, - fields: &Bound<'_, PyDict>, - ) -> PyResult { - Ok(Self { - owner, - fields: InputsFields::from_fields(fields)?, - }) - } -} - -#[pymethods] -impl NativeInputsView { - /// These inputs with `changes` applied; the owner's model inputs are shared. - #[pyo3(signature = (**changes))] - fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { - let current: Bound<'_, PyDict> = self.fields.as_fields(py)?; - if let Some(changes) = changes { - for (name, value) in changes.iter() { - if !current.contains(&name)? { - return Err(PyTypeError::new_err(format!( - "unknown compile input field {name}" - ))); - } - current.set_item(name, value)?; - } - } - Self::new(self.owner.clone_ref(py), ¤t) - } - - /// Every field by name, for an explicit `CompileProjectInputs` projection. - fn fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; - fields.set_item("native_project", self.owner.bind(py))?; - Ok(fields) - } - - #[getter] - pub(crate) fn run_id(&self) -> String { - self.fields.run_id.clone() - } - - #[getter] - pub(crate) fn effective_target_name(&self) -> Option { - self.fields.effective_target_name.clone() - } - - #[getter(no_sql_validation)] - pub(crate) fn is_sql_validation_skipped(&self) -> bool { - self.fields.no_sql_validation - } - - #[getter] - pub(crate) fn model_inputs(&self, py: Python<'_>) -> Py { - self.fields.model_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn native_project(&self, py: Python<'_>) -> Py { - self.owner.clone_ref(py) - } - - #[getter] - pub(crate) fn project_config(&self, py: Python<'_>) -> Py { - self.fields.boundary.project_config.clone_ref(py) - } - - #[getter] - pub(crate) fn local_config(&self, py: Python<'_>) -> Py { - self.fields.boundary.local_config.clone_ref(py) - } - - #[getter] - pub(crate) fn discovered_inputs(&self, py: Python<'_>) -> Py { - self.fields.boundary.discovered_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_lexical_syntax.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_target(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_target.clone_ref(py) - } - - #[getter] - pub(crate) fn compile_cache_dir(&self, py: Python<'_>) -> Py { - self.fields.boundary.compile_cache_dir.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_connection(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_connection.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_vars(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_vars.clone_ref(py) - } - - #[getter] - pub(crate) fn macro_context(&self, py: Python<'_>) -> Py { - self.fields.boundary.macro_context.clone_ref(py) - } - - #[getter] - pub(crate) fn loaded_macros(&self, py: Python<'_>) -> Py { - self.fields.boundary.loaded_macros.clone_ref(py) - } - - #[getter] - pub(crate) fn public_enums(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_enums.clone_ref(py) - } - - #[getter] - pub(crate) fn public_constants(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_constants.clone_ref(py) - } - - #[getter] - pub(crate) fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { - self.fields - .boundary - .external_sql_reference_resolver - .clone_ref(py) - } - - #[getter] - pub(crate) fn effective_settings(&self, py: Python<'_>) -> Py { - self.fields.compiler.effective_settings.clone_ref(py) - } - - #[getter] - pub(crate) fn seed_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.seed_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn source_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.source_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_function_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_function_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn test_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.test_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn scenario_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.scenario_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn audit_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.audit_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn diagnostics(&self, py: Python<'_>) -> Py { - self.fields.compiler.diagnostics.clone_ref(py) - } - - #[getter] - pub(crate) fn scope_index(&self, py: Python<'_>) -> Py { - self.fields.compiler.scope_index.clone_ref(py) - } - - #[getter] - pub(crate) fn declaration_scope(&self, py: Python<'_>) -> Py { - self.fields.compiler.declaration_scope.clone_ref(py) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs index 8bff8fd13e..10fcd4e9e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs @@ -1,22 +1,2 @@ //! Python views of the compiled project: model, input, resource and project views and their records. -pub(crate) mod audit_reads; -pub(crate) mod audit_views; -pub(crate) mod compiled_model_analysis; -pub(crate) mod compiled_model_view; -pub(crate) mod function_reads; -pub(crate) mod function_views; -pub(crate) mod getters; -pub(crate) mod inputs_view; -pub(crate) mod model_input_view; -pub(crate) mod model_reads; -pub(crate) mod project_host; -pub(crate) mod project_reads; -pub(crate) mod project_view; -pub(crate) mod records; pub(crate) mod registration; -pub(crate) mod relation_keys; -pub(crate) mod resource_reads; -pub(crate) mod seed_reads; -pub(crate) mod seed_views; -pub(crate) mod source_reads; -pub(crate) mod source_views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs deleted file mode 100644 index 841770af33..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ /dev/null @@ -1,612 +0,0 @@ -//! A compiled project, read from the native compiled project that owns its models. - -use std::collections::BTreeMap; -use std::path::PathBuf; - -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pymethods}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, model_graph_row, -}; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, authored_expansion, view_classes, -}; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; -use crate::bindings::_helpers::graph::project_graph::{ - NativeProjectGraph, ResourceRow, project_graph, -}; -use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const MODELS_FIELD: &str = "models"; -const RUN_ID_FIELD: &str = "run_id"; -const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; -const KEY_ATTRIBUTE: &str = "key"; -const DEPS_ATTRIBUTE: &str = "deps"; -const TAGS_ATTRIBUTE: &str = "tags"; -const SCHEMA_ENTRY_ATTRIBUTE: &str = "schema_entry"; -const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; -const NAME_ATTRIBUTE: &str = "name"; - -/// Project inputs held as Python objects: user code, discovery or an integration owns them. -struct BoundaryInputs { - /// The resolved connection, adapter integration values. - effective_connection: Py, - /// Project variables, user values. - effective_vars: Py, - /// The native binding catalog. - binding_catalog: Py, - /// The adapter's lexical syntax. - sql_lexical_syntax: Py, - /// The compile cache directory path. - compile_cache_dir: Py, - /// Discovered loader functions, user callables. - loader_functions: Py, - /// Task/asset/check source locations, without retaining additional Python callables. - python_execution_inputs: BTreeMap, - /// Discovered Python hooks, user callables. - hook_functions: Py, - /// Discovered SQL hook files. - sql_hook_files: Py, - /// Discovered custom materializations. - materialization_files: Py, - /// Public enums, user declaration values. - public_enums: Py, - /// Public constants, user declaration values. - public_constants: Py, - /// Loaded macro modules, user code. - loaded_macros: Py, - /// The external reference resolver integration, or `None`. - external_sql_reference_resolver: Py, - /// The native analysis session, or `None`. - native_session: Py, -} - -/// Compiler-derived project parts still held as Python objects until they are owner facts. -struct CompilerHandles { - /// Effective settings. - settings: Py, - /// Effective scenario config. - scenario: Py, - /// Compiled sources. - sources: Py, - /// Compiled seeds. - seeds: Py, - /// Compiled SQL functions. - functions: Py, - /// Compiled audits. - audits: Py, - /// Compiled SQL tests. - sql_tests: Py, - /// Compiled SQL scenarios. - sql_scenarios: Py, - /// Compiler diagnostics. - diagnostics: Py, - /// Literal relations no resource matched. - unmatched_literal_sql_relations: Py, - /// The declaration scope index. - scope_index: Py, -} - -/// The project-level values of one compiled project view. -struct ProjectFields { - run_id: String, - effective_target_name: Option, - effective_target_database: Option, - effective_target_schema: Option, - sql_analysis_dialect: Option, - enforce_explicit_references: bool, - models: Py, - boundary: BoundaryInputs, - compiler: CompilerHandles, -} - -impl ProjectFields { - /// The fields named in `fields`, each required. - fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { - Ok(Self { - run_id: required(fields, RUN_ID_FIELD)?.extract()?, - effective_target_name: required(fields, "effective_target_name")?.extract()?, - effective_target_database: required(fields, "effective_target_database")?.extract()?, - effective_target_schema: required(fields, "effective_target_schema")?.extract()?, - sql_analysis_dialect: required(fields, "sql_analysis_dialect")?.extract()?, - enforce_explicit_references: required(fields, ENFORCE_EXPLICIT_FIELD)?.extract()?, - models: required(fields, MODELS_FIELD)? - .downcast_into::()? - .unbind(), - boundary: BoundaryInputs { - effective_connection: required(fields, "effective_connection")?.unbind(), - effective_vars: required(fields, "effective_vars")?.unbind(), - binding_catalog: required(fields, "binding_catalog")?.unbind(), - sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), - compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), - loader_functions: required(fields, "loader_functions")?.unbind(), - python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, - hook_functions: required(fields, "hook_functions")?.unbind(), - sql_hook_files: required(fields, "sql_hook_files")?.unbind(), - materialization_files: required(fields, "materialization_files")?.unbind(), - public_enums: required(fields, "public_enums")?.unbind(), - public_constants: required(fields, "public_constants")?.unbind(), - loaded_macros: required(fields, "loaded_macros")?.unbind(), - external_sql_reference_resolver: required( - fields, - "external_sql_reference_resolver", - )? - .unbind(), - native_session: required(fields, "native_session")?.unbind(), - }, - compiler: CompilerHandles { - settings: required(fields, "settings")?.unbind(), - scenario: required(fields, "scenario")?.unbind(), - sources: required(fields, "sources")?.unbind(), - seeds: required(fields, "seeds")?.unbind(), - functions: required(fields, "functions")?.unbind(), - audits: required(fields, "audits")?.unbind(), - sql_tests: required(fields, "sql_tests")?.unbind(), - sql_scenarios: required(fields, "sql_scenarios")?.unbind(), - diagnostics: required(fields, "diagnostics")?.unbind(), - unmatched_literal_sql_relations: required( - fields, - "unmatched_literal_sql_relations", - )? - .unbind(), - scope_index: required(fields, "scope_index")?.unbind(), - }, - }) - } - - /// These fields with `changes` applied. - fn changed(&self, py: Python<'_>, changes: &Bound<'_, PyDict>) -> PyResult { - let current: Bound<'_, PyDict> = self.as_fields(py)?; - for (name, value) in changes.iter() { - if !current.contains(&name)? { - return Err(PyTypeError::new_err(format!( - "unknown project field {name}" - ))); - } - current.set_item(name, value)?; - } - Self::from_fields(¤t) - } - - /// Every field by name. - fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item(RUN_ID_FIELD, &self.run_id)?; - fields.set_item("effective_target_name", &self.effective_target_name)?; - fields.set_item("effective_target_database", &self.effective_target_database)?; - fields.set_item("effective_target_schema", &self.effective_target_schema)?; - fields.set_item("sql_analysis_dialect", &self.sql_analysis_dialect)?; - fields.set_item(ENFORCE_EXPLICIT_FIELD, self.enforce_explicit_references)?; - fields.set_item(MODELS_FIELD, self.models.bind(py))?; - fields.set_item( - "effective_connection", - self.boundary.effective_connection.bind(py), - )?; - fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; - fields.set_item("binding_catalog", self.boundary.binding_catalog.bind(py))?; - fields.set_item( - "sql_lexical_syntax", - self.boundary.sql_lexical_syntax.bind(py), - )?; - fields.set_item( - "compile_cache_dir", - self.boundary.compile_cache_dir.bind(py), - )?; - fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; - fields.set_item( - "python_execution_inputs", - &self.boundary.python_execution_inputs, - )?; - fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; - fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; - fields.set_item( - "materialization_files", - self.boundary.materialization_files.bind(py), - )?; - fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; - fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; - fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; - fields.set_item( - "external_sql_reference_resolver", - self.boundary.external_sql_reference_resolver.bind(py), - )?; - fields.set_item("native_session", self.boundary.native_session.bind(py))?; - fields.set_item("settings", self.compiler.settings.bind(py))?; - fields.set_item("scenario", self.compiler.scenario.bind(py))?; - fields.set_item("sources", self.compiler.sources.bind(py))?; - fields.set_item("seeds", self.compiler.seeds.bind(py))?; - fields.set_item("functions", self.compiler.functions.bind(py))?; - fields.set_item("audits", self.compiler.audits.bind(py))?; - fields.set_item("sql_tests", self.compiler.sql_tests.bind(py))?; - fields.set_item("sql_scenarios", self.compiler.sql_scenarios.bind(py))?; - fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; - fields.set_item( - "unmatched_literal_sql_relations", - self.compiler.unmatched_literal_sql_relations.bind(py), - )?; - fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; - Ok(fields) - } -} - -fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { - fields - .get_item(name)? - .ok_or_else(|| PyTypeError::new_err(format!("missing project field {name}"))) -} - -/// One compiled project, its models projected from the owner. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeProjectView { - owner: Py, - fields: ProjectFields, - lineage_graph: PyOnceLock>, - sql_expansions: PyOnceLock>, -} - -impl NativeProjectView { - /// The view over `owner` with these project fields. - pub(crate) fn new( - owner: Py, - fields: &Bound<'_, PyDict>, - ) -> PyResult { - Ok(Self::with_fields( - owner, - ProjectFields::from_fields(fields)?, - )) - } - - /// The project field custom-rule facts and rules runs read by `name`, or `None` for any other. - pub(crate) fn fact_field(&self, py: Python<'_>, name: &str) -> Option> { - let field: &Py = match name { - "settings" => &self.fields.compiler.settings, - "sql_scenarios" => &self.fields.compiler.sql_scenarios, - "scope_index" => &self.fields.compiler.scope_index, - "diagnostics" => &self.fields.compiler.diagnostics, - "sources" => &self.fields.compiler.sources, - "seeds" => &self.fields.compiler.seeds, - "functions" => &self.fields.compiler.functions, - "audits" => &self.fields.compiler.audits, - "sql_tests" => &self.fields.compiler.sql_tests, - "public_enums" => &self.fields.boundary.public_enums, - "public_constants" => &self.fields.boundary.public_constants, - "models" => return Some(self.fields.models.clone_ref(py).into_any()), - _ => return None, - }; - Some(field.clone_ref(py)) - } - - /// The compiled project this view projects. - pub(crate) fn owner(&self) -> &Py { - &self.owner - } - - fn with_fields(owner: Py, fields: ProjectFields) -> Self { - Self { - owner, - fields, - lineage_graph: PyOnceLock::new(), - sql_expansions: PyOnceLock::new(), - } - } -} - -#[pymethods] -impl NativeProjectView { - /// This project with `changes` applied; the owner's models are shared. - #[pyo3(signature = (**changes))] - fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { - let fields: ProjectFields = match changes { - Some(changes) => self.fields.changed(py, &changes)?, - None => self.fields.changed(py, &PyDict::new(py))?, - }; - Ok(Self::with_fields(self.owner.clone_ref(py), fields)) - } - - /// Every field by name, for an explicit `CompiledProject` projection. - #[pyo3(signature = (include_sql_expansions = true))] - pub(crate) fn fields<'py>( - &self, - py: Python<'py>, - include_sql_expansions: bool, - ) -> PyResult> { - let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; - fields.set_item( - "sql_expansions", - if include_sql_expansions { - self.sql_expansions(py)? - } else { - PyDict::new(py).unbind() - }, - )?; - fields.set_item("native_project", self.owner.bind(py))?; - Ok(fields) - } - - #[getter] - pub(crate) fn run_id(&self) -> String { - self.fields.run_id.clone() - } - - #[getter] - pub(crate) fn effective_target_name(&self) -> Option { - self.fields.effective_target_name.clone() - } - - #[getter] - pub(crate) fn effective_target_database(&self) -> Option { - self.fields.effective_target_database.clone() - } - - #[getter] - pub(crate) fn effective_target_schema(&self) -> Option { - self.fields.effective_target_schema.clone() - } - - #[getter] - pub(crate) fn sql_analysis_dialect(&self) -> Option { - self.fields.sql_analysis_dialect.clone() - } - - #[getter(enforce_explicit_references)] - pub(crate) fn is_explicit_reference_enforced(&self) -> bool { - self.fields.enforce_explicit_references - } - - #[getter] - pub(crate) fn models(&self, py: Python<'_>) -> Py { - self.fields.models.clone_ref(py) - } - - #[getter] - pub(crate) fn native_project(&self, py: Python<'_>) -> Py { - self.owner.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_connection(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_connection.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_vars(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_vars.clone_ref(py) - } - - #[getter] - pub(crate) fn binding_catalog(&self, py: Python<'_>) -> Py { - self.fields.boundary.binding_catalog.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_lexical_syntax.clone_ref(py) - } - - #[getter] - pub(crate) fn compile_cache_dir(&self, py: Python<'_>) -> Py { - self.fields.boundary.compile_cache_dir.clone_ref(py) - } - - #[getter] - pub(crate) fn loader_functions(&self, py: Python<'_>) -> Py { - self.fields.boundary.loader_functions.clone_ref(py) - } - - #[getter] - pub(crate) fn python_execution_inputs(&self) -> BTreeMap { - self.fields.boundary.python_execution_inputs.clone() - } - - #[getter] - pub(crate) fn hook_functions(&self, py: Python<'_>) -> Py { - self.fields.boundary.hook_functions.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_hook_files(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_hook_files.clone_ref(py) - } - - #[getter] - pub(crate) fn materialization_files(&self, py: Python<'_>) -> Py { - self.fields.boundary.materialization_files.clone_ref(py) - } - - #[getter] - pub(crate) fn public_enums(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_enums.clone_ref(py) - } - - #[getter] - pub(crate) fn public_constants(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_constants.clone_ref(py) - } - - #[getter] - pub(crate) fn loaded_macros(&self, py: Python<'_>) -> Py { - self.fields.boundary.loaded_macros.clone_ref(py) - } - - #[getter] - pub(crate) fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { - self.fields - .boundary - .external_sql_reference_resolver - .clone_ref(py) - } - - #[getter] - pub(crate) fn native_session(&self, py: Python<'_>) -> Py { - self.fields.boundary.native_session.clone_ref(py) - } - - #[getter] - pub(crate) fn settings(&self, py: Python<'_>) -> Py { - self.fields.compiler.settings.clone_ref(py) - } - - #[getter] - pub(crate) fn scenario(&self, py: Python<'_>) -> Py { - self.fields.compiler.scenario.clone_ref(py) - } - - #[getter] - pub(crate) fn sources(&self, py: Python<'_>) -> Py { - self.fields.compiler.sources.clone_ref(py) - } - - #[getter] - pub(crate) fn seeds(&self, py: Python<'_>) -> Py { - self.fields.compiler.seeds.clone_ref(py) - } - - #[getter] - pub(crate) fn functions(&self, py: Python<'_>) -> Py { - self.fields.compiler.functions.clone_ref(py) - } - - #[getter] - pub(crate) fn audits(&self, py: Python<'_>) -> Py { - self.fields.compiler.audits.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_tests(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_tests.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_scenarios(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_scenarios.clone_ref(py) - } - - #[getter] - pub(crate) fn diagnostics(&self, py: Python<'_>) -> Py { - self.fields.compiler.diagnostics.clone_ref(py) - } - - #[getter] - pub(crate) fn unmatched_literal_sql_relations(&self, py: Python<'_>) -> Py { - self.fields - .compiler - .unmatched_literal_sql_relations - .clone_ref(py) - } - - #[getter] - pub(crate) fn scope_index(&self, py: Python<'_>) -> Py { - self.fields.compiler.scope_index.clone_ref(py) - } - - /// Each model's authored-to-expanded passes by file path, where interpolation kept the query. - #[getter] - pub(crate) fn sql_expansions(&self, py: Python<'_>) -> PyResult> { - self.sql_expansions - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let owner = self.owner.bind(py).borrow(); - let expansions: Bound<'_, PyDict> = PyDict::new(py); - for record in owner.inputs() { - let expansion: Py = authored_expansion(&classes, record)?; - if !expansion.is_none(py) { - let path: Bound<'_, PyAny> = - record.host.model_file.get().path_objects(py)?.0; - expansions.set_item(path, expansion)?; - } - } - Ok::<_, pyo3::PyErr>(expansions.unbind()) - }) - .map(|expansions| expansions.clone_ref(py)) - } - - /// The lineage edges, tags, model folders and names, indexed natively once per project. - #[getter] - pub(crate) fn lineage_graph(&self, py: Python<'_>) -> PyResult> { - self.lineage_graph - .get_or_try_init(py, || { - let mut rows: Vec = Vec::new(); - for model in self.fields.models.bind(py).iter() { - let model: Bound<'_, NativeCompiledModel> = model.downcast_into()?; - rows.push(model_graph_row(model.get(), py)?); - } - let compiler: &CompilerHandles = &self.fields.compiler; - for source in compiler.sources.bind(py).try_iter()? { - let source: Bound<'_, PyAny> = source?; - rows.push(match source.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => resource_row(&source, None)?, - }); - } - for seed in compiler.seeds.bind(py).try_iter()? { - let seed: Bound<'_, PyAny> = seed?; - rows.push(match seed.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => { - let tags: Bound<'_, PyAny> = native_field( - &native_field(&seed, SCHEMA_ENTRY_ATTRIBUTE)?, - TAGS_ATTRIBUTE, - )?; - resource_row(&seed, Some(&tags))? - } - }); - } - for function in compiler.functions.bind(py).try_iter()? { - let function: Bound<'_, PyAny> = function?; - rows.push(match function.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => { - let tags: Bound<'_, PyAny> = native_field(&function, TAGS_ATTRIBUTE)?; - resource_row(&function, Some(&tags))? - } - }); - } - self.owner.bind(py).borrow_mut().record_graph_build(); - count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; - Py::new(py, project_graph(rows)?) - }) - .map(|graph| graph.clone_ref(py)) - } -} - -/// A source, seed or function's graph row, read from its compiled object. -fn resource_row( - resource: &Bound<'_, PyAny>, - tags: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let mut deps: Vec<(String, String)> = Vec::new(); - for key in native_field(resource, DEPS_ATTRIBUTE)?.try_iter()? { - deps.push(key_row(&key?)?); - } - let mut tag_names: Vec = Vec::new(); - if let Some(tags) = tags { - for tag in tags.try_iter()? { - tag_names.push(tag?.str()?.extract()?); - } - } - Ok(ResourceRow { - key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, - deps, - tags: tag_names, - folder: None, - }) -} - -/// `(resource type, name)` of a `CompiledObjectKey`. -pub(crate) fn key_row(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - Ok(( - native_field(key, RESOURCE_TYPE_ATTRIBUTE)? - .str()? - .extract()?, - native_field(key, NAME_ATTRIBUTE)?.extract()?, - )) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs index 348c352052..fb27f03cf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs @@ -1,5 +1,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::compiled_views::records::register(module) + sqlbuild_python_project::compiled_views::main::records::register_records::register_records( + module, + ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs deleted file mode 100644 index 1c898d7800..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Seeds paired with their declarations, held by the compiled project, and their views. - -use pyo3::exceptions::{PyIndexError, PyRuntimeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyclass, pymethods}; -use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; -use sqlbuild_python_base::positional_record; - -use sqlbuild_project::models as project_models; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - defaulted_destination, object_key, relation_location, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_project::models::DestinationFact; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const SEED_RESOURCE: &str = "seed"; -const TAGS_ATTRIBUTE: &str = "tags"; -const NAME_ATTRIBUTE: &str = "name"; -const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const STEM_ATTRIBUTE: &str = "stem"; -const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; - -positional_record! { - extract; - /// `(database, schema, logical database, logical schema)` of a seed's destination. - pub(crate) struct SeedNamespace { - database: Option, - schema: Option, - logical_database: Option, - logical_schema: Option, - } -} - -/// A seed's name, CSV file, declaration and declaring schema file. -pub(crate) struct SeedParts<'a> { - pub(crate) name: &'a str, - pub(crate) seed_file: &'a Py, - pub(crate) schema_entry: &'a Py, - pub(crate) schema_file: &'a Py, -} - -/// One seed CSV paired with its declaration; the discovered objects are discovery's. -pub(crate) type SeedRecord = project_models::SeedRecord; - -/// A seed's discovered objects and the key its views build. -pub(crate) struct SeedHandles { - /// The discovered CSV file. - seed_file: Py, - /// The discovered declaration, whose spec carries user-authored column metadata. - schema_entry: Py, - /// The schema file declaring the seed. - schema_file: Py, - key: PyOnceLock>, -} - -/// What a seed record reads and keeps through its Python values. -pub(crate) trait SeedViews { - /// The seed's name, CSV file, declaration and declaring schema file. - fn parts(&self) -> SeedParts<'_>; - - /// Keep the destination native project assembly resolved for the seed. - fn set_namespace(&mut self, namespace: SeedNamespace); -} - -/// Pair each declaration with the CSV file of its name, raising for the first without one. -pub(crate) fn paired_seeds( - declarations: Vec<(Py, Py)>, - seed_files: &[Py], - compile_error: &Bound<'_, PyAny>, -) -> PyResult> { - let py: Python<'_> = compile_error.py(); - let mut names: Vec = Vec::with_capacity(declarations.len()); - for (entry, _) in &declarations { - names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); - } - let mut stems: Vec = Vec::with_capacity(seed_files.len()); - for seed_file in seed_files { - stems.push( - native_field( - &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, - STEM_ATTRIBUTE, - )? - .extract()?, - ); - } - let pairs: Vec = match pair_seed_declarations(&names, &stems) { - Ok(pairs) => pairs, - Err(missing) => { - let schema_file: &Py = &declarations[missing].1; - let relative_path: String = - native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .extract()?; - let message: String = format!( - "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", - names[missing] - ); - return Err(PyErr::from_value(compile_error.call1((message,))?)); - } - }; - let mut seeds: Vec = Vec::with_capacity(pairs.len()); - for (((schema_entry, schema_file), name), file_index) in - declarations.into_iter().zip(names).zip(pairs) - { - seeds.push(SeedRecord { - name, - destination: None, - host: SeedHandles { - seed_file: seed_files[file_index].clone_ref(py), - schema_entry, - schema_file, - key: PyOnceLock::new(), - }, - }); - } - Ok(seeds) -} - -impl SeedViews for SeedRecord { - fn parts(&self) -> SeedParts<'_> { - SeedParts { - name: &self.name, - seed_file: &self.host.seed_file, - schema_entry: &self.host.schema_entry, - schema_file: &self.host.schema_file, - } - } - - fn set_namespace(&mut self, namespace: SeedNamespace) { - let SeedNamespace { - database, - schema, - logical_database, - logical_schema, - } = namespace; - self.destination = Some(DestinationFact { - database, - schema, - name: self.name.clone(), - qualified_name: None, - logical_schema, - logical_database, - }); - } -} - -fn read_seed<'py, T>( - owner: &Py, - index: usize, - py: Python<'py>, - read: impl FnOnce(&SeedRecord, Python<'py>) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - let seed: &SeedRecord = owner - .seeds() - .get(index) - .ok_or_else(|| PyIndexError::new_err(index))?; - read(seed, py) -} - -/// One attached seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeSeedInput { - owner: Py, - index: usize, -} - -impl NativeSeedInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSeedInput { - #[getter] - pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } -} - -/// One assembled seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledSeed { - owner: Py, - index: usize, - /// The destination with target defaults applied, replacing the assembled one. - defaulted: Option, - destination: PyOnceLock>, -} - -impl NativeCompiledSeed { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { - owner, - index, - defaulted: None, - destination: PyOnceLock::new(), - } - } - - /// The seed's project graph row: its key and its declared tags. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, py| { - let mut tags: Vec = Vec::new(); - for tag in native_field(seed.host.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { - tags.push(tag?.str()?.extract()?); - } - Ok(ResourceRow { - key: (SEED_RESOURCE.to_owned(), seed.name.clone()), - deps: Vec::new(), - tags, - folder: None, - }) - }) - } - - pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_seed(&self.owner, self.index, py, |seed, _| { - seed.destination - .clone() - .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) - }) - } -} - -#[pymethods] -impl NativeCompiledSeed { - /// The seed's planner-facing `CompiledObjectKey`. - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_seed(&self.owner, self.index, py, |seed, py| { - seed.host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SEED_RESOURCE, &seed.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) - } - - #[getter] - pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } - - /// The seed's planner-facing `CompiledRelationLocation`. - #[getter] - pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.destination - .get_or_try_init(py, || { - relation_location(&classes.compiled, &self.current_destination(py)?) - }) - .map(|destination| destination.clone_ref(py)) - } - - /// This seed with the adapter's default schema and database filled into its destination. - pub(crate) fn with_target_defaults( - &self, - py: Python<'_>, - defaults: (Option, Option), - render: Bound<'_, PyAny>, - ) -> PyResult { - let defaulted: DestinationFact = - defaulted_destination(&self.current_destination(py)?, defaults, &render)?; - Ok(Self { - owner: self.owner.clone_ref(py), - index: self.index, - defaulted: Some(defaulted), - destination: PyOnceLock::new(), - }) - } - - #[getter(external)] - pub(crate) fn is_external(&self) -> bool { - false - } -} - -impl NativeCompiledSeed { - /// The `CompiledSeed` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.seed_file(py)?.into_bound(py), - self.schema_entry(py)?.into_bound(py), - self.schema_file(py)?.into_bound(py), - self.destination(py)?.into_bound(py), - pyo3::types::PyBool::new(py, self.is_external()) - .to_owned() - .into_any(), - ]) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs deleted file mode 100644 index ef3d784416..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs +++ /dev/null @@ -1,333 +0,0 @@ -//! Sources attached by the native loop, held by the compiled project, and their views. - -use pyo3::exceptions::PyIndexError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyclass, pymethods}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_python_base::positional_record; - -use sqlbuild_project::models as project_models; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; -use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::audit_key_rows; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; - -const SOURCE_RESOURCE: &str = "source"; - -positional_record! { - into; - /// `(name, has a loader, database, schema)` of one source. - pub(crate) struct SourceAssemblyRow { - name: String, - managed: bool, - database: Option, - schema: Option, - } -} - -/// What the source loop attached for one source: its expanded metadata and usages. -pub(crate) type AttachedSource = project_models::AttachedSource; - -/// One attached source held by the compiled project. -pub(crate) type SourceRecord = project_models::SourceRecord; - -/// An attached source's user values. -pub(crate) struct SourceHandles { - /// The discovered source file. - pub(crate) source_file: Py, - /// The discovered entry; its spec carries user-authored metadata. - pub(crate) entry: Py, - /// Template-expanded user metadata. - pub(crate) meta: Py, - /// Template-expanded column specs, which carry user metadata and audit arguments. - pub(crate) columns: Py, - /// Template-expanded table audit specs, which carry user audit arguments. - pub(crate) audits: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - -/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. -pub(crate) struct SourceKeyFacts { - pub(crate) columns: Vec, - pub(crate) audits: Vec<(String, Option)>, -} - -/// A source record's key facts and entries, each built once from its Python values. -pub(crate) struct SourceProjections { - key_facts: PyOnceLock, - attached_entry: PyOnceLock>, - assembled_entry: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - -/// The source record of an attached source, before assembly resolves its namespace. -pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { - SourceRecord { - attached, - namespace: None, - host: SourceProjections { - key_facts: PyOnceLock::new(), - attached_entry: PyOnceLock::new(), - assembled_entry: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - -/// What a source record projects from its Python values, each built once. -pub(crate) trait SourceViews { - /// The source's declared column facts and audit key rows, read once from its expanded specs. - fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; - - /// The attached `SourceEntry`: the discovered entry with its expanded metadata. - fn attached_entry(&self, py: Python<'_>) -> PyResult>; - - /// The assembled `SourceEntry`: the attached entry in its resolved namespace. - fn assembled_entry(&self, py: Python<'_>) -> PyResult>; -} - -impl SourceViews for SourceRecord { - fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { - self.host.key_facts.get_or_try_init(py, || { - Ok(SourceKeyFacts { - columns: python_column_facts(self.attached.host.columns.bind(py).as_any(), None)?, - audits: audit_key_rows(self.attached.host.audits.bind(py).as_any())?, - }) - }) - } - - fn attached_entry(&self, py: Python<'_>) -> PyResult> { - self.host - .attached_entry - .get_or_try_init(py, || { - let source: &AttachedSource = &self.attached; - let changes: Bound<'_, PyDict> = PyDict::new(py); - changes.set_item("contract", &source.contract)?; - changes.set_item("database", &source.database)?; - changes.set_item("schema", &source.schema)?; - changes.set_item("table", &source.table)?; - changes.set_item("expression", &source.expression)?; - changes.set_item("description", &source.description)?; - changes.set_item("meta", source.host.meta.bind(py))?; - changes.set_item("columns", source.host.columns.bind(py))?; - changes.set_item("audits", source.host.audits.bind(py))?; - replaced(py, source.host.entry.bind(py), &changes) - }) - .map(|entry| entry.clone_ref(py)) - } - - fn assembled_entry(&self, py: Python<'_>) -> PyResult> { - let attached: Py = self.attached_entry(py)?; - let Some(Some((database, schema))) = &self.namespace else { - return Ok(attached); - }; - self.host - .assembled_entry - .get_or_try_init(py, || { - let changes: Bound<'_, PyDict> = PyDict::new(py); - changes.set_item("database", database)?; - changes.set_item("schema", schema)?; - replaced(py, attached.bind(py), &changes) - }) - .map(|entry| entry.clone_ref(py)) - } -} - -/// `dataclasses.replace(value, **changes)`. -pub(crate) fn replaced( - _py: Python<'_>, - value: &Bound<'_, PyAny>, - changes: &Bound<'_, PyDict>, -) -> PyResult> { - Ok(replaced_keywords(value, changes)?.unbind()) -} - -/// Read the source record behind a native source view, or `None` for any other object. -pub(crate) fn read_source_view<'py, T>( - view: &Bound<'py, PyAny>, - read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, -) -> Option> { - let py: Python<'py> = view.py(); - if let Ok(input) = view.cast::() { - let input: &NativeSourceInput = input.get(); - return Some(read_source(&input.owner, input.index, py, read)); - } - let Ok(compiled) = view.cast::() else { - return None; - }; - let compiled: &NativeCompiledSource = compiled.get(); - Some(read_source(&compiled.owner, compiled.index, py, read)) -} - -fn read_source<'py, T>( - owner: &Py, - index: usize, - py: Python<'py>, - read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - let source: &SourceRecord = owner - .sources() - .get(index) - .ok_or_else(|| PyIndexError::new_err(index))?; - read(source, py) -} - -/// One attached source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeSourceInput { - owner: Py, - index: usize, -} - -impl NativeSourceInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSourceInput { - #[getter] - pub(crate) fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.attached_entry(py) - }) - } - - #[getter] - pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.usages.clone_ref(py)) - }) - } - - /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. - #[getter] - pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, py| { - let attached: &AttachedSource = &source.attached; - Ok(SourceAssemblyRow { - name: attached.name.clone(), - managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), - database: attached.database.clone(), - schema: attached.schema.clone(), - }) - }) - } -} - -/// One assembled source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledSource { - owner: Py, - index: usize, -} - -impl NativeCompiledSource { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } - - /// The source's project graph row: its key, with no dependencies, tags or folder. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(ResourceRow { - key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - deps: Vec::new(), - tags: Vec::new(), - folder: None, - }) - }) - } -} - -#[pymethods] -impl NativeCompiledSource { - /// The source's planner-facing `CompiledObjectKey`. - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_source(&self.owner, self.index, py, |source, py| { - source - .host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(source.attached.name.clone()) - }) - } - - #[getter] - pub(crate) fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.assembled_entry(py) - }) - } - - #[getter] - pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } -} - -impl NativeCompiledSource { - /// The `CompiledSource` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.source_entry(py)?.into_bound(py), - self.source_file(py)?.into_bound(py), - ]) - } -} - -/// Keep `attached` in `owner`; return each source's view. -pub(crate) fn retain_sources( - owner: &Bound<'_, NativeCompiledProject>, - attached: Vec, -) -> Vec { - let mut project = owner.borrow_mut(); - let first: usize = project.sources().len(); - let count: usize = attached.len(); - for source in attached { - project.retain_source(source_record(source)); - } - drop(project); - (first..first + count) - .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) - .collect() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs index 88801d0be6..a99b3356d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -7,19 +7,25 @@ use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::bindings::_helpers::compiled_views::{ - function_reads, model_reads, project_reads, seed_reads, -}; use crate::bindings::_helpers::consumption::relation_reads::{ Locations, relation, sql_reads_undeclared, }; -use crate::bindings::_helpers::planner_identity::hashing::{baselined_model_query_hash, graph_key}; use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::function_reads::language::language; +use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; pub(crate) const MODEL: &str = "model"; pub(crate) const SEED: &str = "seed"; @@ -48,8 +54,16 @@ enum Compiled<'py> { impl<'py> Compiled<'py> { fn destination(&self) -> PyResult> { match self { - Self::Model(model) => model_reads::destination(model), - Self::Seed(seed) => seed_reads::destination(seed), + Self::Model(model) => { + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + ) + } + Self::Seed(seed) => { + sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( + seed, + ) + } } } } @@ -221,41 +235,49 @@ pub(crate) fn build_facts( ) -> PyResult { let py: Python<'_> = plan.py(); let mut compiled: HashMap> = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; compiled.insert( - node_key(MODEL, &model_reads::model_name(&model)?), + node_key(MODEL, &model_name(&model)?), Compiled::Model(model), ); } - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed: Bound<'_, PyAny> = seed?; compiled.insert( - node_key(SEED, &seed_reads::name(&seed)?.extract::()?), + node_key( + SEED, + &sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + .extract::()?, + ), Compiled::Seed(seed), ); } let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); let mut function_bodies: HashMap<(String, String), Option> = HashMap::new(); - let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let functions: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; - let key = graph_key(&function_reads::key(&function)?)?; + let key = graph_key( + &sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )?; function_deps.insert( (key.kind.clone(), key.name.clone()), - key_rows(&function_reads::deps(&function)?)?, + key_rows(&deps(&function)?)?, ); - let body: Option = - if enum_text(&function_reads::language(&function)?)? == SQL_LANGUAGE { - let body: String = function_reads::body_sql(&function)?.extract()?; - Some(if key.kind == UDF { - format!("SELECT ({body})") - } else { - body - }) + let body: Option = if enum_text(&language(&function)?)? == SQL_LANGUAGE { + let body: String = body_sql(&function)?.extract()?; + Some(if key.kind == UDF { + format!("SELECT ({body})") } else { - None - }; + body + }) + } else { + None + }; function_bodies.insert((key.kind, key.name), body); } let mut selected: BTreeMap> = BTreeMap::new(); @@ -312,15 +334,12 @@ pub(crate) fn build_facts( }; let (deps, version_hash) = match node { Compiled::Model(model) => ( - model_reads::dep_rows(model)?, + dep_rows(model)?, model_local_identity_hash( &baselined_model_query_hash( py, &key.name, - ( - &model_reads::query_sql(model)?.extract::()?, - dialect.as_deref(), - ), + (&query_sql(model)?.extract::()?, dialect.as_deref()), &mapping_get(&recorded_models, &key.name)?, )?, &model_version_identity_metadata_json( @@ -347,7 +366,7 @@ pub(crate) fn build_facts( (node, &locations, &mut function_checks) && (sql_reads_undeclared( py, - model_reads::query_sql(model)?.extract()?, + query_sql(model)?.extract()?, (&node_inputs, relation.as_ref()), (locations, dialect.as_deref()), )? || function_checks.undeclared(py, &deps)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs index 97b34ea561..ac9ae34ca7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs @@ -9,10 +9,10 @@ use sqlbuild_state::consumption::models::{ ChangeKind, ChangedPolicy, NodeKey, Reach, TriggeredJob, }; -use crate::bindings::_helpers::compiled_project::job_catalog::triggered_jobs; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::main::job_catalog::triggered_jobs::triggered_jobs; +use sqlbuild_python_project::graph::models::NativeProjectGraph; const DEFINITION: &str = "definition"; const SOURCE_DATA: &str = "source_data"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 2b1e918e9e..d5f910f5d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -6,12 +6,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; -use crate::bindings::_helpers::analysis::relation_names::{ - RelationRow, extract_sql_relation_names, -}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::native_host::main::optional_text::optional_text; +use sqlbuild_python_project::analysis::main::extract_sql_relation_names::extract_sql_relation_names; +use sqlbuild_python_project::analysis::types::RelationRow; /// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 8ac95d3422..557235e6ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -24,8 +24,6 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -38,6 +36,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::refactor::main::compare_contracts::compare_contracts; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const GENERATE_ACTION: &str = "generate"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 375db96d18..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,4 +1 @@ -pub(crate) mod contract_diagnostics; -pub(crate) mod exception_types; -pub(crate) mod model_contracts; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index a1f676c557..36bea04a1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -2,18 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::contracts::{contract_diagnostics, exception_types}; use sqlbuild_python_base::contracts::main::register_resource_identities::register_resource_identities; use sqlbuild_python_base::contracts::main::register_state_literals::register_state_literals; use sqlbuild_python_front::authored_values::main::register_sql_values::register_sql_values; use sqlbuild_python_front::contracts::main::register_effective_config::register_effective_config; use sqlbuild_python_front::contracts::main::register_spec_resolution::register_spec_resolution; use sqlbuild_python_front::discovery::main::project::register_project_config::register_project_config; +use sqlbuild_python_project::contracts::main::register_contract_diagnostics::register_contract_diagnostics; +use sqlbuild_python_project::contracts::main::register_exception_types::register_exception_types; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_diagnostics::register(module)?; + register_contract_diagnostics(module)?; register_effective_config(module)?; - exception_types::register(module)?; + register_exception_types(module)?; register_project_config(module)?; register_resource_identities(module)?; register_spec_resolution(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs index 0d6efc02e9..cee82aa1ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -9,10 +9,10 @@ use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInp use sqlbuild_core::json::errors::JsonEmitError; use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; -use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { match error { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs index a9ed9eb49c..f0bc584f2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -17,11 +17,11 @@ use sqlbuild_analysis::dag_artifact::models::{ use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; use sqlbuild_host::entries::models::ThirdPartyHost; -use crate::bindings::_helpers::planner_selection::carrier::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_project::planner_selection::models::Carrier; const PATHLIB_MODULE: &str = "pathlib"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 36f04c5968..956e86a5b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -20,9 +20,6 @@ use sqlbuild_core::dbt_graph::models::{ use sqlbuild_core::dbt_graph::types::DbtSelectorResolver; use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; -use crate::bindings::_helpers::planner_selection::model_selectors::{ - resolve_sqlbuild_model_selector_names, split_selector_expansion, -}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -30,6 +27,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_selection::main::resolve_sqlbuild_model_selector_names::resolve_sqlbuild_model_selector_names; +use sqlbuild_python_project::planner_selection::main::split_selector_expansion::split_selector_expansion; const MODEL_RESOURCE_TYPE: &str = "model"; const SELECTION_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index f34d93ed9b..11cd978ce0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -31,7 +31,6 @@ use crate::bindings::_helpers::diff_report::support::{ call, caused, cli_error, error_text, is_exception, matches_class, matches_native, none, object, request_facts, user_error_parts, }; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::frontier::main::emit_error::emit_error; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const DEFAULT_DIFF_MAX_FULL_ROWS: i64 = 10_000_000; const ADAPTER_ERRORS: AdapterHost = AdapterHost::AdapterUserError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 2f51322b96..1187ed8461 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -47,10 +47,11 @@ use crate::bindings::_helpers::compile_delta::driver::{ use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; use crate::bindings::_helpers::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; -use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; use sqlbuild_python_front::attachments::constants::MICROBATCH_END_SENTINEL; use sqlbuild_python_front::attachments::constants::MICROBATCH_START_SENTINEL; use sqlbuild_python_front::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; +use sqlbuild_python_project::semantic_shapes::constants::VALID_FROM_DEFAULT; +use sqlbuild_python_project::semantic_shapes::constants::VALID_TO_DEFAULT; /// Text constants exported under their Python names. pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs index 846cc60777..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -1,5 +1 @@ -pub(crate) mod latest_rows; -pub(crate) mod query_hashes; pub(crate) mod registration; -pub(crate) mod storage_io; -pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs index 717efad9a0..36c1fdc4c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::fingerprints::{latest_rows, query_hashes, storage_io}; +use sqlbuild_python_project::fingerprints::main::register_latest_rows::register_latest_rows; +use sqlbuild_python_project::fingerprints::main::register_query_hashes::register_query_hashes; +use sqlbuild_python_project::fingerprints::main::register_storage_io::register_storage_io; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - query_hashes::register(module)?; - storage_io::register(module)?; - latest_rows::register(module) + register_query_hashes(module)?; + register_storage_io(module)?; + register_latest_rows(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 539fb0eec0..5b4028aeb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -15,8 +15,6 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::main::within::within; @@ -36,6 +34,8 @@ use sqlbuild_python_front::discovery::main::records::discover_schema_files::disc use sqlbuild_python_front::discovery::main::records::discover_source_files::discover_source_files; use sqlbuild_python_front::discovery::main::records::discover_test_files::discover_test_files; use sqlbuild_python_front::discovery::main::snapshots::directory_snapshot_scope::directory_snapshot_scope; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const SELECTION_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 8c6b61c763..5b59678efc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -17,14 +17,9 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::compiled_project::snapshot_execution::VerifiedRun; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; -use crate::bindings::_helpers::planner_selection::selectors::{ - expand_project_upstream_keys, select_project_graph, -}; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -32,6 +27,10 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::compiled_project::types::VerifiedRun; +use sqlbuild_python_project::planner_selection::main::expand_project_upstream_keys::expand_project_upstream_keys; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index 169e044634..873f93551f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::freshness::support::{ OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_datetime, presumed_utc_value, text, }; -use crate::bindings::_helpers::references::source_relations::render_source_read; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -29,6 +28,7 @@ use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; use sqlbuild_python_front::state_rows::main::event_rows::normalize::normalize; use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; use sqlbuild_python_front::state_rows::main::event_rows::version_hash::version_hash; +use sqlbuild_python_project::references::main::render_source_read::render_source_read; const ADAPTER_STRATEGY: &str = "adapter"; const COLUMN_STRATEGY: &str = "column"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index aabf7ddd0d..14b5a97b82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::freshness::observation::{ }; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::text; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -32,6 +31,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::is_warehouse_access_refused::is_warehouse_access_refused; use sqlbuild_python_front::state_rows::main::event_rows::age_policy::age_policy; use sqlbuild_python_front::state_rows::main::event_rows::equivalent::equivalent; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs deleted file mode 100644 index 3b28885e9b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! `CompiledObjectKey`s for native `(resource type, name)` pairs, converted once per pair. - -use std::cell::RefCell; -use std::collections::HashMap; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - -/// One `CompiledObjectKey` per native pair, so shared keys convert once. -pub(crate) struct KeyCache<'py> { - key_type: Bound<'py, PyAny>, - resource_type: Bound<'py, PyAny>, - keys: RefCell>>, -} - -impl<'py> KeyCache<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { - Ok(Self { - key_type: record_class(py, RecordId::CompiledObjectKey)?, - resource_type: record_class(py, RecordId::CompiledResourceType)?, - keys: RefCell::new(HashMap::new()), - }) - } - - pub(crate) fn py(&self) -> Python<'py> { - self.key_type.py() - } - - /// The `CompiledObjectKey` for one `(resource type, name)` pair. - pub(crate) fn key( - &self, - (resource_type, name): &(String, String), - ) -> PyResult> { - let pair = (resource_type.clone(), name.clone()); - if let Some(existing) = self.keys.borrow().get(&pair) { - return Ok(existing.bind(self.py()).clone()); - } - let created: Bound<'py, PyAny> = self - .key_type - .call1((self.resource_type.call1((resource_type,))?, name))?; - self.keys - .borrow_mut() - .insert(pair, created.clone().unbind()); - Ok(created) - } - - /// `CompiledObjectKey`s for `pairs`, in order. - pub(crate) fn keys(&self, pairs: &[(String, String)]) -> PyResult>> { - pairs.iter().map(|pair| self.key(pair)).collect() - } -} - -/// The native `(resource type, name)` pair of a `CompiledObjectKey`. -pub(crate) fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - Ok(( - native_field(key, "resource_type")?.str()?.extract()?, - native_field(key, "name")?.extract()?, - )) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 0db6458b40..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,11 +1 @@ -pub(crate) mod algorithms; -pub(crate) mod execution_gates; -pub(crate) mod getters; -pub(crate) mod keys; -pub(crate) mod node_set; -pub(crate) mod project_graph; -pub(crate) mod python_lifecycle; -pub(crate) mod python_node_graph; -pub(crate) mod python_selection; pub(crate) mod registration; -pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs deleted file mode 100644 index 488de612b0..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! A Python set as the visited nodes of a native graph walk over Python keys. - -use pyo3::PyErr; -use pyo3::prelude::{Bound, PyAny}; -use pyo3::types::{PySet, PySetMethods}; -use sqlbuild_core::graph_walk::types::NodeSet; - -/// Python keys a walk visited, compared with Python hashing and equality. -pub(crate) struct PyNodeSet<'py>(pub(crate) Bound<'py, PySet>); - -impl<'py> NodeSet> for PyNodeSet<'py> { - type Error = PyErr; - - fn insert_new(&mut self, key: &Bound<'py, PyAny>) -> Result { - if self.0.contains(key)? { - return Ok(false); - } - self.0.add(key)?; - Ok(true) - } - - fn holds(&self, key: &Bound<'py, PyAny>) -> Result { - self.0.contains(key) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs deleted file mode 100644 index e8836bcbaa..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! The compiled project graph and selector resolution, held natively for Python's graph users. - -use sqlbuild_python_base::positional_record; -use std::collections::HashMap; - -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::{pyclass, pymethods}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::errors::SelectorError; -use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::main::build_resources::build_resources; -use sqlbuild_analysis::graph::main::edge_keys::edge_keys; -use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; -use sqlbuild_analysis::graph::main::match_selector::match_selector; -use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; -use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; -use sqlbuild_analysis::graph::models::{ - BuildResources, EdgeDirection, GraphIndexes, GraphResource, ProjectGraph, -}; - -use crate::bindings::_helpers::graph::views::lineage_graph_views; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -positional_record! { - extract; - /// `(key, deps, tags, model folder)`. - pub(crate) struct ResourceRow { - pub(crate) key: ObjectKey, - pub(crate) deps: Vec, - pub(crate) tags: Vec, - pub(crate) folder: Option, - } -} -type Edges = Vec<(ObjectKey, Vec)>; -positional_record! { - extract, into; - /// `(names, upstream, downstream, tags, paths)` in Python's dict order. - struct IndexesRow { - names: Vec<(String, ObjectKey)>, - upstream: Edges, - downstream: Edges, - tags: Vec<(String, Vec)>, - paths: Vec<(ObjectKey, String)>, - } -} -/// `(unique id, neighbour unique ids)` in graph order. -type UniqueIdMap = Vec<(String, Vec)>; -/// `PlannerInputError`'s `(code, message, help)`. -type FailureRow = (&'static str, String, Option); -/// The keys, or the `PlannerInputError` Python raises instead. -type KeysOutcome = (Option>, Option); - -/// A project graph: lineage edges, tag, path and name indexes, and selector resolution. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeProjectGraph { - graph: ProjectGraph, -} - -/// Static compiled project graph held natively; dict views are built only when read. -#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] -pub(crate) struct CompiledProjectGraph { - project: Py, - native: Py, - views: PyOnceLock>, -} - -fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { - deps.iter().map(|dep| (dep.clone(), key.clone())).collect() -} - -fn outcome(result: Result, SelectorError>) -> KeysOutcome { - match result { - Ok(keys) => (Some(keys), None), - Err(error) => (None, Some(failure(error))), - } -} - -fn failure(error: SelectorError) -> FailureRow { - (error.code, error.message, error.help) -} - -impl NativeProjectGraph { - /// The graph this object wraps. - pub(crate) fn graph(&self) -> &ProjectGraph { - &self.graph - } - - /// Wrap a graph another native stage built, such as project assembly. - pub(crate) fn new(graph: ProjectGraph) -> Self { - Self { graph } - } -} - -impl CompiledProjectGraph { - /// The graph of `project` over its native lineage graph. - pub(crate) fn new(project: Py, native: Py) -> Self { - Self { - project, - native, - views: PyOnceLock::new(), - } - } - - /// The compiled project this graph indexes. - pub(crate) fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { - self.project.bind(py) - } - - /// The native lineage graph. - pub(crate) fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { - self.native.bind(py) - } - - fn view(&self, py: Python<'_>, name: &str) -> PyResult> { - Ok(native_field(self.lineage_views(py)?.bind(py), name)?.unbind()) - } -} - -#[pymethods] -impl CompiledProjectGraph { - #[new] - #[pyo3(signature = (*, project, native))] - fn py_new(project: Py, native: Py) -> Self { - Self::new(project, native) - } - - #[getter(project)] - pub(crate) fn project_attribute(&self, py: Python<'_>) -> Py { - self.project.clone_ref(py) - } - - #[getter(native)] - pub(crate) fn native_attribute(&self, py: Python<'_>) -> Py { - self.native.clone_ref(py) - } - - /// Lineage edges and selector indexes, converted once. - #[getter] - pub(crate) fn lineage_views(&self, py: Python<'_>) -> PyResult> { - self.views - .get_or_try_init(py, || { - Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) - }) - .map(|views| views.clone_ref(py)) - } - - #[getter] - pub(crate) fn upstream_deps(&self, py: Python<'_>) -> PyResult> { - self.view(py, "upstream_deps") - } - - #[getter] - pub(crate) fn downstream_deps(&self, py: Python<'_>) -> PyResult> { - self.view(py, "downstream_deps") - } - - #[getter] - pub(crate) fn tag_index(&self, py: Python<'_>) -> PyResult> { - self.view(py, "tag_index") - } - - #[getter] - pub(crate) fn path_index(&self, py: Python<'_>) -> PyResult> { - self.view(py, "path_index") - } - - #[getter] - pub(crate) fn all_keys(&self, py: Python<'_>) -> PyResult> { - self.view(py, "all_keys") - } -} - -/// The project graph indexes of these resources, in project order. -pub(crate) fn project_graph(resources: Vec) -> PyResult { - compiler_guard(|| { - let resources: Vec = resources - .into_iter() - .map( - |ResourceRow { - key, - deps, - tags, - folder, - }| GraphResource { - key, - deps, - tags, - folder, - }, - ) - .collect(); - Ok(NativeProjectGraph::new(build_project_graph(&resources))) - }) -} - -#[pymethods] -impl NativeProjectGraph { - /// Build the indexes from resources in project order: models, sources, seeds, functions. - #[staticmethod] - fn from_resources(resources: Vec) -> PyResult { - project_graph(resources) - } - - /// Wrap indexes a caller already built, such as the planner's. - #[staticmethod] - fn from_indexes(indexes: IndexesRow) -> PyResult { - compiler_guard(|| { - let IndexesRow { - names, - upstream, - downstream, - tags, - paths, - } = indexes; - Ok(Self::new(graph_from_indexes(GraphIndexes { - names, - upstream, - downstream, - tags, - paths, - }))) - }) - } - - /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. - fn indexes(&self) -> IndexesRow { - let graph: &ProjectGraph = &self.graph; - IndexesRow { - names: graph.names.clone(), - upstream: graph.upstream.clone(), - downstream: graph.downstream.clone(), - tags: graph.tags.clone(), - paths: graph.paths.clone(), - } - } - - /// The key's direct upstream lineage keys, in authored dependency order. - pub(crate) fn upstream(&self, key: ObjectKey) -> Vec { - edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() - } - - /// The key's direct downstream keys, sorted by `(resource type, name)`. - fn downstream(&self, key: ObjectKey) -> Vec { - edge_keys(&self.graph, &key, EdgeDirection::Downstream).to_vec() - } - - /// Every selectable key, in project order with later same-name resources replacing earlier. - fn keys(&self) -> Vec { - self.graph - .names - .iter() - .map(|(_, key)| key.clone()) - .collect() - } - - /// `(selector name, key)` in project order. - fn names(&self) -> Vec<(String, ObjectKey)> { - self.graph.names.clone() - } - - /// Every lineage edge as `(dependency, dependent)`, in upstream order. - fn edges(&self) -> Vec<(ObjectKey, ObjectKey)> { - self.graph - .upstream - .iter() - .flat_map(|(key, deps)| dependency_edges(key, deps)) - .collect() - } - - /// The manifest's `(parent_map, child_map)` entries as dbt-style unique ids, in graph order. - fn unique_id_maps( - &self, - project_name: &str, - prefixes: HashMap, - ) -> (UniqueIdMap, UniqueIdMap) { - ( - unique_id_edges( - &self.graph, - EdgeDirection::Upstream, - project_name, - &prefixes, - ), - unique_id_edges( - &self.graph, - EdgeDirection::Downstream, - project_name, - &prefixes, - ), - ) - } - - /// The compile report's execution layer count over models. - fn model_layer_count(&self) -> usize { - model_layer_count(&self.graph) - } - - /// `--select` minus `--exclude`, plus the functions they need to build. - fn resolve(&self, select: Vec, exclude: Vec) -> PyResult { - compiler_guard(|| Ok(outcome(resolve_selectors(&self.graph, &select, &exclude)))) - } - - /// The keys selector tokens match, without build resources. - fn tokens(&self, selectors: Vec) -> PyResult { - compiler_guard(|| Ok(outcome(resolve_selector_tokens(&self.graph, &selectors)))) - } - - /// The keys one `kind:value` selector names before `+` expansion. - fn matched(&self, kind: &str, value: &str) -> PyResult { - compiler_guard(|| Ok(outcome(match_selector(&self.graph, kind, value)))) - } - - /// The selected keys plus the functions and seeds Python adds to build them. - fn build_resources( - &self, - selected: Vec, - include: (bool, bool, bool), - ) -> PyResult> { - compiler_guard(|| { - let (upstream_functions, upstream_seeds, downstream_functions) = include; - Ok(build_resources( - &self.graph, - &selected, - BuildResources { - upstream_functions, - upstream_seeds, - downstream_functions, - }, - )) - }) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index 971b8975e3..af6b8d4689 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -1,11 +1,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::graph::project_graph::register(module)?; - crate::bindings::_helpers::graph::algorithms::register(module)?; - crate::bindings::_helpers::graph::execution_gates::register(module)?; - crate::bindings::_helpers::graph::python_selection::register(module)?; - crate::bindings::_helpers::graph::python_lifecycle::register(module)?; - crate::bindings::_helpers::graph::python_node_graph::register(module)?; - crate::bindings::_helpers::graph::views::register(module) + sqlbuild_python_project::graph::main::project_graph::register_project_graph::register_project_graph(module)?; + sqlbuild_python_project::graph::main::algorithms::register_algorithms::register_algorithms( + module, + )?; + sqlbuild_python_project::graph::main::execution_gates::register_execution_gates::register_execution_gates(module)?; + sqlbuild_python_project::graph::main::python_selection::register_python_selection::register_python_selection(module)?; + sqlbuild_python_project::graph::main::python_lifecycle::register_python_lifecycle::register_python_lifecycle(module)?; + sqlbuild_python_project::graph::main::python_node_graph::register_python_node_graph::register_python_node_graph(module)?; + sqlbuild_python_project::graph::main::views::register_views::register_views(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index 5740df9c2a..9a364b646f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::analysis::binding_locations::{ }; use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; -use crate::bindings::models::Positional; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; @@ -16,6 +15,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::analysis::models::Positional; type Object<'py> = Bound<'py, PyAny>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index de4ef660c3..a12c692d64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -14,23 +14,10 @@ use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCR use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::DiagnosticSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::LoaderRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestBlockRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestCaseRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::input_loops::resource_attachment::{ attach_audits, attach_functions, attach_sources, attach_sql_tests, }; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -44,6 +31,19 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::AuditBlockRow; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::TestBlockRow; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::LoaderRow; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::TestCaseRow; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index 7c03d6d5f0..f52c5310b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -22,10 +22,6 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - AuditBlockRow, AuditFileRow, AuditSettings, -}; -use crate::bindings::_helpers::compiled_views::audit_views::{AttachedAudit, AuditHandles}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; @@ -47,6 +43,11 @@ use sqlbuild_python_front::model_config::models::HeaderAudits; use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::model_config::models::PyNode; +use sqlbuild_python_project::compiled_project::models::AuditBlockRow; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_project::compiled_views::models::AuditHandles; +use sqlbuild_python_project::compiled_views::types::AttachedAudit; const THRESHOLDS_KEY: &str = "thresholds"; const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 19050d2648..11da72c9af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -28,13 +28,6 @@ use sqlbuild_attachments::functions::models::{ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use crate::bindings::_helpers::analysis_session::query_columns::query_columns_error; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - FunctionRow, FunctionSettings, -}; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, FunctionHandles, -}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, }; @@ -50,6 +43,11 @@ use sqlbuild_project::types::NamedTypeRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; use sqlbuild_python_front::model_config::models::NativeModelValidator; +use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_views::models::FunctionHandles; +use sqlbuild_python_project::compiled_views::types::AttachedFunction; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs index b3f4d41d67..589559d103 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs @@ -378,10 +378,7 @@ pub(crate) fn model_loop_classes<'py>( py, vec![( "declaration_rows", - pyo3::wrap_pyfunction!( - crate::bindings::_helpers::compiled_views::records::declaration_rows, - py - )? + sqlbuild_python_project::compiled_views::main::records::declaration_rows_function::declaration_rows_function(py)? .into_any(), )], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 962dbc097b..7e1e2b5e99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -31,23 +31,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SqlTestSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::AttachedAudit; -use crate::bindings::_helpers::compiled_views::function_views::AttachedFunction; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, NativeSourceInput, retain_sources, -}; use crate::bindings::_helpers::input_loops::audit_loop::{AttachmentOwner, attached_audits}; use crate::bindings::_helpers::input_loops::function_loop::{ attached_functions, checked_table_function_arities, @@ -116,6 +99,23 @@ use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_bat use sqlbuild_python_front::sqltext::models::InterpolationRow as SqlInterpolationRow; use sqlbuild_python_front::sqltext::models::InterpolationSources; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_views::main::source_views::retain_sources::retain_sources; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; +use sqlbuild_python_project::compiled_views::models::NativeSourceInput; +use sqlbuild_python_project::compiled_views::types::AttachedAudit; +use sqlbuild_python_project::compiled_views::types::AttachedFunction; +use sqlbuild_python_project::compiled_views::types::AttachedSource; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index 0becc419a9..87ff131735 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -17,7 +17,6 @@ use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_model_config::model_validation::models::ColumnMigration; use sqlbuild_model_config::types::NodeKind; -use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, compile_error, config_error, }; @@ -46,6 +45,7 @@ use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::HeaderFamilies; use sqlbuild_python_front::model_config::models::NativeColumnSpec; use sqlbuild_python_front::model_config::types::HeaderValue; +use sqlbuild_python_project::compiled_project::main::schema_facts::family_fact::family_fact; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index d75cfcf5b0..be3d594607 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -13,15 +13,6 @@ use sqlbuild_analysis::compiled_project::models::{ use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_discovery::model_files::models::HeaderFacts; -use crate::bindings::_helpers::compiled_project::project::{ - CONTRACT_KEY, INCREMENTAL_MODE_KEY, materialization, text_value, -}; -use crate::bindings::_helpers::compiled_project::project::{ - DeclarationPy, config_values, declarations, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputProjections, ModelInputHandles, ModelInputRecord, SchemaHandles, SchemaRecord, -}; use crate::bindings::_helpers::input_loops::model_loop_header::AttachedHeader; use crate::bindings::_helpers::input_loops::model_loop_inputs::{ HOOK_KEYS, reference_fact, stripped_values, @@ -41,6 +32,18 @@ use sqlbuild_python_front::render::models::ExpandedDeclarations; use sqlbuild_python_front::render::models::ExpansionOutput; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::models::ScannedReference; +use sqlbuild_python_project::compiled_project::constants::CONTRACT_KEY; +use sqlbuild_python_project::compiled_project::constants::INCREMENTAL_MODE_KEY; +use sqlbuild_python_project::compiled_project::main::project::config_values::config_values; +use sqlbuild_python_project::compiled_project::main::project::declarations::declarations; +use sqlbuild_python_project::compiled_project::main::project::materialization::materialization; +use sqlbuild_python_project::compiled_project::main::project::text_value::text_value; +use sqlbuild_python_project::compiled_project::models::DeclarationPy; +use sqlbuild_python_project::compiled_views::models::InputProjections; +use sqlbuild_python_project::compiled_views::models::ModelInputHandles; +use sqlbuild_python_project::compiled_views::models::SchemaHandles; +use sqlbuild_python_project::compiled_views::types::ModelInputRecord; +use sqlbuild_python_project::compiled_views::types::SchemaRecord; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index 7e6418483a..c3618fe353 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -22,10 +22,10 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::input_loops::model_loop_prepass::spawned; use sqlbuild_python_base::cache::main::parse_key::parse_key; use sqlbuild_python_base::cache::models::NativeParseStore; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs index 0a25bfd463..94f123ba8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs @@ -5,11 +5,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::input_loops::model_loop_inputs::present; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::models::HeaderColumn; +use sqlbuild_python_project::compiled_project::main::schema_facts::python_column_fact::python_column_fact; const MODEL_SCHEMA_KEY: &str = "model_schema"; const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs index b18d7efcaa..98aceeb651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs @@ -1,10 +1,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compiled_project::attachment_inputs; use crate::bindings::_helpers::input_loops::resource_attachment::RESOURCE_ATTACHMENT; +use sqlbuild_python_project::compiled_project::main::attachment_inputs::install::install; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - attachment_inputs::install(RESOURCE_ATTACHMENT); + install(RESOURCE_ATTACHMENT); crate::bindings::_helpers::input_loops::model_loop::register(module)?; crate::bindings::_helpers::input_loops::attachment::register(module)?; crate::bindings::_helpers::input_loops::expanded_offsets::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index 9b584bb8e4..eeeffff647 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -4,17 +4,6 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - AttachedSqlTests, AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, - ResourceAttachment, ScenarioRow, SourceRow, SourceSettings, SqlTestSettings, TestFileRow, -}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, NativeFunctionInput, -}; -use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; -use crate::bindings::_helpers::compiled_views::source_views::{AttachedSource, NativeSourceInput}; use crate::bindings::_helpers::input_loops::audit_loop::{ AttachmentOwner, model_owner, resource_owner, }; @@ -22,6 +11,25 @@ use crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::ResourceAttachment; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_views::models::NativeAuditInput; +use sqlbuild_python_project::compiled_views::models::NativeFunctionInput; +use sqlbuild_python_project::compiled_views::models::NativeSourceInput; +use sqlbuild_python_project::compiled_views::models::SeedParts; +use sqlbuild_python_project::compiled_views::types::AttachedFunction; +use sqlbuild_python_project::compiled_views::types::AttachedSource; +use sqlbuild_python_project::compiled_views::types::SeedViews; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; const SOURCE_OWNER: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 4ea914fa10..80d65ebb2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -22,12 +22,6 @@ use sqlbuild_pysource::definition_locations::main::decorator_location::decorator use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - DiagnosticSettings, HookDescriptionRow, LoaderRow, PythonResourceRow, ResourceDiagnostics, - ScenarioDescriptionRow, -}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; @@ -35,6 +29,15 @@ use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; +use sqlbuild_python_project::compiled_project::models::HookDescriptionRow; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::PythonResourceRow; +use sqlbuild_python_project::compiled_project::models::ScenarioDescriptionRow; +use sqlbuild_python_project::compiled_project::types::LoaderRow; +use sqlbuild_python_project::compiled_project::types::ResourceDiagnostics; +use sqlbuild_python_project::compiled_views::models::SeedParts; +use sqlbuild_python_project::compiled_views::types::SeedViews; const MISSING_DESCRIPTION_CODE: &str = "P010"; const DESCRIPTION_LENGTH_CODE: &str = "P027"; @@ -402,7 +405,7 @@ fn docstring_text(description: &Bound<'_, PyString>) -> String { /// A source's P010, falling back to its loader's description and naming the loader in help. fn source_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - source: &crate::bindings::_helpers::compiled_views::source_views::SourceRecord, + source: &sqlbuild_python_project::compiled_views::types::SourceRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = source.attached(); @@ -449,7 +452,7 @@ fn source_description<'py>( /// A SQL or Python function's P010, located at its header or `@udf` decorator. fn function_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - function: &crate::bindings::_helpers::compiled_views::function_views::FunctionRecord, + function: &sqlbuild_python_project::compiled_views::types::FunctionRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = &function.attached; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs index 3745d12287..5147d83a59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs @@ -14,7 +14,6 @@ use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenari use sqlbuild_attachments::test_targets::models::ScenarioCteSources; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; use crate::bindings::_helpers::input_loops::model_loop::compile_error; use crate::bindings::_helpers::input_loops::model_loop_calls::origin_references; use crate::bindings::_helpers::input_loops::source_loop::{ @@ -24,6 +23,7 @@ use crate::bindings::_helpers::input_loops::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index fb42efb29e..b2f49ad343 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -7,10 +7,6 @@ use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_fre use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{SourceRow, SourceSettings}; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, SourceHandles, replaced, -}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, record_reads, }; @@ -39,6 +35,11 @@ use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_views::main::source_views::replaced::replaced; +use sqlbuild_python_project::compiled_views::models::SourceHandles; +use sqlbuild_python_project::compiled_views::types::AttachedSource; const MACRO_TOKEN: char = '@'; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 5a75d64f90..7ae05f852a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -35,10 +35,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - SqlTestSettings, TestBlockRow, TestCaseRow, TestFileRow, -}; -use crate::bindings::_helpers::compiled_views::source_views::replaced; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; @@ -52,6 +48,11 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::sql_macros::without_with_list_calls::without_with_list_calls; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestBlockRow; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::TestCaseRow; +use sqlbuild_python_project::compiled_views::main::source_views::replaced::replaced; const MODEL_MODE: &str = "model"; const MACRO_MODE: &str = "macro"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs index 97e2c26b3b..b36f00df8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs @@ -26,8 +26,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ relation_age_timestamp, sorted_schemas, text, timestamp_fields, tuple, utc_datetime, }; use crate::bindings::_helpers::planner_scenario::artifacts::is_scenario_artifact_physical_name; -use crate::bindings::_helpers::relation_naming::fitting::shared_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; /// Protection rules for one plan. pub(crate) struct Protection { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 96bee69a6d..ac79a21cd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -38,7 +38,6 @@ use crate::bindings::_helpers::janitor::janitor_phases::{ }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::scenario_command::setup::stream_isatty; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -47,6 +46,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_project::planner_phases::main::object::object; const COMMAND_NAME: &str = "janitor"; /// The code of a cleanup that needs a confirmation stdin cannot give. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index b01d11d52c..c66981ddf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::janitor_planning::constants::UNTRACKED_QUERY_DIFF_REASON; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, dedupe_schema_keys, folded_schema, integer, optional, record, sorted_schemas, text, tuple, @@ -18,6 +17,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; const INSPECTION_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 4da294beb4..b5010bee7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_nam use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, }; @@ -21,6 +20,7 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_front::state_rows::main::migrations::read_histories::read_histories as read_view_histories; use sqlbuild_python_front::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; @@ -219,7 +219,7 @@ fn project_destinations<'py>(project: &Bound<'py, PyAny>) -> PyResult = project.py(); let destinations: Bound<'py, PyDict> = PyDict::new(py); for (key, model) in model_destinations(project)? { - destinations.set_item(key.to_python(py)?, model_reads::name_object(&model)?)?; + destinations.set_item(key.to_python(py)?, name_object(&model)?)?; } Ok(destinations) } @@ -377,7 +377,10 @@ pub(crate) fn pending_migration_origins( if listed.contains(&destination.folded()) { continue; } - let model_name: String = model_reads::model_name(&model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( + &model, + )?; for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { let folded: Folded = Key { database: origin.database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 671e95bf4b..c8b64ad572 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -28,13 +28,13 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ use crate::bindings::_helpers::janitor::janitor_old_names::{ OldNamePlanning, pending_migration_origins, plan_old_name_views, }; -use crate::bindings::_helpers::relation_naming::fitting::shared_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; const LIFECYCLE: ExecutorHost = ExecutorHost::OperationLifecycle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index dbbccf2edb..0346325648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -11,11 +11,12 @@ use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; +use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; /// Where a compact model's facts are: given, or projected from its compiled model on first read. pub(crate) enum CompactColumns { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 1b11fbaa7d..3ba074d40d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -38,13 +38,6 @@ use crate::bindings::_helpers::lineage::lineage_cache::{ }; use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; use crate::bindings::_helpers::lineage::project_lineage::build_direct_semantic_uses; -use crate::bindings::_helpers::planner_selection::selectors::resolve_project_selector_tokens; -use crate::bindings::_helpers::planner_selection::selectors::resolve_project_selectors; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::{ - match_project_selector, parse_project_selector, -}; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::build_progress::main::transient_status::maybe_status::maybe_status; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; @@ -56,6 +49,12 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::planner_selection::main::match_project_selector::match_project_selector; +use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::resolve_project_selector_tokens::resolve_project_selector_tokens; +use sqlbuild_python_project::planner_selection::main::resolve_project_selectors::resolve_project_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index 5f73ca5eb7..26032298d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod column_lineage_graph; pub(crate) mod lineage_cache; pub(crate) mod lineage_command; -pub(crate) mod lineage_facts; pub(crate) mod lineage_index; pub(crate) mod lineage_objects; pub(crate) mod project_lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 605c5b84a1..9cf0abb825 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -21,8 +21,6 @@ use sqlbuild_analysis::lineage_graph::main::analysis_dialect::analysis_dialect; use sqlbuild_analysis::lineage_graph::models::AnalysisDialect; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::star_flags; use crate::bindings::_helpers::lineage::column_lineage_graph::{ CompactColumns, CompactModel, ProjectColumnLineage, }; @@ -31,11 +29,13 @@ use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, }; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::compiled_views::main::model_reads::star_flags::star_flags; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const FAST_MODE: &str = "fast"; const RICH_MODE: &str = "rich"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs index 975b6b7511..1092f50850 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs @@ -5,8 +5,9 @@ use sqlbuild_analysis::lineage::models::{ LineageResourceType, LineageSchemaResource, RichSchemaResource, }; -use crate::bindings::_helpers::compiled_views::model_reads::{declared_rows, inferred_rows}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::declared_rows::declared_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; type TypedColumns = Vec<(String, Option)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 4169796e01..a1d82374fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -29,8 +29,6 @@ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::result_output::documents::format_load_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -54,6 +52,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const EXECUTION_COMMAND: &str = "sqb load"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 937ce717d6..5f0ff9bd7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -20,9 +20,6 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::NodeKeys; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::snapshot::{ @@ -39,6 +36,9 @@ use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::compiled_project::types::NodeKeys; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; fn status(py: Python<'_>, message: &str) -> PyResult<()> { write_stderr(py, &format!("{message}\n")) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 7b5c76219a..86260de532 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -14,21 +14,9 @@ use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - compile_declarations, python_declarations, -}; -use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; -use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; -use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; -use crate::bindings::_helpers::planner_scope::values::project_graph; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; @@ -36,6 +24,17 @@ use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_project::compiled_project::main::snapshot_declarations::compile_declarations::compile_declarations; +use sqlbuild_python_project::compiled_project::main::snapshot_declarations::python_declarations::python_declarations; +use sqlbuild_python_project::compiled_project::main::snapshot_models::model_nodes::model_nodes; +use sqlbuild_python_project::compiled_project::main::snapshot_resources::resource_nodes::resource_nodes; +use sqlbuild_python_project::compiled_project::main::snapshot_roles::compile_project_roles::compile_project_roles; +use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::graph::models::ProjectSelection; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::project_graph::project_graph; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index 4090cb4ff6..edb35a9292 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -6,7 +6,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use serde_json::{Value, json}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_path; use crate::bindings::_helpers::location_command::constants::{ MODEL_NODE_KIND, SNAPSHOT_HIDDEN_SETTING, SOURCE_NODE_KIND, }; @@ -14,6 +13,7 @@ use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_se use sqlbuild_python_base::orchestration_reads::main::provider_class_settings::provider_class_settings; use sqlbuild_python_base::orchestration_reads::main::setting_rows::setting_rows; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_project::compiled_project::main::snapshot_boundary::relative_path::relative_path; /// How a consumer's code runs inside other snapshot nodes, besides its own node. #[derive(Clone, Copy)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index c1507efb2e..4fc281df93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -27,11 +27,6 @@ use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, wareho use crate::bindings::_helpers::command_runtime::warehouses::{ WarehouseInputs, configured_warehouses, }; -use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild}; -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; -use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::{ @@ -52,6 +47,12 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::compiled_project::models::NativeOrchestrationWorker; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::compiled_project::types::NodeKeys; +use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; /// Each audit and SQL test the snapshot publishes, with the node keys it checks. fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index fb8434e43a..e1f2510e89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -13,8 +13,6 @@ use sqlbuild_analysis::manifest::models::ManifestInput; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::native_graph_from_edges; use crate::bindings::_helpers::manifest::rows::{ ProjectRows, plan_input, project_rows, python_hooks, }; @@ -24,6 +22,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::main::views::native_graph_from_edges::native_graph_from_edges; +use sqlbuild_python_project::graph::models::NativeProjectGraph; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index a3c45ce10a..87df77c561 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,8 +1,6 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. -pub(crate) mod model_lookup; pub(crate) mod orchestration_sensors; -pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod resource_costs_state; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 6216853b0b..af841ab91b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -11,7 +11,6 @@ use sqlbuild_orchestration::sensors::models::{ CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, }; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -24,6 +23,7 @@ use sqlbuild_python_front::discovery::main::python_files::provider_by_name::prov use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::compiled_project::main::snapshot_declarations::python_declarations::python_declarations; pub(crate) fn evaluate_sensor( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 6aebbf4d28..aa60faa737 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -34,8 +34,6 @@ use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ @@ -51,6 +49,8 @@ use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; use sqlbuild_python_front::authored_values::main::sql_value::sql_value; use sqlbuild_python_front::authored_values::models::SqlValueClasses; +use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; /// One read-only adapter session serves every page of one synchronization. pub(crate) struct HistorySession<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index 069c096f37..3cde5a20f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -8,10 +8,8 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; -use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; use crate::bindings::_helpers::consumption::relation_reads::relation; use crate::bindings::_helpers::observations::source_capture::take_source_capture; -use crate::bindings::_helpers::planner_phases::objects::config_str; use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -21,6 +19,10 @@ use sqlbuild_python_front::native_host::main::append_observations::append_observ use sqlbuild_python_front::native_host::models::ConsumptionConnection; use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; +use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; /// Dependency kind of a source in a model's `dep_rows`. const SOURCE_DEPENDENCY: &str = "source"; @@ -43,9 +45,9 @@ fn cursor_reads( ) -> PyResult> { let succeeded: HashSet<&str> = successful_models.iter().map(String::as_str).collect(); let mut reads: Vec = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let name: String = model_reads::name_object(&model)?.extract()?; + let name: String = name_object(&model)?.extract()?; if !succeeded.contains(name.as_str()) || config_str(&model, "materialized")?.as_deref() != Some("incremental") { @@ -54,7 +56,10 @@ fn cursor_reads( let Some(cursor_column) = config_str(&model, "cursor")? else { continue; }; - let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let destination: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?; let qualified: Bound<'_, PyAny> = native_field(&destination, "qualified_name")?; if qualified.is_none() || native_field(&destination, "schema")?.is_none() { continue; @@ -90,13 +95,16 @@ fn source_column_rows( if described.is_empty() { return Ok(BTreeMap::new()); } - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let destination: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?; if native_field(&destination, "schema")?.is_none() { continue; } - let sources: Vec = model_reads::dep_rows(&model)? + let sources: Vec = dep_rows(&model)? .into_iter() .filter(|(kind, name)| kind == SOURCE_DEPENDENCY && described.contains_key(name)) .map(|(_, name)| name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index 16aae0f96a..a6a82dc62d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -11,8 +11,6 @@ use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::rules::catalogue::build_rule_catalogue; use crate::bindings::_helpers::rules::guidance::{format_rule, install_rules_skills}; use crate::bindings::_helpers::rules::results::{ @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const LIST_ACTION: &str = "list"; const SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 6d53e00ba8..2b433681a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -5,11 +5,11 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_scope::constants::MODEL; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CREATE_TABLE_ACTION: &str = "create_table"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 149d4844fc..cfcb2bfb8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -3,9 +3,6 @@ use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; @@ -14,6 +11,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_front::model_config::main::config::config_text::config_text; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::function_reads::language::language; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; use sqlbuild_analysis::planner_changes::constants::{ CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, @@ -31,20 +31,14 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::planner_changes::renames::origin_names; use crate::bindings::_helpers::planner_changes::results::{ backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, }; -use crate::bindings::_helpers::planner_identity::hashing::{ - baselined_function_definition_hash, baselined_model_query_hash, -}; use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -55,6 +49,12 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal as native_types_equal; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::star_flags::star_flags; +use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::semantic_shapes::main::snapshot_validity_columns::snapshot_validity_columns; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; @@ -114,20 +114,25 @@ pub(crate) fn detect_changes<'py>( let query_change_tracking: bool = native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + .try_iter()? + { let function: Bound<'py, PyAny> = function?; - let name: Bound<'py, PyAny> = function_reads::name(&function)?; + let name: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = vec![( "fingerprint_sql", PyString::new(py, &fingerprint_sql).into_any(), )]; - if selected - .selected - .contains(&crate::bindings::_helpers::graph::keys::native_key( - &function_reads::key(&function)?, - )?) - { + if selected.selected.contains( + &sqlbuild_python_project::graph::main::keys::native_key::native_key( + &sqlbuild_python_project::compiled_views::main::function_reads::key::key( + &function, + )?, + )?, + ) { let reason: &str = if full_refresh { "full_refresh" } else { @@ -140,9 +145,7 @@ pub(crate) fn detect_changes<'py>( &name.extract::()?, ( &fingerprint_sql, - &function_reads::language(&function)? - .str()? - .extract::()?, + &language(&function)?.str()?.extract::()?, dialect.as_deref(), ), &fingerprint, @@ -227,7 +230,9 @@ pub(crate) fn scope_model_changes<'py>( } }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &project_reads::functions(project)?, + &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )?, native_field(project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -260,7 +265,7 @@ pub(crate) fn scope_model_changes<'py>( let Some(model) = scope.model(py, &key.1) else { continue; }; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; let expected: (Option, Option) = ( lookup(&version_hashes, &name)?, lookup(&metadata_jsons, &name)?, @@ -284,7 +289,7 @@ pub(crate) fn model_change<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; - let model_name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let model_name: Bound<'py, PyAny> = name_object(model)?; let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, @@ -322,7 +327,7 @@ pub(crate) fn model_change<'py>( py, &model_name.extract::()?, ( - &model_reads::query_sql(model)?.extract::()?, + &query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), ), &fingerprint, @@ -368,7 +373,7 @@ pub(crate) fn model_change<'py>( let renames: Bound<'py, PyAny> = native_field(snapshot, "reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( - (model_reads::query_sql(model)?.extract()?, &fingerprint), + (query_sql(model)?.extract()?, &fingerprint), &renames, (&recorded.definition_hash, &dialect), )?, @@ -477,7 +482,7 @@ fn log_query_check( &log_sql, &[ ("logger", logger.clone()), - ("sql", model_reads::query_sql(model)?), + ("sql", query_sql(model)?), ("action", text("compiled_query")), ], )?; @@ -508,7 +513,10 @@ fn model_schema_findings<'py>( for column in &validity { warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); } - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 71ae68c2c9..704dad962f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -1,8 +1,8 @@ //! Model renames that downstream `__ref` names may have been rewritten for. -use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -13,9 +13,6 @@ use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_ref use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_identity::hashing::baselined_model_query_hash; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::value_error::value_error; @@ -24,6 +21,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::model_reads::references::references; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; @@ -235,13 +238,12 @@ fn changed_referencing_query( let ref_kind: Bound<'_, PyAny> = enum_member(py, RecordId::SqlReferenceKind, "REF")?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; for model in selected { - let recorded: Bound<'_, PyAny> = - mapping_get(&recorded_models, model_reads::name_object(&model)?)?; + let recorded: Bound<'_, PyAny> = mapping_get(&recorded_models, name_object(&model)?)?; if recorded.is_none() { continue; } let mut references_models: bool = false; - for reference in model_reads::references(&model)?.try_iter()? { + for reference in references(&model)?.try_iter()? { if native_field(&reference?, "ref_kind")?.eq(&ref_kind)? { references_models = true; break; @@ -250,11 +252,8 @@ fn changed_referencing_query( if references_models && baselined_model_query_hash( py, - &model_reads::model_name(&model)?, - ( - &model_reads::query_sql(&model)?.extract::()?, - dialect.as_deref(), - ), + &model_name(&model)?, + (&query_sql(&model)?.extract::()?, dialect.as_deref()), &recorded, )? != native_field(&recorded, "definition_hash")?.extract::()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 3ecf29754a..91738ddaaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -1,28 +1,16 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; use pyo3::PyRef; @@ -41,6 +29,19 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::graph::main::views::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::graph::models::PyNodeSet; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_selection::main::resolve_graph_selection::resolve_graph_selection; const ENABLED_KEY: &str = "enabled"; @@ -125,7 +126,7 @@ struct CloneResolution<'py> { /// Whether a selected, enabled model is recreated from its SQL rather than copied. fn recreated_view(model: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> PyResult { - Ok(selected.contains(model_reads::model_key(model)?)? + Ok(selected.contains(model_key(model)?)? && !disabled_model(model)? && materialization_of(model)?.eq(VIEW_MATERIALIZATION)?) } @@ -140,15 +141,23 @@ fn resolved_source_entries<'py>( let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let read = PySet::empty(py)?; let mut readers: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; if recreated_view(&model, &selected)? { readers.push(model); } } - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + .try_iter()? + { let function = function?; - if selected.contains(function_reads::key(&function)?)? { + if selected.contains( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )? { readers.push(function); } } @@ -163,8 +172,10 @@ fn resolved_source_entries<'py>( let mut entries: Vec> = Vec::new(); for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; - if read.contains(source_reads::name(&source)?)? { - entries.push(source_reads::source_entry(&source)?); + if read.contains( + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + )? { + entries.push(source_entry(&source)?); } } tuple_of(py, entries) @@ -176,18 +187,32 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = object(py, build_model_locations(&project_reads::models(project)?)?)?; - let seed_locations = object(py, build_seed_locations(&project_reads::seeds(project)?)?)?; + let model_locations = object( + py, + build_model_locations( + &sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)?, + )?, + )?; + let seed_locations = object( + py, + build_seed_locations( + &sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, + )?, + )?; let function_locations = object( py, - build_function_locations(&project_reads::functions(project)?)?, + build_function_locations( + &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )?, + )?, )?; let source_map = PyDict::new(py); for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; source_map.set_item( - source_reads::name(&source)?, - source_reads::source_entry(&source)?, + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + source_entry(&source)?, )?; } let source_warehouse_columns = object( @@ -223,8 +248,8 @@ fn model_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model_reads::model_key(model)?), - ("name", model_reads::name_object(model)?), + ("key", model_key(model)?), + ("name", name_object(model)?), ("relative_path", native_field(model, "relative_path")?), ("materialization_type", materialization_type), ("action", planner_type(py, RecordId::PlanAction, action)?), @@ -232,7 +257,7 @@ fn model_entry<'py>( "reason", planner_type(py, RecordId::PlanReason, "NO_CHANGE")?, ), - ("destination", model_reads::destination(model)?), + ("destination", sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(model)?), ("fingerprint_query_sql", string(py, "")), ("resolved_sql", resolved_sql), ("logical_ddl", string(py, "")), @@ -248,17 +273,30 @@ fn clone_model_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; - if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { + if !selected.contains(model_key(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )? + .try_iter()? + { let function = function?; - functions.set_item(function_reads::name(&function)?, function)?; + functions.set_item( + sqlbuild_python_project::compiled_views::main::function_reads::name::name( + &function, + )?, + function, + )?; } let context = Bound::new( py, @@ -303,7 +341,7 @@ fn clone_model_entries<'py>( string(py, "") }; entries.set_item( - model_reads::model_key(&model)?, + model_key(&model)?, model_entry(&model, materialization_type, resolved_sql)?, )?; } @@ -317,11 +355,15 @@ fn clone_source_entries<'py>( ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { let source = source?; - let entry = source_reads::source_entry(&source)?; - if !selected.contains(source_reads::key(&source)?)? - || !native_field(&entry, "managed")?.is_truthy()? + let entry = source_entry(&source)?; + if !selected.contains( + sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + )? || !native_field(&entry, "managed")?.is_truthy()? || native_field(&entry, "loader")?.is_none() || !native_field(&entry, "expression")?.is_none() { @@ -352,13 +394,23 @@ fn clone_source_entries<'py>( }), )?; entries.set_item( - source_reads::key(&source)?, + sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, planner_model( py, RecordId::CloneSourcePlanEntry, &[ - ("key", source_reads::key(&source)?), - ("name", source_reads::name(&source)?), + ( + "key", + sqlbuild_python_project::compiled_views::main::source_reads::key::key( + &source, + )?, + ), + ( + "name", + sqlbuild_python_project::compiled_views::main::source_reads::name::name( + &source, + )?, + ), ("destination", destination), ], )?, @@ -375,9 +427,14 @@ fn clone_function_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + .try_iter()? + { let function = function?; - if !selected.contains(function_reads::key(&function)?)? { + if !selected.contains( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )? { continue; } let body_sql = resolve_function_sql( @@ -427,7 +484,10 @@ fn clone_function_entries<'py>( ("packages", field("packages")?), ], )?; - entries.set_item(function_reads::key(&function)?, entry)?; + entries.set_item( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + entry, + )?; } in_order(&entries, &native_field(plan, "execution_order")?) } @@ -438,17 +498,17 @@ fn seed_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(seed)?), - ("name", seed_reads::name(seed)?), - ("destination", seed_reads::destination(seed)?), + ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(seed)?), + ("name", sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?), + ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(seed)?), ( "file_path", - native_field(&seed_reads::seed_file(seed)?, "file_path")?, + native_field(&seed_file(seed)?, "file_path")?, ), ("columns", PyTuple::empty(py).into_any()), ( "csv_settings", - native_field(&seed_reads::schema_entry(seed)?, "csv_settings")?, + native_field(&schema_entry(seed)?, "csv_settings")?, ), ], ) @@ -461,7 +521,9 @@ fn seed_entries<'py>( wanted: &Bound<'py, PyAny>, ) -> PyResult> { let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)? + .try_iter()? + { let seed = seed?; if wanted.contains(native_field(&seed, field)?)? { entries.push(seed_entry(&seed)?); @@ -476,9 +538,12 @@ fn origin_model_entries<'py>( ) -> PyResult> { let py = project.py(); let mut entries: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; - if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { + if !selected_names.contains(name_object(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = materialization_of(&model)?; @@ -503,12 +568,19 @@ fn origin_source_entries<'py>( let py = project.py(); let mut keys: Vec> = Vec::new(); let mut order: Vec> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { let source = source?; - if selected_names.contains(source_reads::name(&source)?)? { - keys.push(source_reads::key(&source)?); + if selected_names.contains( + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + )? { + keys.push( + sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + ); } - order.push(source_reads::key(&source)?); + order.push(sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?); } clone_source_entries( project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 9adb537838..9cd6b18a13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,10 +1,5 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, @@ -15,7 +10,6 @@ use crate::bindings::_helpers::planner_entries::values::{ }; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; @@ -33,6 +27,15 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_python_project::planner_phases::main::object::object; use std::collections::{BTreeMap, BTreeSet}; /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. @@ -162,10 +165,10 @@ pub(crate) fn build_audit_command_plan<'py>( ) -> PyResult> { let py = project.py(); let materializations = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; materializations.set_item( - model_reads::name_object(&model)?, + name_object(&model)?, native_field(&materialization_of(&model)?, "value")?, )?; } @@ -203,9 +206,15 @@ fn called_function_keys<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let functions: Bound<'py, PyDict> = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + .try_iter()? + { let function = function?; - functions.set_item(function_reads::key(&function)?, function)?; + functions.set_item( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + function, + )?; } let called: Bound<'py, PySet> = PySet::empty(py)?; let mut pending: Vec> = direct; @@ -217,7 +226,7 @@ fn called_function_keys<'py>( let Some(function) = functions.get_item(&key)? else { continue; }; - for dependency in function_reads::deps(&function)?.try_iter()? { + for dependency in deps(&function)?.try_iter()? { let dependency = dependency?; if functions.contains(&dependency)? { pending.push(dependency); @@ -257,7 +266,7 @@ pub(crate) fn build_test_command_plan<'py>( )?; let (entries, warnings) = (selected.entries, selected.warnings); let mut case_names: BTreeSet = BTreeSet::new(); - for test in project_reads::sql_tests(&project)?.try_iter()? { + for test in sql_tests(&project)?.try_iter()? { let test = test?; let case_name = native_field(&test, "case_name")?; if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { @@ -274,9 +283,16 @@ pub(crate) fn build_test_command_plan<'py>( } let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); - for function in project_reads::functions(&project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + &project, + )? + .try_iter()? + { let function = function?; - if !function_keys.contains(function_reads::key(&function)?)? { + if !function_keys.contains( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )? { continue; } let fingerprint_sql = object(py, build_compiled_function_fingerprint_sql(&function)?)?; @@ -334,14 +350,19 @@ pub(crate) fn build_seed_command_plan<'py>( }; let selected_keys = native_field(scope, "selected_keys")?; let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed = seed?; - if !selected_keys.contains(seed_reads::key(&seed)?)? { + if !selected_keys + .contains(sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?)? + { continue; } let identity = object(py, build_seed_identity(&seed)?)?; let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); - let previous = mapping_get(&existing, seed_reads::name(&seed)?)?; + let previous = mapping_get( + &existing, + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?, + )?; let reason = if previous.is_none() { "FIRST_RUN" } else if native_field(&previous, "version_hash")?.ne(&version_hash)? { @@ -353,17 +374,17 @@ pub(crate) fn build_seed_command_plan<'py>( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), - ("name", seed_reads::name(&seed)?), - ("destination", seed_reads::destination(&seed)?), + ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?), + ("name", sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?), + ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + native_field(&seed_file(&seed)?, "file_path")?, ), ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ("fingerprint_definition", metadata_json.clone()), ("fingerprint_version_hash", version_hash), @@ -409,7 +430,10 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( BTreeMap::new(); for seed in seeds.try_iter()? { let seed = seed?; - let destination = seed_reads::destination(&seed)?; + let destination = + sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( + &seed, + )?; let schema = native_field(&destination, "schema")?; if schema.is_none() { continue; @@ -420,7 +444,10 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( .entry((database.clone().unwrap_or_default(), schema.clone())) .or_insert_with(|| ((database, schema), BTreeSet::new())) .1 - .insert(seed_reads::name(&seed)?.extract()?); + .insert( + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + .extract()?, + ); } let fingerprints = PyDict::new(py); let mut relations_by_target: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index bed57d5494..28ced3eb0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,15 +1,9 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; @@ -19,6 +13,14 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::graph::main::views::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; @@ -60,7 +62,8 @@ fn display_entry<'py>( resolve_model_full_refresh(model, boolean(py, full_refresh)?.is_truthy()?)?, )? .is_truthy()?; - let query_sql = model_reads::query_sql(model)?; + let query_sql = + sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql(model)?; let custom_name = if materialization_type.eq(CUSTOM)? { config(model, "materialized")? } else { @@ -70,8 +73,8 @@ fn display_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model_reads::model_key(model)?), - ("name", model_reads::name_object(model)?), + ("key", model_key(model)?), + ("name", name_object(model)?), ("relative_path", native_field(model, "relative_path")?), ( "action", @@ -94,7 +97,7 @@ fn display_entry<'py>( }, )?, ), - ("destination", model_reads::destination(model)?), + ("destination", destination(model)?), ("fingerprint_query_sql", query_sql.clone()), ("resolved_sql", query_sql), ("logical_ddl", string(py, "")), @@ -141,9 +144,9 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .collect::>>()?, )?; let mut entries: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; - if selected.contains(model_reads::name_object(&model)?)? { + if selected.contains(name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index e81fdf2378..1599f248c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -33,7 +33,6 @@ use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -45,6 +44,7 @@ use sqlbuild_python_base::build_command::main::utc_now::utc_now; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::time::Instant; const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 908a16a460..5551fef3cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,8 +1,5 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_entries::models::{ @@ -16,7 +13,6 @@ use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -33,14 +29,10 @@ use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::models::{ - CursorInputLookups, ModelEntryDecisions, -}; use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; use crate::bindings::_helpers::planner_strategy::values::text; -use crate::bindings::_helpers::semantic_shapes::source_bindings::source_binding_diagnostics; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -51,6 +43,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::microbatch_guidance; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; +use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics::source_binding_diagnostics; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; @@ -211,11 +209,8 @@ fn plan_model_from_change<'py>( } else { cursor_bounds }; - let warnings = with_cursor_safety_warnings( - &entry_warnings.into_any(), - &model_reads::name_object(model)?, - &bounds, - )?; + let warnings = + with_cursor_safety_warnings(&entry_warnings.into_any(), &name_object(model)?, &bounds)?; Ok((entry, warnings)) } @@ -428,20 +423,22 @@ pub(crate) fn plan_entries<'py>( runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); - for function in project_reads::functions(&project)?.try_iter()? { + for function in functions(&project)?.try_iter()? { let function = function?; - functions_by_name.set_item(function_reads::name(&function)?, function)?; + functions_by_name.set_item( + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?, + function, + )?; } let settings = native_field(&project, "settings")?; - let scope_data = crate::bindings::_helpers::planner_scope::retained::scope_data(&scope)?; + let scope_data = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&scope)?; let context = Bound::new( py, ModelPlanContext { model_locations: relations_context.model_locations.clone_ref(py), - models_by_name: - crate::bindings::_helpers::native_host::model_lookup::ModelLookup::Native( - scope_data.clone(), - ), + models_by_name: sqlbuild_python_project::native_host::models::ModelLookup::Native( + scope_data.clone(), + ), functions_by_name: functions_by_name.into_any().unbind(), seed_locations: relations_context.seed_locations.clone_ref(py), function_locations: relations_context.function_locations.clone_ref(py), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index ac269cd208..e29ce386a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -1,8 +1,8 @@ //! Typed requests of plan entry construction. -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; +use sqlbuild_python_project::native_host::models::ModelLookup; /// What resolving relation locations and source metadata for one scope reads. #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 6ffe15d500..dddf2165ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,23 +1,13 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_scope::loaders::upstream_loader_dependency_names; -use crate::bindings::_helpers::planner_scope::sources::build_source_load_entries; use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; -use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; -use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -28,6 +18,25 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::compiled_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::compiled_views::main::function_reads::fingerprint_destination::fingerprint_destination; +use sqlbuild_python_project::compiled_views::main::function_reads::language::language; +use sqlbuild_python_project::compiled_views::main::function_reads::packages::packages; +use sqlbuild_python_project::compiled_views::main::function_reads::relative_path::relative_path; +use sqlbuild_python_project::compiled_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::compiled_views::main::function_reads::returns::returns; +use sqlbuild_python_project::compiled_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::compiled_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::build_source_load_entries::build_source_load_entries; +use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; +use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; +use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; +use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; const LOADER_CONSUMER: &str = "loader"; const HOOK_CONSUMER: &str = "hook"; @@ -108,27 +117,31 @@ fn seed_entries<'py>( let metadata_jsons = native_field(extras, "seed_metadata_jsons")?; let plan_reasons = native_field(extras, "seed_plan_reasons")?; let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)? + .try_iter()? + { let seed = seed?; - if !selected_keys.contains(seed_reads::key(&seed)?)? { + if !selected_keys + .contains(sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?)? + { continue; } - let name = seed_reads::name(&seed)?; + let name = sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?; entries.push(planner_model( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), + ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?), ("name", name.clone()), - ("destination", seed_reads::destination(&seed)?), + ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + native_field(&seed_file(&seed)?, "file_path")?, ), ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ( "fingerprint_definition", @@ -376,30 +389,30 @@ pub(super) fn function_entry<'py>( py, RecordId::FunctionPlanEntry, &[ - ("key", function_reads::key(function)?), - ("name", function_reads::name(function)?), - ("relative_path", function_reads::relative_path(function)?), - ("destination", function_reads::destination(function)?), - ("arguments", function_reads::arguments(function)?), - ("returns", function_reads::returns(function)?), + ("key", sqlbuild_python_project::compiled_views::main::function_reads::key::key(function)?), + ("name", sqlbuild_python_project::compiled_views::main::function_reads::name::name(function)?), + ("relative_path", relative_path(function)?), + ("destination", sqlbuild_python_project::compiled_views::main::function_reads::destination::destination(function)?), + ("arguments", arguments(function)?), + ("returns", returns(function)?), ("body_sql", body_sql), ("fingerprint_query_sql", fingerprint_query_sql), ( "fingerprint_destination", - function_reads::fingerprint_destination(function)?, + fingerprint_destination(function)?, ), - ("return_columns", function_reads::return_columns(function)?), - ("language", function_reads::language(function)?), + ("return_columns", return_columns(function)?), + ("language", language(function)?), ( "source_file_path", - function_reads::source_file_path(function)?, + source_file_path(function)?, ), ( "runtime_version", - function_reads::runtime_version(function)?, + runtime_version(function)?, ), - ("entry_point", function_reads::entry_point(function)?), - ("packages", function_reads::packages(function)?), + ("entry_point", entry_point(function)?), + ("packages", packages(function)?), ("previous_query_sql", previous_query_sql), ("reason", reason), ], @@ -416,12 +429,18 @@ fn function_entries<'py>( let functions = native_field(changes, "functions")?; let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let mut entries: Vec> = Vec::new(); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + .try_iter()? + { let function = function?; - if !selected_keys.contains(function_reads::key(&function)?)? { + if !selected_keys.contains( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )? { continue; } - let name = function_reads::name(&function)?; + let name = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?; let change = functions.get_item(&name)?; let previous_query_sql = if previous.contains(&name)? { native_field(&previous.get_item(&name)?, "definition")? @@ -476,7 +495,7 @@ pub(super) fn selected_test_entries<'py>( ) -> PyResult> { let py = project.py(); let mut selected: Vec> = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test = test?; if !test_selected(&test, selected_keys, selection)? { continue; @@ -778,6 +797,6 @@ fn build_selected_test_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_model_chain_names_by_key, module)?)?; + module.add_function(sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function(module)?)?; module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 10a6299a67..af455f52a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,20 +1,13 @@ //! Relation locations and source metadata for plan entry construction. -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; use crate::bindings::_helpers::planner_entries::values::planner_model; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::sources::source_load_map; -use crate::bindings::_helpers::planner_scope::values::keys; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; @@ -29,6 +22,16 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; +use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::keys::keys; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::main::source_load_map::source_load_map; use std::collections::HashSet; /// Relation keys needed to render selected audits without selecting those relations. @@ -37,10 +40,10 @@ fn selected_audit_relation_keys( selected_keys: &HashSet, ) -> PyResult> { let mut keys: HashSet = HashSet::new(); - for audit in project_reads::audits(project)?.try_iter()? { + for audit in audits(project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, selected_keys)? { - for dependency in audit_reads::scope_deps(&audit)?.try_iter()? { + for dependency in scope_deps(&audit)?.try_iter()? { keys.insert(native_key(&dependency?)?); } } @@ -103,15 +106,9 @@ pub(crate) fn planner_relations_context<'py>( deferral }; let selected_keys = native_field(&scope, "selected_keys")?; - let mut model_locations = object( - py, - build_model_locations(&project_reads::models(&project)?)?, - )?; - let mut seed_locations = object(py, build_seed_locations(&project_reads::seeds(&project)?)?)?; - let function_locations = object( - py, - build_function_locations(&project_reads::functions(&project)?)?, - )?; + let mut model_locations = object(py, build_model_locations(&models(&project)?)?)?; + let mut seed_locations = object(py, build_seed_locations(&seeds(&project)?)?)?; + let function_locations = object(py, build_function_locations(&functions(&project)?)?)?; let deferred_locations = native_field(&deferral, "deferred_locations")?; if !deferred_locations.is_none() { let applied = object( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index b0dddacebb..f0fa1f9685 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -15,14 +15,9 @@ use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, Ident use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::planner_identity::hashing::{baselined_model_query_hash, graph_key}; use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -31,6 +26,12 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; fn key_of(kind: &str, name: &str) -> GraphKey { GraphKey { @@ -121,14 +122,24 @@ pub(crate) fn direct_model_version_identities<'py>( let mut graph = IdentityGraph::default(); for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; - let name: String = function_reads::name(&function)?.extract()?; + let name: String = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + .extract()?; let Some(function_hash) = function_hashes.get_item(&name)? else { continue; }; graph.nodes.insert( - graph_key(&function_reads::key(&function)?)?, + graph_key( + &sqlbuild_python_project::compiled_views::main::function_reads::key::key( + &function, + )?, + )?, IdentityNode { - upstream_keys: graph_keys(&function_reads::deps(&function)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::compiled_views::main::function_reads::deps::deps( + &function, + )?, + )?, local_hash: Some(function_hash.extract()?), }, ); @@ -139,7 +150,8 @@ pub(crate) fn direct_model_version_identities<'py>( for seed in seeds.try_iter()? { let seed: Bound<'py, PyAny> = seed?; let (seed_hash, metadata_json): (String, String) = seed_identity(&seed)?; - let name: Bound<'py, PyAny> = seed_reads::name(&seed)?; + let name: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?; seed_hashes.set_item(&name, &seed_hash)?; seed_metadata.set_item(&name, metadata_json)?; } @@ -176,7 +188,7 @@ pub(crate) fn direct_model_version_identities<'py>( let Some(model) = scope.model(py, &order_key.name) else { continue; }; - let name: String = model_reads::model_name(&model)?; + let name: String = model_name(&model)?; let metadata_json: String = model_version_identity_metadata_json( &model, Some(function_hashes.as_any()), @@ -186,10 +198,7 @@ pub(crate) fn direct_model_version_identities<'py>( let query_fingerprint: String = baselined_model_query_hash( py, &name, - ( - &model_reads::query_sql(&model)?.extract::()?, - dialect.as_deref(), - ), + (&query_sql(&model)?.extract::()?, dialect.as_deref()), &recorded_model, )?; let local_hash: String = model_local_identity_hash(&query_fingerprint, &metadata_json); @@ -198,7 +207,11 @@ pub(crate) fn direct_model_version_identities<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name), IdentityNode { - upstream_keys: graph_keys(&model_reads::deps(&model)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::compiled_views::main::model_reads::deps::deps( + &model, + )?, + )?, local_hash: Some(local_hash), }, ); @@ -296,11 +309,15 @@ pub(crate) fn with_honest_model_write_hashes<'py>( let local_hashes: Bound<'py, PyAny> = native_field(&identities, "model_local_hashes")?; for model in scope.models.values() { let model = model.bind(py); - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { - upstream_keys: graph_keys(&model_reads::deps(&model)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::compiled_views::main::model_reads::deps::deps( + &model, + )?, + )?, local_hash: mapping_get(&local_hashes, &name)?.extract()?, }, ); @@ -339,7 +356,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( let Some(model) = scope.model(py, &order_key.name) else { continue; }; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; if mapping_get(&local_hashes, &name)?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 9508c1d437..bf77c9e628 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -16,13 +16,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::microbatches::conversion::timestamp; -use crate::bindings::_helpers::planner_identity::hashing::{ - baselined_function_definition_hash, function_definition_hash, -}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -35,6 +29,20 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::compiled_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::compiled_views::main::function_reads::packages::packages; +use sqlbuild_python_project::compiled_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::compiled_views::main::function_reads::returns::returns; +use sqlbuild_python_project::compiled_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::compiled_views::main::model_reads::config::config; +use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::seed_reads::external::external; +use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; +use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -102,7 +110,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let py: Python<'py> = model.py(); let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in model_reads::deps(model)?.try_iter()? { + for upstream_key in deps(model)?.try_iter()? { let upstream_key: Bound<'py, PyAny> = upstream_key?; let kind: String = native_field(&upstream_key, "resource_type")? .str()? @@ -117,11 +125,11 @@ pub(crate) fn model_version_identity_metadata_json<'py>( } } } - let config_values: Bound<'py, PyAny> = native_field(&model_reads::config(model)?, "values")?; + let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( - &model_reads::name_object(model)?, + &name_object(model)?, &config_values, ( Some(local_function_hashes.into_any()), @@ -182,7 +190,10 @@ fn contract_output_signature<'py>( config_values: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = model.py(); - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; if schema_entry.is_none() { return Ok(None); } @@ -196,7 +207,10 @@ fn contract_output_signature<'py>( if !enforced && native_field(&schema_entry, "model_schema")?.is_none() { return Ok(None); } - let enum_columns: Bound<'py, PyAny> = model_reads::enum_columns(model)?; + let enum_columns: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::enum_columns::enum_columns( + model, + )?; let rendered: Bound<'py, PyList> = PyList::empty(py); for column in columns.try_iter()? { let column: Bound<'py, PyAny> = column?; @@ -259,7 +273,10 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; let payload: Bound<'_, PyDict> = PyDict::new(py); payload.set_item( "contract", @@ -299,7 +316,12 @@ fn recorded_function_unchanged( return Ok(false); } let py: Python<'_> = function.py(); - let language: String = function_reads::language(function)?.str()?.extract()?; + let language: String = + sqlbuild_python_project::compiled_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?; let fingerprint_sql: String = compiled_function_fingerprint_sql(function)?; let current: String = function_definition_hash(py, name, (&fingerprint_sql, &language, dialect))?; @@ -352,36 +374,36 @@ pub(crate) fn function_local_hashes<'py>( let mut recorded_local: Option> = None; for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; - let name: String = function_reads::name(&function)?.extract()?; - let language: Bound<'py, PyAny> = - native_field(&function_reads::language(&function)?, "value")?; + let name: String = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + .extract()?; + let language: Bound<'py, PyAny> = native_field( + &sqlbuild_python_project::compiled_views::main::function_reads::language::language( + &function, + )?, + "value", + )?; let body_hash: String = function_definition_hash( py, &name, ( - &function_reads::body_sql(&function)?.extract::()?, + &body_sql(&function)?.extract::()?, &language.extract::()?, dialect, ), )?; let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item( - "arguments", - name_type_pairs(&function_reads::arguments(&function)?)?, - )?; - payload.set_item("returns", function_reads::returns(&function)?)?; + payload.set_item("arguments", name_type_pairs(&arguments(&function)?)?)?; + payload.set_item("returns", returns(&function)?)?; payload.set_item( "return_columns", - name_type_pairs(&function_reads::return_columns(&function)?)?, + name_type_pairs(&return_columns(&function)?)?, )?; payload.set_item("body_sql", body_hash)?; payload.set_item("language", language)?; - payload.set_item( - "runtime_version", - function_reads::runtime_version(&function)?, - )?; - payload.set_item("entry_point", function_reads::entry_point(&function)?)?; - payload.set_item("packages", function_reads::packages(&function)?)?; + payload.set_item("runtime_version", runtime_version(&function)?)?; + payload.set_item("entry_point", entry_point(&function)?)?; + payload.set_item("packages", packages(&function)?)?; let mut local_hash: String = stable_identity_hash(&sorted_dumps(payload.as_any(), SPACED, true)?); if recorded_function_unchanged(&function, (&name, dialect), recorded)? { @@ -441,39 +463,50 @@ fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { /// Stable function definition text used for change fingerprints. pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { Ok(function_fingerprint_sql(&FunctionFingerprintFields { - language: function_reads::language(function)?.str()?.extract()?, - arguments: rendered_pairs(&function_reads::arguments(function)?)?, - returns: function_reads::returns(function)?.str()?.extract()?, - return_columns: rendered_pairs(&function_reads::return_columns(function)?)?, - runtime_version: text_of(function_reads::runtime_version(function))?, - entry_point: text_of(function_reads::entry_point(function))?, - packages: function_reads::packages(function)? + language: + sqlbuild_python_project::compiled_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?, + arguments: rendered_pairs(&arguments(function)?)?, + returns: returns(function)?.str()?.extract()?, + return_columns: rendered_pairs(&return_columns(function)?)?, + runtime_version: text_of(runtime_version(function))?, + entry_point: text_of(entry_point(function))?, + packages: packages(function)? .try_iter()? .map(|package| package?.extract::()) .collect::>()?, - body_sql: function_reads::body_sql(function)?.extract()?, + body_sql: body_sql(function)?.extract()?, })) } /// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { let py: Python<'_> = seed.py(); - let schema_entry: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry( + seed, + )?; let csv_settings: Bound<'_, PyAny> = native_field(&schema_entry, "csv_settings")?; let columns: Bound<'_, PyList> = PyList::empty(py); for column in native_field(&schema_entry, "columns")?.try_iter()? { columns.append(column_signature(&column?)?)?; } - let rows: Bound<'_, PyAny> = if seed_reads::external(seed)?.is_truthy()? { + let rows: Bound<'_, PyAny> = if external(seed)?.is_truthy()? { PyList::empty(py).into_any() } else { seed_rows( - &native_field(&seed_reads::seed_file(seed)?, "file_path")?, + &native_field(&seed_file(seed)?, "file_path")?, &csv_settings, )? }; let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("name", seed_reads::name(seed)?)?; + payload.set_item( + "name", + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?, + )?; payload.set_item( "csv_settings", imported(py, DATACLASSES_MODULE, "asdict")?.call1((&csv_settings,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs index 2b7ae821ed..f8f6364a35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -1,5 +1,3 @@ -pub(crate) mod graph; -pub(crate) mod hashing; pub(crate) mod identities; pub(crate) mod metadata; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs index f9df4e384b..cce0b3f494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -2,13 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_identity::{ - graph, hashing, identities, metadata, staleness, -}; +use crate::bindings::_helpers::planner_identity::{identities, metadata, staleness}; +use sqlbuild_python_project::planner_identity::main::register_graph::register_graph; +use sqlbuild_python_project::planner_identity::main::register_hashing::register_hashing; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - graph::register(module)?; - hashing::register(module)?; + register_graph(module)?; + register_hashing(module)?; identities::register(module)?; metadata::register(module)?; staleness::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index e2b5244dc2..bbdd4ec732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -18,13 +18,13 @@ use sqlbuild_analysis::planner_identity::models::{ }; use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; -use crate::bindings::_helpers::planner_identity::hashing::graph_key; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; /// Change kinds that mean the model's own identity changed. const OWN_CHANGE_KINDS: [&str; 5] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index ee87e56253..f4ea999b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,6 +1,5 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, }; @@ -8,11 +7,14 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnCandidate, ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(super) fn candidates<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), @@ -36,7 +38,7 @@ pub(super) fn candidates<'py>( if !materialized.eq("incremental")? && !materialized.eq("snapshot")? { continue; } - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; if columns.full_refresh_model_names.contains(&name)? || !native_field(&snapshot, "existing_relations")?.contains(&name)? { @@ -45,7 +47,7 @@ pub(super) fn candidates<'py>( let warehouse = native_field(&snapshot, "existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let declared = PyDict::new(py); - let schema = model_reads::schema_entry(&model)?; + let schema = schema_entry(&model)?; if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { let column = column?; @@ -82,7 +84,7 @@ pub(super) fn candidates<'py>( }; let current = parse_query_shape( py, - &model_reads::query_sql(&model)?.extract::()?, + &query_sql(&model)?.extract::()?, dialect.extract::>()?.as_deref(), )?; result.push(ColumnCandidate { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 6e6385e0df..b950a5095c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -2,15 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::inferred_rows; use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; use crate::bindings::_helpers::planner_migrations::columns::identical_renames; use crate::bindings::_helpers::planner_migrations::columns::rename_hints; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; @@ -23,6 +20,9 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_column::newest_column; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn decide<'py>( runtime: &Bound<'py, PyAny>, @@ -69,8 +69,11 @@ pub(super) fn decide<'py>( outputs.push(name.to_lowercase()); } } - let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)? - .call1((model_reads::destination(&candidate.model)?,))?; + let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)?.call1(( + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &candidate.model, + )?, + ))?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; @@ -107,8 +110,8 @@ pub(super) fn decide<'py>( py, RecordId::ColumnMigrationPlanEntry, &[ - ("model_name", model_reads::name_object(&candidate.model)?), - ("destination", model_reads::destination(&candidate.model)?), + ("model_name", name_object(&candidate.model)?), + ("destination", sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&candidate.model)?), ( "origin_column", if let Some(origin) = origin { @@ -162,7 +165,10 @@ pub(super) fn checked<'py>( warnings: Vec::new(), }); } - let destination = model_reads::destination(&candidate.model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &candidate.model, + )?; let qualified = native_field(&destination, "qualified_name")?; let relation = if physical.is_truthy()? { physical.clone() @@ -243,7 +249,7 @@ pub(super) fn hints<'py>( Some(keywords( py, &[ - ("model_name", model_reads::name_object(&candidate.model)?), + ("model_name", name_object(&candidate.model)?), ("previous", candidate.previous.clone()), ("current", candidate.current.clone()), ("excluded", excluded.call_method1("union", (claimed,))?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 5c552ee897..8939aadbb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,8 +1,5 @@ //! Boundary conversions for the column migration planner. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -11,6 +8,10 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::planner_identity::main::model_definition_hash::model_definition_hash; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() @@ -61,8 +62,8 @@ pub(super) fn definition_hash<'py>( model.py(), model_definition_hash( model.py(), - &model_reads::name_object(model)?.extract::()?, - &model_reads::query_sql(model)?.extract::()?, + &name_object(model)?.extract::()?, + &query_sql(model)?.extract::()?, native_field(snapshot, "column_dialect")? .extract::>()? .as_deref(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 4a89487ac9..c069479ec7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,7 +1,5 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; @@ -10,8 +8,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -19,6 +15,12 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(super) fn overlay<'py>( snapshot: &Bound<'py, PyAny>, @@ -36,7 +38,7 @@ pub(super) fn overlay<'py>( let fingerprints = imported(py, "builtins", "dict")?.call1((native_field(&stored, "models")?,))?; for candidate in candidates { - let name = model_reads::name_object(&candidate.model)?; + let name = name_object(&candidate.model)?; let renames = PyDict::new(py); let pending = PyDict::new(py); let mut blocked = false; @@ -96,7 +98,7 @@ pub(super) fn overlay<'py>( replace( &candidate.fingerprint, &[ - ("definition", model_reads::query_sql(&candidate.model)?), + ("definition", query_sql(&candidate.model)?), ( "definition_hash", definition_hash(&candidate.model, snapshot)?, @@ -160,7 +162,7 @@ pub(super) fn cursors<'py>( physical } else { let qualified = native_field( - &model_reads::destination(&models.get(py, &name.extract::()?)?)?, + &destination(&models.get(py, &name.extract::()?)?)?, "qualified_name", )?; if qualified.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 2f7c689c54..80fed3706d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -1,6 +1,5 @@ //! Host column migration planning while Python owns warehouse inspection. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use crate::bindings::_helpers::planner_migrations::models::{ @@ -14,6 +13,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; /// Decide column renames and return the existing planning result model. #[pyfunction(name = "plan_column_migrations")] @@ -81,7 +82,7 @@ pub(crate) fn plan_column_migrations<'py>( let state = super::state_inspection::inspection(&runtime, None)?; let schemas = PySet::empty(py)?; for candidate in &candidates { - let schema = native_field(&model_reads::destination(&candidate.model)?, "schema")?; + let schema = native_field(&destination(&candidate.model)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -108,10 +109,7 @@ pub(crate) fn plan_column_migrations<'py>( &native_field(&state, "column_events")?, &inputs, )?; - let physical = mapping_get( - &columns.physical_relations, - model_reads::name_object(&candidate.model)?, - )?; + let physical = mapping_get(&columns.physical_relations, name_object(&candidate.model)?)?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { hints.push(hint?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index ef732eccbf..46c66a6a9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -1,7 +1,5 @@ //! Convert existing planner contracts for origin compatibility checks. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::inferred_rows; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_migrations::models::{ CompatibilityCheck, MigrationCompatibilityResult, @@ -20,6 +18,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { config_value(model.py(), &model_values(model)?, key)?.extract() @@ -92,12 +92,9 @@ pub(super) fn check_migration_compatibility<'py>( ); } let relations = PyDict::new(py); - relations.set_item(model_reads::name_object(&model)?, relation)?; + relations.set_item(name_object(&model)?, relation)?; let columns = PyDict::new(py); - columns.set_item( - model_reads::name_object(&model)?, - check.origin_columns.clone(), - )?; + columns.set_item(name_object(&model)?, check.origin_columns.clone())?; let snapshot = new_record_with( py, RecordId::WarehouseSnapshot, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index fb395b498c..fd05522d0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,6 +1,5 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::matching::{ match_migration_fingerprints, migration_destination_fingerprints, }; @@ -14,6 +13,7 @@ use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; use std::collections::BTreeSet; /// Manual and automatically discovered migration requests with their destination fingerprints. @@ -57,7 +57,7 @@ pub(super) fn discover_model_migrations<'py>( let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); for item in &models { - names.set_item(model_reads::name_object(item)?, item)?; + names.set_item(name_object(item)?, item)?; } let event_renames = discovery_events::renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { @@ -75,7 +75,7 @@ pub(super) fn discover_model_migrations<'py>( } let unbuilt = PyDict::new(py); for item in &models { - let name = model_reads::name_object(item)?; + let name = name_object(item)?; if !declared_names.contains(&name)? && call_with( &native_field(&state, "named_fingerprint")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 1b93b679da..4db9bddcab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,6 +10,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; @@ -111,11 +111,14 @@ pub(super) fn resumed<'py>( let target = native_field(&native_field(runtime, "project")?, "effective_target_name")?; let events = native_field(state, "events")?; for item in models { - let name = model_reads::name_object(item)?; + let name = name_object(item)?; if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { continue; } - let destination = model_reads::destination(item)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + item, + )?; let event = newest(&events, &destination)?; if event.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 4eaac7b808..6f8938dff5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -2,14 +2,9 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -17,18 +12,26 @@ use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_orde use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? + for item in sqlbuild_python_project::compiled_views::main::project_reads::models::models( + &runtime_field(runtime, |runtime| &runtime.project)?, + )? + .try_iter()? { let item = item?; - let name: String = model_reads::name_object(&item)?.extract()?; + let name: String = name_object(&item)?.extract()?; let mut upstream: Vec = Vec::new(); - for dependency in model_reads::deps(&item)?.try_iter()? { + for dependency in deps(&item)?.try_iter()? { let dependency = dependency?; if native_field(&dependency, "resource_type")?.eq("model")? { upstream.push(native_field(&dependency, "name")?.extract()?); @@ -54,8 +57,8 @@ pub(super) fn triggered( let relevant = reachable_keys(&selected, &scope.upstream); let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?; for item in models { - let name = model_reads::name_object(item)?; - if relevant.contains(&native_key(&model_reads::model_key(item)?)?) + let name = name_object(item)?; + if relevant.contains(&native_key(&model_key(item)?)?) && !fingerprints.contains(&name)? && !declared.contains(&name)? { @@ -74,7 +77,9 @@ pub(super) fn metadata<'py>( let functions = object( py, build_function_local_hashes( - &project_reads::functions(&project)?, + &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + &project, + )?, native_field(&project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -93,7 +98,7 @@ pub(super) fn metadata<'py>( let result = PyDict::new(py); for item in models { result.set_item( - model_reads::name_object(item)?, + name_object(item)?, object( py, build_model_version_identity_metadata_json( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index a4138280ad..dfbe2bf286 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -1,7 +1,5 @@ //! Live candidate inspection and automatic request materialization. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ @@ -10,7 +8,6 @@ use crate::bindings::_helpers::planner_migrations::discovery_events::{ use crate::bindings::_helpers::planner_migrations::models::{ ModelMigrationDeclaration, ModelMigrationRequest, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, @@ -22,6 +19,10 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; fn location<'py>( @@ -118,13 +119,10 @@ pub(super) fn automatic<'py>( let scope = scope_data(scope)?; let mut requests: Vec> = Vec::new(); for item in models { - if !scope - .selected - .contains(&native_key(&model_reads::model_key(item)?)?) - { + if !scope.selected.contains(&native_key(&model_key(item)?)?) { continue; } - let name = model_reads::name_object(item)?; + let name = name_object(item)?; let event = matches.1.get_item(&name)?; let candidate = mapping_get(matches.0, &name)?; let origin = if let Some(event) = event { @@ -170,11 +168,8 @@ pub(super) fn warnings<'py>( let scope = scope_data(scope)?; let selected = PySet::empty(py)?; for item in models { - if scope - .selected - .contains(&native_key(&model_reads::model_key(item)?)?) - { - selected.add(model_reads::name_object(item)?)?; + if scope.selected.contains(&native_key(&model_key(item)?)?) { + selected.add(name_object(item)?)?; } } let ambiguous: BTreeMap = ambiguous.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 7a7b416f7d..9dc771ef71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -1,7 +1,5 @@ //! Migration identity binding with unchanged metadata and hash format. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -10,6 +8,8 @@ use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migrati use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::native_host::models::ModelLookup; use std::collections::BTreeMap; /// Hash normalized SQL and non-storage metadata through rename identities. @@ -103,7 +103,7 @@ pub(crate) fn with_migration_fingerprints<'py>( let fingerprint = call_with( &native_field(cache, "fingerprint")?, &[ - ("query_sql", model_reads::query_sql(&model)?), + ("query_sql", query_sql(&model)?), ("metadata_json", metadata), ("ref_identities", PyDict::new(py).into_any()), ("dialect", dialect.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index ac808dabce..45c2568b3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,6 +1,5 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::models::{ FingerprintInputs, ModelMigrationRequest, }; @@ -12,6 +11,9 @@ use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; use std::collections::{BTreeMap, BTreeSet}; fn variants<'py>( @@ -19,7 +21,7 @@ fn variants<'py>( model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { - let name = model_reads::name_object(model)?; + let name = name_object(model)?; let metadata = inputs.metadata_jsons.get_item(name)?; let cache = native_field(&inputs.fingerprints, "fingerprint")?; let mut hashes: Vec = Vec::new(); @@ -27,7 +29,7 @@ fn variants<'py>( let hash: Option = call_with( &cache, &[ - ("query_sql", model_reads::query_sql(model)?), + ("query_sql", query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), ("dialect", inputs.dialect.clone()), @@ -54,7 +56,7 @@ pub(super) fn match_migration_fingerprints<'py>( let mut by_name: BTreeMap> = BTreeMap::new(); let mut names: Vec = Vec::new(); for model in models { - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; if new_names.contains(&name) { names.push(name.clone()); } @@ -93,7 +95,7 @@ pub(super) fn migration_destination_fingerprints<'py>( let result = PyDict::new(py); for request in requests { let model = &request.model; - let name = model_reads::name_object(model)?; + let name = name_object(model)?; if mapping_get(&inputs.metadata_jsons, &name)?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index ccafab948a..23cbf5d42a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; use crate::bindings::_helpers::planner_migrations::model_handover::{ @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -76,7 +76,10 @@ pub(super) fn decide<'py>( let py = runtime.py(); let request = decision_request.request; let item = request.model.clone(); - let destination = model_reads::destination(&item)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &item, + )?; let origin = decision_request.origin.clone(); let snapshot = decision_request.snapshot.clone(); let state = decision_request.state.clone(); @@ -102,7 +105,7 @@ pub(super) fn decide<'py>( py, RecordId::ModelMigrationPlanEntry, &[ - ("model_name", model_reads::name_object(&item)?), + ("model_name", name_object(&item)?), ("discovery", request.discovery.clone()), ( "decision", @@ -163,7 +166,7 @@ pub(super) fn decide<'py>( } let inherited = if recorded && !applies(&item, &newest, &snapshot)? { native_field(&native_field(&snapshot, "fingerprints")?, "models")? - .call_method1("get", (model_reads::name_object(&item)?,))? + .call_method1("get", (name_object(&item)?,))? } else { empty_version(&item, &fingerprint)? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index fdaea4ab1e..7ad3c7c271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -1,15 +1,16 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::event_rows::presumed_utc::presumed_utc; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn applies( model: &Bound<'_, PyAny>, @@ -17,7 +18,7 @@ pub(super) fn applies( snapshot: &Bound<'_, PyAny>, ) -> PyResult { let own = native_field(&native_field(snapshot, "fingerprints")?, "models")? - .call_method1("get", (model_reads::name_object(model)?,))?; + .call_method1("get", (name_object(model)?,))?; Ok(own.is_none() || presumed_utc(&native_field(&own, "ts")?)? < presumed_utc(&native_field(event, "created_at")?)?) @@ -45,18 +46,21 @@ pub(super) fn handover<'py>( if let Ok(payload) = payload.cast::() && payload.contains(key)? { - payload.set_item(key, model_reads::name_object(model)?)?; + payload.set_item(key, name_object(model)?)?; let kwargs = PyDict::new(py); kwargs.set_item("sort_keys", true)?; kwargs.set_item("separators", (",", ":"))?; kwargs.set_item("default", imported(py, "builtins", "str")?)?; metadata = native_field(&json, "dumps")?.call((payload,), Some(&kwargs))?; } - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?; replace( fingerprint, &[ - ("node_name", model_reads::name_object(model)?), + ("node_name", name_object(model)?), ("target_database", native_field(&destination, "database")?), ("target_schema", native_field(&destination, "schema")?), ("target_name", native_field(&destination, "name")?), @@ -88,14 +92,14 @@ pub(super) fn equivalent<'py>( replace( &handover, &[ - ("definition", model_reads::query_sql(model)?), + ("definition", query_sql(model)?), ( "definition_hash", object( py, compute_query_hash( py, - &model_reads::query_sql(model)?.extract::()?, + &query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), )?, )?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 12d630e1e7..73adf1da2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -1,7 +1,5 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; @@ -12,9 +10,7 @@ use crate::bindings::_helpers::planner_migrations::models::{ use crate::bindings::_helpers::planner_migrations::planning_origin::{ require_same_database, resolve_origin, }; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -29,6 +25,10 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const MIGRATE_FROM_KEY: &str = "migrate_from"; const MIGRATE_FORCE_KEY: &str = "migrate_force"; @@ -102,9 +102,7 @@ fn discover<'py>( let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in - project_reads::models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? - { + for item in models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? { if let Some(request) = declared(&item?)? { declared_requests.push(request); } @@ -119,7 +117,7 @@ fn discover<'py>( for request in &declared_requests { let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; declarations.push(ModelMigrationDeclaration { - model_name: model_reads::name_object(&request.model)?, + model_name: name_object(&request.model)?, origin_location, origin_model, }); @@ -185,7 +183,7 @@ fn finish<'py>( let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); - let name = model_reads::name_object(&item)?; + let name = name_object(&item)?; let decided = super::model_entries::decide(&MigrationDecisionRequest { runtime: runtime.clone(), request, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index d2bd1aaf94..3dbd2b5546 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,6 +1,5 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; @@ -9,7 +8,6 @@ use crate::bindings::_helpers::planner_migrations::old_name_objects::{ }; use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; @@ -21,6 +19,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; impl<'py> OldNamePlanner<'py> { pub(super) fn new( @@ -117,7 +118,7 @@ impl<'py> OldNamePlanner<'py> { &self, item: &Bound<'py, PyAny>, ) -> PyResult>> { - let Some(history) = self.newest(&relation(&model_reads::destination(item)?)?, None)? else { + let Some(history) = self.newest(&relation(&destination(item)?)?, None)? else { return Ok(None); }; if !native_field(&native_field(&history, "move")?, "destination_model")? @@ -327,7 +328,7 @@ impl<'py> OldNamePlanner<'py> { py, RecordId::OldNameViewPlanEntry, &[ - ("model_name", model_reads::name_object(item)?), + ("model_name", name_object(item)?), ( "origin", location(&self.runtime, &native_field(history, "old")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 13face79d2..a08ea9ba27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -12,6 +11,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -44,7 +44,10 @@ pub(super) fn retention<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { old_name_retention( - &native_field(&model_reads::config(item)?, "values")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::config::config(item)?, + "values", + )?, project, ) } @@ -72,7 +75,7 @@ pub(super) fn recorded_view<'py>( let created = native_field(history, "created")?; let mut reused = py.None().into_bound(py); for item in models { - if relation(&model_reads::destination(item)?)? + if relation(&destination(item)?)? .call_method1("matches", (native_field(history, "old")?,))? .is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 6136b66601..de43d54488 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -2,15 +2,11 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -18,6 +14,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::HashMap; fn schemas<'py>( @@ -29,7 +30,7 @@ fn schemas<'py>( let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { - let schema = native_field(&model_reads::destination(item)?, "schema")?; + let schema = native_field(&destination(item)?, "schema")?; if !schema.is_none() { needed.add(schema)?; } @@ -48,7 +49,7 @@ fn schemas<'py>( if !include_project { for item in selected { let existing = native_field(&snapshot, "existing_relations")? - .call_method1("get", (model_reads::name_object(item)?,))?; + .call_method1("get", (name_object(item)?,))?; if !existing.is_none() && host_entry(py, AdapterHost::NormalizeRelationType)? .call1((native_field(&existing, "relation_type")?,))? @@ -80,12 +81,13 @@ pub(crate) fn plan_old_name_views<'py>( let runtime = phase.runtime.clone(); let scope = scope_data(&phase.scope)?; let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let models: Vec> = project_reads::models(&project)? - .try_iter()? - .collect::>()?; + let models: Vec> = + sqlbuild_python_project::compiled_views::main::project_reads::models::models(&project)? + .try_iter()? + .collect::>()?; let mut by_name: HashMap> = HashMap::new(); for item in &models { - by_name.insert(model_reads::model_name(item)?, item.clone()); + by_name.insert(model_name(item)?, item.clone()); } let mut selected: Vec> = Vec::new(); for key in &scope.order { @@ -140,7 +142,7 @@ pub(crate) fn plan_old_name_views<'py>( } } for item in &selected { - if !covered.contains(model_reads::name_object(item)?)? + if !covered.contains(name_object(item)?)? && let Some(planned) = planner.pending_for(item)? { entries.push(planned); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index 6f46d3a0d4..4a6a085759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -3,10 +3,6 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::references::explicit_references::hard_coded_relation_remedy; -use crate::bindings::_helpers::references::explicit_references::old_name_reference_message; -use crate::bindings::_helpers::references::relations::match_project_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -15,6 +11,10 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::references::main::hard_coded_relation_remedy::hard_coded_relation_remedy; +use sqlbuild_python_project::references::main::match_project_relation::match_project_relation; +use sqlbuild_python_project::references::main::old_name_reference_message::old_name_reference_message; fn old_relation<'py>( old: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 11c0f75d6f..d2f3f40209 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -17,12 +16,14 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry; pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { - let schema = model_reads::schema_entry(model)?; + let schema = schema_entry(model)?; let mut columns: Vec<(String, Option)> = Vec::new(); if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { @@ -88,7 +89,7 @@ pub(crate) fn old_name_claim_errors<'py>( let mut errors: Vec> = Vec::new(); for model in arguments.value("selected")?.try_iter()? { let model = model?; - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; let existing = arguments .value("snapshot")? .getattr("existing_relations")? @@ -101,7 +102,11 @@ pub(crate) fn old_name_claim_errors<'py>( if !kind.eq("view")? { continue; } - let destination = relation(&model_reads::destination(&model)?)?; + let destination = relation( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?, + )?; for view in arguments.value("views")?.try_iter()? { let view = view?; if native_field(&view, "destination_model")?.eq(&name)? @@ -155,11 +160,15 @@ pub(crate) fn old_name_view_skip_reason<'py>( let mut reuser: Option = None; for model in arguments.value("models")?.try_iter()? { let model = model?; - if relation(&model_reads::destination(&model)?)? - .call_method1("matches", (&old,))? - .is_truthy()? + if relation( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?, + )? + .call_method1("matches", (&old,))? + .is_truthy()? { - reuser = Some(model_reads::name_object(&model)?.extract()?); + reuser = Some(name_object(&model)?.extract()?); break; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index f562594ae0..0960fe2804 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -1,7 +1,5 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -14,6 +12,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, @@ -34,11 +34,9 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? - { + for item in models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - if model_reads::name_object(&item)?.eq(name)? { + if name_object(&item)?.eq(name)? { return Ok((native_field(&item, "destination")?, name.clone())); } } @@ -73,8 +71,13 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; - for item in project_reads::models(&project)?.try_iter()? { - let database = native_field(&model_reads::destination(&item?)?, "database")?; + for item in models(&project)?.try_iter()? { + let database = native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &item?, + )?, + "database", + )?; if !database.is_none() { return Ok(database); } @@ -85,10 +88,13 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? - { - let schema = native_field(&model_reads::destination(&item?)?, "schema")?; + for item in models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { + let schema = native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &item?, + )?, + "schema", + )?; if !schema.is_none() { schemas.add(schema)?; } @@ -142,7 +148,10 @@ pub(super) fn resolve_origin<'py>( _ => super::column_objects::text(py, ""), }; let item = request.model.clone(); - let destination = model_reads::destination(&item)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &item, + )?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { parts.push( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index f47da108cf..e7f376f2ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -1,7 +1,7 @@ //! Read-only warehouse evidence gathered once for all model migrations in one plan. -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index f0d4ba8962..3ff4f9fe2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,22 +1,23 @@ //! Selected-scope buildability validation before change planning. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_identity::graph::execution_origins; use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::planner_error; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::seed_reads::external::external; +use sqlbuild_python_project::compiled_views::main::seed_reads::key::key; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(crate) fn check_selected_scope_buildability<'py>( runtime: &Bound<'py, PyAny>, @@ -29,10 +30,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let scope = scope_data(&selected_scope)?; let mut satisfied_keys: std::collections::HashSet = std::collections::HashSet::new(); - for seed in project_reads::seeds(&project)?.try_iter()? { + for seed in seeds(&project)?.try_iter()? { let seed = seed?; - if seed_reads::external(&seed)?.is_truthy()? { - satisfied_keys.insert(GraphKey::from(&native_key(&seed_reads::key(&seed)?)?)); + if external(&seed)?.is_truthy()? { + satisfied_keys.insert(GraphKey::from(&native_key(&key(&seed)?)?)); } } let snapshot = warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs index f665c6b7b8..c009111ea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -7,7 +7,7 @@ use pyo3::{PyClass, pyclass::boolean_struct::True}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerScopeResolution, }; -use crate::bindings::_helpers::planner_scope::retained::project_scope; +use sqlbuild_python_project::planner_scope::main::project_scope::project_scope; #[pymethods] impl PlannerScopeResolution { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index cacbbee94f..fbdb9de575 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,9 +1,6 @@ //! Per-model full-refresh resolution and full-rebuild refusals. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyBool; @@ -12,6 +9,12 @@ use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::strings::strings; const FULL_REFRESH_KEY: &str = "full_refresh"; @@ -37,10 +40,10 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( cli_full_refresh: bool, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(model_reads::name_object(&model)?); + names.push(name_object(&model)?); } } frozenset(project.py(), &names) @@ -79,7 +82,7 @@ pub(crate) fn check_full_rebuild_allowed( { return Ok(()); } - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; Err(planner_error( py, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs index 948f3e9440..ca87274851 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs @@ -6,9 +6,9 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerRuntime, PlannerScopeResolution, PlannerWarehouseState, }; -use crate::bindings::_helpers::planner_scope::retained::RetainedScope; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::RetainedScope; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 6d78148ab2..37c74cb8b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,14 +1,11 @@ //! Expected version identities and stale-warning change detection. -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::native; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; @@ -20,6 +17,10 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeMap, HashSet}; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); @@ -53,10 +54,10 @@ fn version_identities<'py>( ) -> PyResult> { identity_work(project.py())?.identity_builds += 1; direct_model_version_identities( - (&project_reads::functions(project)?, &scope), + (&functions(project)?, &scope), native_field(project, "sql_analysis_dialect")?.extract()?, ( - &project_reads::seeds(project)?, + &seeds(project)?, &project.py().None().into_bound(project.py()), &native_field(project, "hook_functions")?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index 0fd4cce629..c952ce457a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -9,7 +9,6 @@ pub(crate) mod full_refresh; pub(crate) mod getters; pub(crate) mod identities; pub(crate) mod models; -pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; pub(crate) mod retention; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 7755dfe65b..26493cd00c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,6 +1,5 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; @@ -12,14 +11,9 @@ use crate::bindings::_helpers::planner_phases::models::{ PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, }; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - native, replace, sorted_names, strings, text, tuple, -}; use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; @@ -30,6 +24,15 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::strings::strings; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -285,7 +288,7 @@ pub(crate) fn add_plan_warnings<'py>( /// The unselected models the selected scope reads, sorted. pub(crate) fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let selected = crate::bindings::_helpers::planner_scope::retained::scope_data(&selected)?; + let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; let mut names: Vec = Vec::new(); for (key, deps) in &selected.upstream { if !selected.selected.contains(key) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index fb4fc0679b..5a57fe9c6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -4,38 +4,44 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; + module.add_class::()?; + module + .add_class::()?; + module + .add_class::()?; + module + .add_class::()?; module.add_function(wrap_pyfunction!( - super::scopes::resolve_planner_scopes, + crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes, module )?)?; module.add_function(wrap_pyfunction!( - super::identities::planner_identity_work_info, + crate::bindings::_helpers::planner_phases::identities::planner_identity_work_info, module )?)?; module.add_function(wrap_pyfunction!( - super::identities::detect_stale_warning_changes_of, + crate::bindings::_helpers::planner_phases::identities::detect_stale_warning_changes_of, module )?)?; module.add_function(wrap_pyfunction!( - super::full_refresh::describe_full_rebuild_cause, + crate::bindings::_helpers::planner_phases::full_refresh::describe_full_rebuild_cause, module )?)?; module.add_function(wrap_pyfunction!( - super::full_refresh::check_full_rebuild_allowed, + crate::bindings::_helpers::planner_phases::full_refresh::check_full_rebuild_allowed, module )?)?; module.add_function(wrap_pyfunction!( - super::retention::table_type_copy_name, + crate::bindings::_helpers::planner_phases::retention::table_type_copy_name, module )?)?; module.add_function(wrap_pyfunction!( - super::retention::plan_table_types, + crate::bindings::_helpers::planner_phases::retention::plan_table_types, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::planner_phases::retention::plan_retention, module )?)?; - module.add_function(wrap_pyfunction!(super::retention::plan_retention, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 3cf8f14a69..e7b19fb1b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -2,19 +2,12 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, TRANSIENT_TABLE_TYPE, }; -use crate::bindings::_helpers::planner_phases::objects::{ - construct, member, native, native_member, optional_text, planner_error, planner_model, replace, - sorted_names, text, tuple, -}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -31,6 +24,20 @@ use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::construct::construct; +use sqlbuild_python_project::planner_phases::main::member::member; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::native_member::native_member; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; @@ -62,13 +69,15 @@ fn selected_models<'py>( ) -> PyResult>> { let scope = scope_data(scope)?; let mut models: Vec<_> = Vec::new(); - for model in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? + for model in sqlbuild_python_project::compiled_views::main::project_reads::models::models( + &runtime_field(runtime, |runtime| &runtime.project)?, + )? + .try_iter()? { let model = model?; if scope .selected - .contains(&("model".to_owned(), model_reads::model_name(&model)?)) + .contains(&("model".to_owned(), model_name(&model)?)) { models.push(model); } @@ -94,12 +103,15 @@ pub(crate) fn plan_table_types<'py>( let mut declared: Vec<_> = Vec::new(); for model in &models { if native_field( - &native_field(&model_reads::config(model)?, "table_type")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + "table_type", + )?, "declared", )? .is_truthy()? { - declared.push(model_reads::model_name(model)?); + declared.push(model_name(model)?); } } if !declared.is_empty() && !snowflake { @@ -136,7 +148,7 @@ pub(crate) fn plan_table_types<'py>( if !backed.is_truthy()? { continue; } - let relation = mapping_get(&relations, model_reads::name_object(&model)?)?; + let relation = mapping_get(&relations, name_object(&model)?)?; if relation.is_none() { continue; } @@ -152,7 +164,10 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = model_reads::destination(&model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?; let limit: i64 = adapter .call_method0("maximum_identifier_length")? .extract()?; @@ -161,7 +176,7 @@ pub(crate) fn plan_table_types<'py>( py, RecordId::TableTypePlanEntry, &[ - ("model_name", model_reads::name_object(&model)?), + ("model_name", name_object(&model)?), ("destination", destination), ("copy_name", text(py, ©_name(&target_name, limit))), ("desired_type", native_field(&native_field(&table_type, "value")?, "value")?), @@ -326,7 +341,7 @@ fn relation_retention<'py>( model: &Bound<'py, PyAny>, ) -> PyResult>> { let py = runtime.py(); - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let config = native_field(model, "config")?; let retention = native_field(&config, "time_travel_retention")?; if native_field(&retention, "desired_days")?.is_none() @@ -339,7 +354,10 @@ fn relation_retention<'py>( )); } let desired_days: i64 = native_field(&retention, "desired_days")?.extract()?; - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?; let schema = native_field(&destination, "schema")?; if schema.is_none() { return Err(planner_error( @@ -352,7 +370,7 @@ fn relation_retention<'py>( py, AdapterHost::RetentionRequest, &[ - ("request_id", model_reads::name_object(model)?), + ("request_id", name_object(model)?), ( "scope", member(py, AdapterHost::RetentionScope, "relation")?, @@ -365,14 +383,14 @@ fn relation_retention<'py>( )?; let mut base = RetentionEntry { request: request.clone(), - model_names: tuple(py, vec![model_reads::name_object(model)?])?, + model_names: tuple(py, vec![name_object(model)?])?, state: None, source: native_field(&native_field(&retention, "source")?, "value")?, }; let snowflake = adapter_is(runtime, SNOWFLAKE_ADAPTER)?; let table_type = native_field(&native_field(&config, "table_type")?, "value")?; let relations = native_field(&native_field(warehouse, "snapshot")?, "existing_relations")?; - if !relations.contains(model_reads::name_object(model)?)? { + if !relations.contains(name_object(model)?)? { if snowflake && desired_days > 1 && table_type.eq(TRANSIENT_TABLE_TYPE)? { return Err(transient_error(py, &name, desired_days, "set ")); } @@ -383,7 +401,7 @@ fn relation_retention<'py>( flatten(&changes)?, )?]); } - let relation = relations.get_item(model_reads::name_object(model)?)?; + let relation = relations.get_item(name_object(model)?)?; let relation_transient = native_field(&relation, "is_transient")?; let lookup = call_with( &host_entry(py, AdapterHost::RelationLookup)?.getattr("key")?, @@ -477,7 +495,10 @@ fn bigquery_retention<'py>( { let model = model?; let retention = native_field(&native_field(&model, "config")?, "time_travel_retention")?; - let destination = model_reads::destination(&model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?; if native_field(&retention, "unmanaged")?.is_truthy()? || native_field(&retention, "desired_days")?.is_none() || native_field(&destination, "schema")?.is_none() @@ -494,7 +515,10 @@ fn bigquery_retention<'py>( } let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); for model in models { - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?; let database: Option = native_field(&destination, "database")?.extract()?; let schema: String = native_field(&destination, "schema")?.str()?.extract()?; scopes.insert((database.clone().unwrap_or_default(), schema, database)); @@ -599,7 +623,10 @@ pub(crate) fn plan_retention<'py>( let py = runtime.py(); let mut models: Vec<_> = Vec::new(); for model in selected_models(runtime, scope)? { - let retention = native_field(&model_reads::config(&model)?, "time_travel_retention")?; + let retention = native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::config::config(&model)?, + "time_travel_retention", + )?; if !native_field(&retention, "unmanaged")?.is_truthy()? && !native_field(&retention, "desired_days")?.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 9aec02133f..6e72967165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,14 +1,14 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; use pyo3::types::PyTuple; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index d229588e9f..ab52663c8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,6 +1,5 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::graph::views::compiled_graph_keys; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; @@ -13,15 +12,16 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; -use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::views::compiled_graph_keys::compiled_graph_keys; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 30246d89b9..0a9324e570 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -2,12 +2,9 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ @@ -34,6 +31,9 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py: Python<'py> = model.py(); @@ -61,11 +61,14 @@ pub(crate) fn unbounded(py: Python<'_>) -> PyResult> { fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let relation = native_field(snapshot, "existing_relations")? - .call_method1("get", (model_reads::name_object(model)?,))?; + .call_method1("get", (name_object(model)?,))?; if relation.is_none() { return Ok(false); } - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?; let key = call_with( &host_entry(model.py(), AdapterHost::RelationLookup)?.getattr("key")?, &[ @@ -81,7 +84,9 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> let target = native_field(snapshot, "target_relation")?; Ok(!target.is_none() && target.ne(native_field( - &model_reads::destination(model)?, + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?, "qualified_name", )?)?) } @@ -124,8 +129,8 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(bounded); } let snapshot = &request.snapshot; - let mut cursor_snapshot = native_field(snapshot, "cursor_snapshots")? - .call_method1("get", (model_reads::name_object(&model)?,))?; + let mut cursor_snapshot = + native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(&model)?,))?; let roles = resolve_cursor_input_roles(&model)?; let producers = class_field(context, |c: &ModelPlanContext| { &c.runtime_cursor_producer_names @@ -152,7 +157,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let refusal = object( py, empty_input_rebuild_refusal( - &model_reads::name_object(&model)?.extract::()?, + &name_object(&model)?.extract::()?, empty.extract::>()?, cursor_type.extract::>()?.as_deref(), ), @@ -178,7 +183,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( }; return Err(PyErr::from_value(unreadable_cursor_inputs_error( py, - &model_reads::name_object(&model)?.extract::()?, + &name_object(&model)?.extract::()?, cursor_type.extract::>()?.as_deref(), names.extract::>()?, )?)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 428d39f17b..46dc2c47a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -1,6 +1,5 @@ //! Selected-model inspection and untyped override conversion. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_resolution::values::{method, string}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; @@ -66,7 +66,7 @@ pub(crate) fn type_cursor_overrides<'py>( } for kind in [INTEGER_CURSOR, TIMESTAMP_CURSOR] { if config_text_is(py, &values, CURSOR_TYPE_KEY, kind)? { - models.push((model_reads::name_object(model)?.extract()?, kind.to_owned())); + models.push((name_object(model)?.extract()?, kind.to_owned())); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index ae4eb4b540..a1372a0ce0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -3,11 +3,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ @@ -46,6 +44,8 @@ use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; const INCREMENTAL: &str = "incremental"; @@ -362,8 +362,7 @@ fn model_cursor_snapshot<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { - native_field(snapshot, "cursor_snapshots")? - .call_method1("get", (model_reads::name_object(model)?,)) + native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(model)?,)) } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { @@ -556,7 +555,7 @@ pub(crate) fn model_cursor_overrides<'py>( } } let session = CursorSession::new(py)?; - let name = model_reads::name_object(model)?.str()?; + let name = name_object(model)?.str()?; let cursor_start = config_bound(model, "cursor_start")?; let cursor_end = config_bound(model, "cursor_end")?; if !resolved_start.is_none() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs index 6610d71335..ca79e40fb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -1,18 +1,16 @@ //! Location maps retaining the original compiled Python destination objects. -use crate::bindings::_helpers::compiled_views::resource_reads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::resource_reads::name::name; fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let result = PyDict::new(items.py()); for item in items.try_iter()? { let item = item?; - result.set_item( - resource_reads::name(&item)?, - resource_reads::destination(&item)?, - )?; + result.set_item(name(&item)?, destination(&item)?)?; } Ok(result) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 1e4e4779fc..faa31a2276 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -1,6 +1,5 @@ //! Maximum automatic-start recovery and evidence boundary. -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ optional_scalar, required_scalar, scalars, @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 30858a454f..bda4cad401 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,8 +1,5 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::{ @@ -35,6 +32,10 @@ use sqlbuild_python_front::model_config::main::config::config_text_is::config_te use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; const APPEND_STRATEGY: &str = "append"; @@ -216,12 +217,9 @@ pub(crate) fn resolve_model_sql<'py>( cursor_type: cursor_type.clone(), lower_bound_inclusive: append_cursor_inclusive(model)?, }; - let mut sql = resolution.resolve( - &model_reads::query_sql(model)?.extract::()?, - &filtering, - )?; + let mut sql = resolution.resolve(&query_sql(model)?.extract::()?, &filtering)?; let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; if uses_intrinsics { let cause = match &request.full_rebuild_cause { Some(cause) => Some(cause.clone()), @@ -331,11 +329,10 @@ pub(crate) fn resolve_function_sql<'py>( function_locations: args.value("function_locations")?, }; let function = args.value("function")?; - let name: String = function_reads::name(&function)?.extract()?; - let sql: String = resolution.unbounded( - function_reads::body_sql(&function)?, - &format!("Function '{name}'"), - )?; + let name: String = + sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + .extract()?; + let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; Ok(PyString::new(py, &sql).into_any()) } @@ -358,7 +355,7 @@ pub(crate) fn model_hook_entries<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let values = model_values(model)?; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let mut hooks: Vec> = Vec::new(); for key in ["pre_hooks", "post_hooks"] { hooks.push(resolution.hook( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 8e3a4d36af..10bf24d5ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::planner_resolution::values::{ cursor_wrap, filtering_of, method, string, }; use crate::bindings::_helpers::planner_strategy::values::input_error; -use crate::bindings::_helpers::references::source_relations::render_source_read; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -17,6 +16,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; +use sqlbuild_python_project::references::main::render_source_read::render_source_read; use sqlbuild_sqltext::resolution::main::column_name_key::column_name_key; use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index a0ab6bfff6..d941f2657e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_RELATION_COLLISION, }; use crate::bindings::_helpers::planner_scenario::values::{error, string}; -use crate::bindings::_helpers::relation_naming::fitting::fitted_logical_name; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; @@ -14,6 +13,7 @@ use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_v use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::relation_naming::main::fitted_logical_name::fitted_logical_name; const SCENARIO_ARTIFACT_LABEL: &str = "Scenario artifact"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index f07449c2cd..f8779a0d4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,8 +1,5 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; @@ -22,6 +19,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -142,14 +141,17 @@ fn namespace<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let models = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } for field in ["target_model_names", "model_names"] { for name in native_field(graph, field)?.try_iter()? { if let Some(model) = models.get_item(name?)? { - let destination = model_reads::destination(&model)?; + let destination = sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?; return Ok(( native_field(&destination, "database")?, native_field(&destination, "schema")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 5375ac0efa..2a8feb2211 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -1,7 +1,5 @@ //! Existing seed and function plan dataclass construction. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; @@ -17,6 +15,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::seed_reads::key::key; +use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; pub(super) fn seed_entries<'py>( project: &Bound<'py, PyAny>, @@ -24,7 +25,10 @@ pub(super) fn seed_entries<'py>( relations: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let seeds = by_attribute(&project_reads::seeds(project)?, "name")?; + let seeds = by_attribute( + &sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, + "name", + )?; let fixtures = native_field(graph, "seed_fixture_names")?; let mut result: Vec> = Vec::new(); for name in native_field(graph, "seed_names")?.try_iter()? { @@ -52,17 +56,17 @@ pub(super) fn seed_entries<'py>( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), - ("name", seed_reads::name(&seed)?), - ("destination", destination), + ("key", key(&seed)?), ( - "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + "name", + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?, ), + ("destination", destination), + ("file_path", native_field(&seed_file(&seed)?, "file_path")?), ("columns", extract_seed_columns(&seed)?.into_any()), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 2e69862e57..20b15a47c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,13 +1,7 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::records::compiled_model_record; -use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_scenario::constants::{ CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, @@ -22,11 +16,6 @@ use crate::bindings::_helpers::planner_scenario::{ sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, values::{error, string}, }; -use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::sql_test_glue::constants::{ - DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, -}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; @@ -45,6 +34,20 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::records::compiled_model_record::compiled_model_record; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_strategy::main::complete_fixtures::complete_fixtures as complete_relation_fixtures; +use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_END; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_START; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -99,9 +102,22 @@ pub(crate) fn build_scenario_execution_plan<'py>( } else { analysis.is_truthy()? }; - let models = by_attribute(&project_reads::models(&project)?, "name")?; - let functions = by_attribute(&project_reads::functions(&project)?, "key")?; - let locations = build_function_locations(&project_reads::functions(&project)?)?.into_any(); + let models = by_attribute( + &sqlbuild_python_project::compiled_views::main::project_reads::models::models(&project)?, + "name", + )?; + let functions = by_attribute( + &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + &project, + )?, + "key", + )?; + let locations = build_function_locations( + &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + &project, + )?, + )? + .into_any(); let upstream = execution_deps(&project)?; let ordered = execution_order(py, &upstream, &[])?; let ordered = KeyCache::new(py)?.keys(&ordered)?; @@ -137,13 +153,11 @@ pub(crate) fn build_scenario_execution_plan<'py>( let destination = required_target( &context.relations, "model_locations", - &model_reads::model_name(&model)?, + &model_name(&model)?, "model", )?; - let query = resolve_model_dbt_refs( - &model_reads::query_sql(&model)?.extract::()?, - &context.relations, - )?; + let query = + resolve_model_dbt_refs(&query_sql(&model)?.extract::()?, &context.relations)?; let cursor_window = scenario_cursor_window(&model)?; let record = compiled_model_record(&model)?; let changes = PyDict::new(py); @@ -274,7 +288,7 @@ fn model_context<'py>( ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), models_by_name: models.clone().into_any().into(), - functions_by_name: by_attribute(&project_reads::functions(project)?, "name")? + functions_by_name: by_attribute(&sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)?, "name")? .into_any() .unbind(), seed_locations: native_field(&context.relations, "seed_locations")?.unbind(), @@ -311,7 +325,7 @@ fn compared_columns( ) -> PyResult>> { let py = scenario.py(); let model_columns: Option> = match model { - Some(model) => model_reads::inferred_names(&model)?, + Some(model) => inferred_names(&model)?, None => None, }; let dialect: Option = @@ -402,7 +416,7 @@ fn expectations<'py>( /// The SQL-test wide window for a non-microbatch model whose SQL reads the cursor intrinsics. fn scenario_cursor_window<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py = model.py(); - let query: String = model_reads::query_sql(model)?.extract()?; + let query: String = query_sql(model)?.extract()?; let values = model_values(model)?; let microbatch: bool = config_value(py, &values, INCREMENTAL_MODE_KEY)?.eq(MICROBATCH_INCREMENTAL_MODE)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 02b2e7d4e5..da872f0498 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,10 +1,5 @@ //! Convert project views into pure scenario dependency facts. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; @@ -15,6 +10,12 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; #[pyfunction] #[pyo3(signature = (*, scenario, project, sql_analysis_enabled=true, sql_analysis_dialect=None))] @@ -37,25 +38,25 @@ pub(crate) fn plan_scenario_graph<'py>( dbt_refs: native_field(scenario, "dbt_ref_fixture_names")?.extract()?, ..ScenarioGraphInput::default() }; - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; let deps = native_field(&model, "deps")? .try_iter()? .map(|dep| native_key(&dep?)) .collect::>()?; - input - .models - .insert(model_reads::name_object(&model)?.extract()?, deps); + input.models.insert(name_object(&model)?.extract()?, deps); } - for source in project_reads::sources(project)?.try_iter()? { - input - .project_sources - .insert(source_reads::name(&source?)?.extract()?); + for source in sources(project)?.try_iter()? { + input.project_sources.insert( + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source?)? + .extract()?, + ); } - for seed in project_reads::seeds(project)?.try_iter()? { - input - .project_seeds - .insert(seed_reads::name(&seed?)?.extract()?); + for seed in seeds(project)?.try_iter()? { + input.project_seeds.insert( + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed?)? + .extract()?, + ); } let output = graph(&input); input.targets.sort(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index fe958d78cd..a62aa5995d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -1,7 +1,5 @@ //! Build scenario relation locations through adapter qualification callbacks. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, @@ -16,6 +14,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; #[pyfunction(name = "build_scenario_relation_plan")] #[pyo3(signature = (**kwargs))] @@ -66,11 +65,15 @@ pub(crate) fn build_scenario_relation_plan<'py>( )?; } let sources = PyDict::new(py); - for source in project_reads::sources(&args.project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project_reads::sources::sources( + &args.project, + )? + .try_iter()? + { let source = source?; sources.set_item( - source_reads::name(&source)?, - source_reads::source_entry(&source)?, + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + source_entry(&source)?, )?; } let models = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index b10b6a00fe..323632ffd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -1,6 +1,5 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, }; @@ -8,7 +7,6 @@ use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, values::{error, string}, }; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; use sqlbuild_analysis::planner_scenario::{ @@ -18,6 +16,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(super) fn resolve_sql( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs index ad41a18f00..63ff52ed86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -1,8 +1,3 @@ //! Planner scope Python boundary models. -pub(crate) mod loaders; pub(crate) mod registration; -pub(crate) mod retained; -pub(crate) mod scope; -pub(crate) mod sources; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs index 6ba143a726..7508b63a1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs @@ -1,25 +1,16 @@ //! Register planner scope and loader entry points. -use crate::bindings::_helpers::planner_scope::{loaders, sources}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map_function::build_intermediate_source_map_function; +use sqlbuild_python_project::planner_scope::main::build_source_load_entries_function::build_source_load_entries_function; +use sqlbuild_python_project::planner_scope::main::build_source_load_map_function::build_source_load_map_function; +use sqlbuild_python_project::planner_scope::main::expand_selected_loader_dependencies_function::expand_selected_loader_dependencies_function; +use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names_function::upstream_loader_dependency_names_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - loaders::upstream_loader_dependency_names, - module - )?)?; - module.add_function(wrap_pyfunction!( - loaders::build_intermediate_source_map, - module - )?)?; - module.add_function(wrap_pyfunction!( - loaders::expand_selected_loader_dependencies, - module - )?)?; - module.add_function(wrap_pyfunction!(sources::build_source_load_map, module)?)?; - module.add_function(wrap_pyfunction!( - sources::build_source_load_entries, - module - )?) + module.add_function(upstream_loader_dependency_names_function(module)?)?; + module.add_function(build_intermediate_source_map_function(module)?)?; + module.add_function(expand_selected_loader_dependencies_function(module)?)?; + module.add_function(build_source_load_map_function(module)?)?; + module.add_function(build_source_load_entries_function(module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs deleted file mode 100644 index 737e6c0694..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Native planner scope ownership, with projection only at public record boundaries. - -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyclass; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; - -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_scope::values::{edges, keys}; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::models::NativeRecord; -use sqlbuild_python_base::records::types::RecordId; - -/// The graph and selection used throughout planning; model values remain compiled-resource views. -#[derive(Debug)] -pub(crate) struct ScopeData { - pub(crate) upstream: Vec<(ObjectKey, Vec)>, - pub(crate) downstream: Vec<(ObjectKey, Vec)>, - pub(crate) names: Vec<(String, ObjectKey)>, - pub(crate) models: HashMap, - pub(crate) model_order: Vec, - pub(crate) selected: HashSet, - pub(crate) order: Vec, - pub(crate) user_selected: HashSet, - pub(crate) python_reads: HashSet, - pub(crate) tests: Py, - pub(crate) load_sources: bool, - pub(crate) end_audit_sources: HashSet, -} - -/// Internal carrier, never returned in place of a supported public PlannerScope record. -#[pyclass(frozen)] -pub(crate) struct RetainedScope { - pub(crate) data: Arc, - projection: PyOnceLock>, -} - -impl ScopeData { - pub(crate) fn object(self, py: Python<'_>) -> PyResult> { - Ok(Bound::new( - py, - RetainedScope { - data: Arc::new(self), - projection: PyOnceLock::new(), - }, - )? - .into_any()) - } - - pub(crate) fn model<'py>(&self, py: Python<'py>, name: &str) -> Option> { - self.models.get(name).map(|model| model.bind(py)) - } - - pub(crate) fn with_selected(&self, py: Python<'_>, selected: HashSet) -> Self { - Self { - upstream: self.upstream.clone(), - downstream: self.downstream.clone(), - names: self.names.clone(), - model_order: self.model_order.clone(), - models: self - .models - .iter() - .map(|(name, model)| (name.clone(), model.clone_ref(py))) - .collect(), - selected, - order: self.order.clone(), - user_selected: self.user_selected.clone(), - python_reads: self.python_reads.clone(), - tests: self.tests.clone_ref(py), - load_sources: self.load_sources, - end_audit_sources: self.end_audit_sources.clone(), - } - } - - /// Public field projection; internal graph consumers read the native fields above instead. - pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let cache = KeyCache::new(py)?; - let key_set = |keys: &HashSet| -> PyResult> { - Ok( - PyFrozenSet::new(py, cache.keys(&keys.iter().cloned().collect::>())?)? - .into_any(), - ) - }; - match name { - "upstream_deps" => Ok(edge_dict(&cache, &self.upstream)?.into_any()), - "downstream_deps" => Ok(edge_dict(&cache, &self.downstream)?.into_any()), - "selected_keys" => key_set(&self.selected), - "user_selected_keys" => key_set(&self.user_selected), - "end_audit_source_keys" => key_set(&self.end_audit_sources), - "execution_order" => Ok(PyTuple::new(py, cache.keys(&self.order)?)?.into_any()), - "all_keys" => { - let values = PyDict::new(py); - for (name, key) in &self.names { - values.set_item(name, cache.key(key)?)?; - } - Ok(values.into_any()) - } - "models_by_name" => { - let values = PyDict::new(py); - for name in &self.model_order { - if let Some(model) = self.models.get(name) { - values.set_item(name, model.bind(py))?; - } - } - Ok(values.into_any()) - } - "python_read_source_names" => Ok(PyFrozenSet::new(py, &self.python_reads)?.into_any()), - "sql_test_selection" => Ok(self.tests.bind(py).clone().into_any()), - "load_sources" => self.load_sources.into_bound_py_any(py), - _ => Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), - } - } -} - -/// Read native ownership directly, or import a caller-authored public scope at the API edge. -pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(scope) = value.cast::() { - return Ok(scope.get().data.clone()); - } - let mut models: HashMap = HashMap::new(); - let mut model_order: Vec = Vec::new(); - for item in mapping_items(&field(value, "models_by_name")?)?.try_iter()? { - let item = item?; - let name: String = item.get_item(0)?.extract()?; - model_order.push(name.clone()); - models.insert(name, ModelHandle::read(&item.get_item(1)?)); - } - Ok(Arc::new(ScopeData { - upstream: edges(&field(value, "upstream_deps")?)?, - downstream: edges(&field(value, "downstream_deps")?)?, - names: mapping_items(&field(value, "all_keys")?)? - .try_iter()? - .map(|item| { - let item = item?; - Ok(( - item.get_item(0)?.extract()?, - crate::bindings::_helpers::graph::keys::native_key(&item.get_item(1)?)?, - )) - }) - .collect::>()?, - models, - model_order, - selected: keys(&field(value, "selected_keys")?)?.into_iter().collect(), - order: keys(&field(value, "execution_order")?)?, - user_selected: keys(&field(value, "user_selected_keys")?)? - .into_iter() - .collect(), - python_reads: field(value, "python_read_source_names")?.extract()?, - tests: field(value, "sql_test_selection")?.extract()?, - load_sources: field(value, "load_sources")?.is_truthy()?, - end_audit_sources: keys(&field(value, "end_audit_source_keys")?)? - .into_iter() - .collect(), - })) -} - -/// Preserve public record identity and dataclass behavior on explicit projection. -pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let Ok(scope) = value.cast::() else { - return Ok(value.clone()); - }; - scope - .get() - .projection - .get_or_try_init(value.py(), || { - let names = [ - "upstream_deps", - "downstream_deps", - "all_keys", - "models_by_name", - "selected_keys", - "execution_order", - "user_selected_keys", - "python_read_source_names", - "sql_test_selection", - "load_sources", - "end_audit_source_keys", - ]; - let fields = names - .iter() - .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) - .collect::>>()?; - Ok::<_, pyo3::PyErr>( - new_record_with(value.py(), RecordId::PlannerScope, &fields)? - .cast_into::() - .map(Bound::unbind)?, - ) - }) - .map(|value| value.bind(scope.py()).clone().into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs index 0097ebb362..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs @@ -1,5 +1 @@ -pub(crate) mod carrier; -pub(crate) mod model_selectors; pub(crate) mod registration; -pub(crate) mod selectors; -pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs index 46c95951b4..c8ba321fdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs @@ -2,9 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_selection::model_selectors; +use sqlbuild_python_project::planner_selection::main::register_model_selectors::register_model_selectors; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_selectors::register(module)?; + register_model_selectors(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 3e331b8bbf..10ff313842 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -5,14 +5,6 @@ use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; use std::collections::HashSet; -use crate::bindings::_helpers::auditing::render::render_audit_sql; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::audits::{ attachment, outside_target_lineage, resolve_effective_run_scope, }; @@ -23,17 +15,41 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; +use sqlbuild_python_project::auditing::main::render_audit_sql::render_audit_sql; +use sqlbuild_python_project::compiled_views::main::audit_reads::always_run::always_run; +use sqlbuild_python_project::compiled_views::main::audit_reads::attached_column_name::attached_column_name; +use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::compiled_views::main::audit_reads::definition_name::definition_name; +use sqlbuild_python_project::compiled_views::main::audit_reads::description::description; +use sqlbuild_python_project::compiled_views::main::audit_reads::evaluation_mode::evaluation_mode; +use sqlbuild_python_project::compiled_views::main::audit_reads::evidence_limit::evidence_limit; +use sqlbuild_python_project::compiled_views::main::audit_reads::evidence_sql::evidence_sql; +use sqlbuild_python_project::compiled_views::main::audit_reads::measurement_contract::measurement_contract; +use sqlbuild_python_project::compiled_views::main::audit_reads::meta::meta; +use sqlbuild_python_project::compiled_views::main::audit_reads::minimum_samples::minimum_samples; +use sqlbuild_python_project::compiled_views::main::audit_reads::run_scope::run_scope; +use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::compiled_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::compiled_views::main::audit_reads::thresholds::thresholds; +use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::edges::edges; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; -use crate::bindings::_helpers::planner_strategy::constants::{ - AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, -}; -use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_strategy::constants::AUDIT_TYPES_MODULE; +use sqlbuild_python_project::planner_strategy::constants::COMPILE_TYPES_MODULE; +use sqlbuild_python_project::planner_strategy::constants::MODEL_ATTACHMENT; +use sqlbuild_python_project::planner_strategy::constants::SOURCE_ATTACHMENT; +use sqlbuild_python_project::planner_strategy::models::AuditPlanning; /// One attribute read of a compiled audit. type AuditRead<'py> = fn(&Bound<'py, PyAny>) -> PyResult>; @@ -113,13 +129,15 @@ fn plan_audit<'py>( ) -> PyResult> { let py = audit.py(); let audit = audit.clone(); - let name: String = audit_reads::name(&audit)?.extract()?; + let name: String = + sqlbuild_python_project::compiled_views::main::audit_reads::name::name(&audit)? + .extract()?; let resolved = render_query( request, - audit_reads::sql_body(&audit)?, + sql_body(&audit)?, &format!("audit '{name}' planned SQL"), )?; - let evidence = audit_reads::evidence_sql(&audit)?; + let evidence = evidence_sql(&audit)?; let evidence_resolved = if evidence.is_none() { py.None().into_bound(py) } else { @@ -142,33 +160,43 @@ fn plan_audit<'py>( } else { py.None().into_bound(py) }; - let requested = audit_enum(audit_reads::run_scope(&audit)?, "AuditRunScope", "final")?; - let severity = audit_enum(audit_reads::severity(&audit)?, "AuditSeverity", "warn")?; + let requested = audit_enum(run_scope(&audit)?, "AuditRunScope", "final")?; + let severity = audit_enum( + sqlbuild_python_project::compiled_views::main::audit_reads::severity::severity(&audit)?, + "AuditSeverity", + "warn", + )?; let effective = resolve_effective_run_scope(&requested, &materialization)?; - let mut target_kind = audit_reads::attached_target_kind(&audit)?; + let mut target_kind = attached_target_kind(&audit)?; if target_kind.is_none() && (kind_text == MODEL_ATTACHMENT || kind_text == SOURCE_ATTACHMENT) { target_kind = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind_text,))?; } let values = PyDict::new(py); let reads: [(&str, AuditRead<'py>); 12] = [ - ("key", audit_reads::key), - ("name", audit_reads::name), - ("definition_name", audit_reads::definition_name), - ("evaluation_mode", audit_reads::evaluation_mode), - ("thresholds", audit_reads::thresholds), - ("minimum_samples", audit_reads::minimum_samples), - ("evidence_limit", audit_reads::evidence_limit), - ("scope_deps", audit_reads::scope_deps), - ("attached_column_name", audit_reads::attached_column_name), - ("always_run", audit_reads::always_run), - ("description", audit_reads::description), - ("meta", audit_reads::meta), + ( + "key", + sqlbuild_python_project::compiled_views::main::audit_reads::key::key, + ), + ( + "name", + sqlbuild_python_project::compiled_views::main::audit_reads::name::name, + ), + ("definition_name", definition_name), + ("evaluation_mode", evaluation_mode), + ("thresholds", thresholds), + ("minimum_samples", minimum_samples), + ("evidence_limit", evidence_limit), + ("scope_deps", scope_deps), + ("attached_column_name", attached_column_name), + ("always_run", always_run), + ("description", description), + ("meta", meta), ]; for (field, read) in reads { values.set_item(field, read(&audit)?)?; } - let measurement = audit_reads::measurement_contract(&audit)?; + let measurement = measurement_contract(&audit)?; for field in ["value_column", "sample_count_column", "sample_unit"] { values.set_item( field, @@ -181,16 +209,13 @@ fn plan_audit<'py>( } for (field, value) in [ ("resolved_sql", resolved), - ("unresolved_sql", audit_reads::sql_body(&audit)?), + ("unresolved_sql", sql_body(&audit)?), ("attachment_kind", kind), ("severity", severity), ("requested_run_scope", requested), ("effective_run_scope", effective), ("evidence_resolved_sql", evidence_resolved), - ( - "evidence_unresolved_sql", - audit_reads::evidence_sql(&audit)?, - ), + ("evidence_unresolved_sql", evidence_sql(&audit)?), ("attached_target_kind", target_kind), ("attached_target_name", target), ( @@ -211,8 +236,8 @@ pub(crate) fn audit_is_selected( selected_keys: &HashSet, ) -> PyResult { let py = audit.py(); - let kind = audit_reads::attached_target_kind(audit)?; - let name = audit_reads::attached_target_name(audit)?; + let kind = attached_target_kind(audit)?; + let name = attached_target_name(audit)?; if !kind.is_none() && !name.is_none() { let target_value = native_field( &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, @@ -221,7 +246,7 @@ pub(crate) fn audit_is_selected( let key = (target_value.extract()?, name.extract()?); return Ok(selected_keys.contains(&key)); } - for dependency in audit_reads::scope_deps(audit)?.try_iter()? { + for dependency in scope_deps(audit)?.try_iter()? { if selected_keys.contains(&native_key(&dependency?)?) { return Ok(true); } @@ -238,7 +263,7 @@ pub(crate) fn build_selected_audit_entries<'py>( let project = request.get("project")?; let scope = scope_data(&request.get("scope")?)?; let mut selected: Vec> = Vec::new(); - for audit in project_reads::audits(&project)?.try_iter()? { + for audit in audits(&project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, &scope.selected)? { selected.push(audit); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 7602b752fc..9a11ddb71e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -15,13 +15,16 @@ use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_ru use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::compiled_views::main::audit_reads::references::references; +use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::planner_scope::main::edges::edges; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -31,7 +34,7 @@ const FINAL_RUN_SCOPE: &str = "final"; /// The names of an audit's references of one kind, in order. fn reference_names(audit: &Bound<'_, PyAny>, kind: &str) -> PyResult> { let mut names: Vec = Vec::new(); - for reference in audit_reads::references(audit)?.try_iter()? { + for reference in references(audit)?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; if text(&native_field(&reference, "ref_kind")?)? == kind { names.push(text(&native_field(&reference, "ref_name")?)?); @@ -77,7 +80,7 @@ pub(crate) fn attachment( graph: &ProjectGraph, ) -> PyResult<(String, Option)> { let py = audit.py(); - let attached_kind = audit_reads::attached_target_kind(audit)?; + let attached_kind = attached_target_kind(audit)?; let (kind, name): (String, Option) = if attached_kind.is_none() { let reached = RefCell::new(HashMap::new()); let reaches = |from: &str, to: &str| -> PyResult { @@ -100,14 +103,18 @@ pub(crate) fn attachment( )?; (attachment_value(kind).to_owned(), name) } else { - let name = audit_reads::attached_target_name(audit)?; + let name = attached_target_name(audit)?; if name.is_none() { return Err(input_error( py, ErrorId::PlannerInput, &format!( "audit '{}': attached audit is missing an attached target name", - text(&audit_reads::name(audit)?)? + text( + &sqlbuild_python_project::compiled_views::main::audit_reads::name::name( + audit + )? + )? ), None, )); @@ -141,8 +148,8 @@ pub(crate) fn outside_target_lineage( graph: &ProjectGraph, ) -> PyResult { let py: Python<'_> = audit.py(); - let kind: Bound<'_, PyAny> = audit_reads::attached_target_kind(audit)?; - let name: Bound<'_, PyAny> = audit_reads::attached_target_name(audit)?; + let kind: Bound<'_, PyAny> = attached_target_kind(audit)?; + let name: Bound<'_, PyAny> = attached_target_name(audit)?; if kind.is_none() || name.is_none() { return Ok(false); } @@ -150,7 +157,7 @@ pub(crate) fn outside_target_lineage( text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; let target = (kind, text(&name)?); let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); - for dependency in audit_reads::scope_deps(audit)?.try_iter()? { + for dependency in scope_deps(audit)?.try_iter()? { if !lineage.contains(&native_key(&dependency?)?) { return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs deleted file mode 100644 index eff32c12da..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Audit plan boundary modules and attachment values. - -pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -pub(crate) const MODEL_ATTACHMENT: &str = "model"; -pub(crate) const SOURCE_ATTACHMENT: &str = "source"; -pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -pub(crate) const MICROBATCH_MODE: &str = "microbatch"; -pub(crate) const CUSTOM_MATERIALIZATION: &str = "custom"; -pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; -pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; -pub(crate) const MODEL_REFERENCE: &str = "ref"; -pub(crate) const SEED_REFERENCE: &str = "seed"; -pub(crate) const SOURCE_REFERENCE: &str = "source"; -pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; -/// The lineage confidence whose reads fixtures must provide. -pub(crate) const HIGH_CONFIDENCE: &str = "high"; -/// The lineage transform kind that passes an input column's type through. -pub(crate) const DIRECT_TRANSFORM: &str = "direct"; -/// Resource type values keying fixture relations. -pub(crate) const MODEL_REFERENCE_KIND: &str = "model"; -pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; -pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; -/// The SQL analysis dialect whose adapter lays tables out by the physical storage keys. -pub(crate) const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; -/// The error code for physical storage keys on an adapter without them. -pub(crate) const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; -/// Plan action values that create the model's table anew or leave it untouched. -pub(crate) const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 457d7b7b85..19479b4834 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -7,18 +7,9 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_identity::metadata::or_empty; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_strategy::constants::{ - ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, - SOURCE_REFERENCE, TIMESTAMP_CURSOR, -}; use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_strategy::models::CursorInputLookups; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -32,6 +23,16 @@ use sqlbuild_python_front::model_config::main::config::config_cursor_bound::conf use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; +use sqlbuild_python_project::planner_strategy::constants::INCREMENTAL_MATERIALIZATION; +use sqlbuild_python_project::planner_strategy::constants::MODEL_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::SEED_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::SOURCE_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::TIMESTAMP_CURSOR; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; /// Whether a model is incremental with a cursor column. fn incremental_cursor_flag( @@ -109,7 +110,8 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let models_by_name = &lookups.models_by_name; let source_map: &Bound<'py, PyAny> = &lookups.source_map; let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; - let model_name: String = model_reads::model_name(model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name(model)?; for role in ["filter_inputs", "watermark_inputs"] { for item in mapping_items(&native_field(&roles, role)?)?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 30baee3ec5..c68388c7db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -9,10 +9,6 @@ use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_refere use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -22,6 +18,10 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::model_reads::config::config; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::native_host::models::ModelLookup; const MICROBATCH_MODE: &str = "microbatch"; @@ -44,7 +44,9 @@ fn mapped_references<'py>( } fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { - resource_reads::references(owner)?.try_iter()?.collect() + sqlbuild_python_project::compiled_views::main::resource_reads::references::references(owner)? + .try_iter()? + .collect() } /// Whether a model's `incremental_mode` is microbatch. @@ -90,7 +92,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { fn producer_facts(&self, name: &str) -> PyResult { let producer = self.models_by_name.get(self.functions_by_name.py(), name)?; - let values: Bound<'py, PyAny> = native_field(&model_reads::config(&producer)?, "values")?; + let values: Bound<'py, PyAny> = native_field(&config(&producer)?, "values")?; let capped: bool = match microbatch_limit_config(&values)?.1 { Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), None => false, @@ -131,7 +133,7 @@ pub(crate) fn resolve_lineage_reference<'py>( models_by_name, functions_by_name, }; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; lineage_reference(&graph, (&name, references(model)?), input_name) .map_err(|failure| lineage_error(py, failure)) } @@ -154,7 +156,7 @@ pub(crate) fn reject_capped_watermark_inputs<'py>( models_by_name, functions_by_name, }; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let inputs: Vec = watermark_names(model)?; match capped_watermark_input(&graph, (&name, references(model)?), inputs) { Ok(None) => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs index cb4ead6819..f21d6ca8e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -14,7 +14,6 @@ use sqlbuild_core::plan_display::errors::UnknownCursorGrain; use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use sqlbuild_core::plan_display::models::TimestampGrainFacts; -use crate::bindings::_helpers::compiled_views::records::replace; use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, @@ -23,6 +22,7 @@ use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::records::replace::replace; const ERROR_ACTION: &str = "error"; const SKIP_ACTION: &str = "skip"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index b29e2442b1..8176115560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,23 +1,11 @@ pub(crate) mod audit_entries; pub(crate) mod audits; -pub(crate) mod constants; pub(crate) mod cursor_inputs; pub(crate) mod cursor_types; -pub(crate) mod fixture_completion; -pub(crate) mod fixture_context; -pub(crate) mod fixture_metadata; -pub(crate) mod fixture_preparation; -pub(crate) mod fixture_reads; -pub(crate) mod fixture_selection; -pub(crate) mod fixture_sql; -pub(crate) mod fixture_types; -pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatch_limits; pub(crate) mod microbatches; pub(crate) mod model_entries; -pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; -pub(crate) mod types; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 20efc7a4c6..4a18d4b8f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -16,18 +16,11 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::planner_strategy::constants::{ - CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, - PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, TABLE_RECREATING_ACTIONS, -}; use crate::bindings::_helpers::planner_strategy::cursor_types::{ cursor_type_warning_record, warehouse_column_types, }; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; -use crate::bindings::_helpers::planner_strategy::models::ModelEntryDecisions; use crate::bindings::_helpers::planner_strategy::strategy::{ model_warning_records, resolve_schema_actions, warning_inputs, }; @@ -52,6 +45,20 @@ use sqlbuild_python_front::model_config::main::config::config_text_tuple::config use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_front::model_config::types::ConfigValues; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::model_reads::relative_path::relative_path; +use sqlbuild_python_project::planner_strategy::constants::CUSTOM_MATERIALIZATION; +use sqlbuild_python_project::planner_strategy::constants::DDL_SEPARATOR; +use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; +use sqlbuild_python_project::planner_strategy::constants::MICROBATCH_MODE; +use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_DIALECT; +use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_UNSUPPORTED_CODE; +use sqlbuild_python_project::planner_strategy::constants::TABLE_RECREATING_ACTIONS; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; /// Model config keys copied to the plan entry as optional strings, with their entry fields. const STRING_FIELDS: [(&str, &str); 15] = [ @@ -111,7 +118,10 @@ fn schema_columns<'py>( typed_only: bool, ) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = resource_reads::schema_entry(model)?; + let entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::resource_reads::schema_entry::schema_entry( + model, + )?; if entry.is_none() { return Ok(PyTuple::empty(py)); } @@ -141,7 +151,7 @@ fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult = model.py(); let class: Bound<'py, PyAny> = host_entry(py, AdapterHost::ColumnInfo)?; let mut columns: Vec> = Vec::new(); - for (name, column_type, _) in model_reads::inferred_rows(model)?.unwrap_or_default() { + for (name, column_type, _) in inferred_rows(model)?.unwrap_or_default() { if let Some(column_type) = column_type { columns.push(column_info( &class, @@ -194,7 +204,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; if !entry.is_none() { let enforcement: Bound<'py, PyAny> = native_field(&entry, "type_enforcement")?; if !enforcement.is_none() { @@ -261,7 +274,10 @@ fn logical_ddl<'py>( let py: Python<'py> = request.model.py(); let adapter: &Bound<'py, PyAny> = &request.adapter; let model: &Bound<'py, PyAny> = &request.model; - let destination: Bound<'py, PyAny> = model_reads::destination(model)?; + let destination: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?; let qualified: Bound<'py, PyAny> = native_field(&destination, "qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified @@ -324,7 +340,7 @@ fn logical_ddl<'py>( /// The output column names SQL analysis inferred for the model, when it inferred them. fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let inferred: Bound<'py, PyAny> = model_reads::inferred_columns(model)?; + let inferred: Bound<'py, PyAny> = sqlbuild_python_project::compiled_views::main::model_reads::inferred_columns::inferred_columns(model)?; if inferred.is_none() { return Ok(none(py)); } @@ -360,7 +376,7 @@ fn storage_fields<'py>( ErrorId::CompileInput, &format!( "model '{}' sets {}, but adapter '{}' has no physical table layout settings", - model_reads::model_name(&request.model)?, + model_name(&request.model)?, keys.join(", "), native_field(&request.adapter, "adapter_name")?.str()? ), @@ -410,7 +426,7 @@ fn model_warnings<'py>( ) -> PyResult> { let py: Python<'py> = request.model.py(); let model: &Bound<'py, PyAny> = &request.model; - let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let name: Bound<'py, PyAny> = name_object(model)?; let inputs = warning_inputs( &name, &request.materialization_type, @@ -591,7 +607,10 @@ fn contract_fields<'py>( mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?, )?; entry.set_item("contract_columns", schema_columns(model, false)?)?; - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + model, + )?; entry.set_item( "contract_dynamic_columns", if schema_entry.is_none() { @@ -649,7 +668,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let values: ConfigValues<'py> = model_values(model)?; let change: &Bound<'py, PyAny> = &request.change_result; let snapshot: &Bound<'py, PyAny> = &request.snapshot; - let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let name: Bound<'py, PyAny> = name_object(model)?; let policy: Bound<'py, PyAny> = on_schema_change(model)?; let schema_actions: Bound<'py, PyAny> = resolve_schema_actions(&native_field(change, "schema_findings")?, &policy)?.into_any(); @@ -667,7 +686,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( .unwrap_or_else(|| PyTuple::empty(py)); let table_type: Bound<'py, PyAny> = native_field( &native_field( - &native_field(&model_reads::config(model)?, "table_type")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + "table_type", + )?, "value", )?, "value", @@ -678,11 +700,16 @@ pub(crate) fn assemble_model_plan_entry<'py>( (&warehouse_columns, ddl_cursor_bounds(request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - entry.set_item("key", model_reads::model_key(model)?)?; + entry.set_item("key", model_key(model)?)?; entry.set_item("name", &name)?; - entry.set_item("relative_path", model_reads::relative_path(model)?)?; - entry.set_item("destination", model_reads::destination(model)?)?; - entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; + entry.set_item("relative_path", relative_path(model)?)?; + entry.set_item( + "destination", + sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + model, + )?, + )?; + entry.set_item("fingerprint_query_sql", query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; passed_fields(request, &entry)?; for field in CHANGE_FIELDS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs deleted file mode 100644 index 4f5a155c6e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! Extension-owned fixture analysis reuse and the project-wide fixture planning context. - -use std::collections::{BTreeSet, HashMap}; -use std::sync::{Arc, Mutex}; - -use pyo3::prelude::{Bound, Py}; -use pyo3::pyclass; -use pyo3::types::PyAny; -use sqlbuild_analysis::planner_fixtures::models::{ - FixtureColumn, FixtureReadCache, FixtureTypeProjection, -}; -use sqlbuild_analysis::required_columns::models::ColumnRead; - -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; - -/// Per-extension cache replacing the Python fixture resolver's LRU. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FixtureReadCacheHost { - pub(crate) cache: FixtureReadCache, - /// Fixture planning contexts built since the last reset. - pub(crate) context_builds: u64, - /// SQL-test model chain batches resolved since the last reset. - pub(crate) chain_resolutions: u64, -} - -/// One relation's known fixture columns and whether they form an authoritative shape. -#[derive(Clone, Debug)] -pub(crate) struct FixtureRelation { - pub(crate) columns: Vec, - pub(crate) authoritative: bool, -} - -/// One model's high-confidence lineage reads and direct type projections. -#[derive(Debug)] -pub(crate) struct ModelFixtureFacts { - pub(crate) reads: Vec, - pub(crate) projections: Vec, -} - -/// Project-wide fixture metadata shared across test and scenario planning. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct RelationFixturePlanningContext { - pub(crate) models_by_name: HashMap, - pub(crate) relations: HashMap, - pub(crate) authoritative_columns: HashMap>, - pub(crate) expected_types: HashMap>, - pub(crate) fixture_inferences: Py, - pub(crate) model_facts: Mutex>>, -} - -/// The expected column types of one completion: its chain's propagated types over the context's. -#[derive(Debug)] -pub(crate) struct ExpectedTypes<'a> { - pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, - pub(crate) base: &'a HashMap>, -} - -/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. -#[derive(Debug)] -pub(crate) struct ModelEntryDecisions<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) change_result: Bound<'py, PyAny>, - pub(crate) action: Bound<'py, PyAny>, - pub(crate) reason: Bound<'py, PyAny>, - pub(crate) materialization_type: Bound<'py, PyAny>, - pub(crate) resolved_sql: Bound<'py, PyAny>, - pub(crate) sql_analysis_enabled: bool, - pub(crate) hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) cursor_column: Bound<'py, PyAny>, - pub(crate) cursor_input_relations: Bound<'py, PyAny>, - pub(crate) runtime_owned_cursor_bounds: bool, - pub(crate) cursor_bounds: Bound<'py, PyAny>, - pub(crate) microbatch_range: Bound<'py, PyAny>, - pub(crate) empty_cursor_inputs: Bound<'py, PyAny>, - pub(crate) waiting_on_empty_inputs: bool, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) invocation_time: Bound<'py, PyAny>, -} - -/// The project lookups that resolve one model's cursor inputs. -#[derive(Debug)] -pub(crate) struct CursorInputLookups<'py> { - pub(crate) models_by_name: ModelLookup, - pub(crate) functions_by_name: Bound<'py, PyAny>, - pub(crate) source_map: Bound<'py, PyAny>, - pub(crate) source_warehouse_columns: Bound<'py, PyAny>, - pub(crate) model_locations: Bound<'py, PyAny>, - pub(crate) seed_locations: Bound<'py, PyAny>, - pub(crate) runtime_cursor_producer_names: Bound<'py, PyAny>, -} - -/// The relations and lineage that render and schedule selected audits. -#[derive(Debug)] -pub(crate) struct AuditPlanning<'py> { - pub(crate) model_locations: Bound<'py, PyAny>, - pub(crate) seed_locations: Bound<'py, PyAny>, - pub(crate) source_map: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) model_materializations: Bound<'py, PyAny>, - pub(crate) lineage: Py, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index fdbc656dbd..1752fef375 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -3,44 +3,43 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; -use crate::bindings::_helpers::planner_strategy::{ - audits, cursor_types, fixture_completion, microbatches, strategy, -}; +use crate::bindings::_helpers::planner_strategy::{audits, cursor_types, microbatches, strategy}; +use sqlbuild_python_project::planner_strategy::main::register_fixture_completion::register_fixture_completion; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( - super::audit_entries::plan_audit_of, + crate::bindings::_helpers::planner_strategy::audit_entries::plan_audit_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::cursor_input_columns_of, + crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_columns_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::cursor_input_relations_of, + crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_relations_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::runtime_cursor_producer_names, + crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names, module )?)?; module.add_function(wrap_pyfunction!( - super::model_entries::resolve_microbatch_lookback, + crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback, module )?)?; module.add_function(wrap_pyfunction!( - super::microbatch_limits::apply_microbatch_limit, + crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_microbatch_limit, module )?)?; module.add_function(wrap_pyfunction!( - super::microbatch_limits::apply_configured_microbatch_limits, + crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits, module )?)?; audits::register(module)?; cursor_types::register(module)?; - fixture_completion::register(module)?; - super::fixture_context::register(module)?; - super::fixture_reads::register(module)?; + register_fixture_completion(module)?; + sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context(module)?; + sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads(module)?; microbatches::register(module)?; strategy::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index c19156be0b..6cc856aad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -1,6 +1,5 @@ //! Per-model plan actions, schema actions and plan warnings. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; @@ -82,7 +81,9 @@ pub(crate) fn resolve_model_plan_action<'py>( let py: Python<'py> = model.py(); let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; let strategy = ModelStrategy { - name: model_reads::model_name(model)?, + name: sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( + model, + )?, disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, materialization: model_materialization(model)?, incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index c51fb8fe18..adf6e684b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,15 +1,5 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, - text, tuple, -}; use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; @@ -23,7 +13,6 @@ use crate::bindings::_helpers::planner_warehouse::snapshot::{ SnapshotRequest, inspection_concurrency, }; use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -50,6 +39,21 @@ use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::planner_phases::main::by_name::by_name; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; +use sqlbuild_python_project::planner_phases::main::model_key::model_key; +use sqlbuild_python_project::planner_phases::main::native_member::native_member; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -210,10 +214,13 @@ fn runtime_producer_names( return Ok(names); } let mut loaded: HashSet = HashSet::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let source = source?; - if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { - loaded.insert(source_reads::name(&source)?.extract()?); + if !native_field(&source_entry(&source)?, "loader")?.is_none() { + loaded.insert( + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + .extract()?, + ); } } for key in selected.try_iter()? { @@ -239,7 +246,7 @@ fn upstream_relation<'py>( let name_text: String = name.extract()?; let is_selected = selected.is_some_and(|names| names.contains(&name_text)); let qualified = |node: Bound<'py, PyAny>| { - native_field(&resource_reads::destination(&node)?, "qualified_name") + native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "qualified_name") }; if (kind.eq("ref")? || kind.eq("seed")?) && !deferred.is_none() @@ -265,7 +272,7 @@ fn upstream_relation<'py>( return object( py, render_source_relation( - &source_reads::source_entry(&source)?, + &source_entry(&source)?, Some(adapter).filter(|value| !value.is_none()), )?, ); @@ -380,9 +387,12 @@ fn collect_models<'py>( }; let runtime_names = runtime_producer_names(project, &producers)?; let mut models: Vec<_> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; - let name_value = model_reads::name_object(&model)?; + let name_value = name_object(&model)?; let model_name: String = name_value.extract()?; if !inputs.selected_keys.is_none() && !inputs.selected_keys.contains(model_key(py, &name_value)?)? @@ -422,7 +432,12 @@ fn collect_models<'py>( None => None, }; let target_info = existing_relations.get_item(&name_value)?; - let qualified = native_field(&model_reads::destination(&model)?, "qualified_name")?; + let qualified = native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?, + "qualified_name", + )?; if let Some(redirected) = redirected.filter(|value| !value.is_none()) { target_tag = Some(format!("{model_name}__target__max")); target_relation = Some(redirected.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 2c063e4700..d5c1e961ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; use pyo3::PyRefMut; use pyo3::exceptions::{PyOSError, PyValueError}; @@ -20,6 +19,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::object::object; use std::collections::{BTreeMap, HashMap}; use std::hash::{DefaultHasher, Hash, Hasher}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index 68d93023e6..9d2f515934 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -1,8 +1,5 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -17,6 +14,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::references::references; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; use std::collections::BTreeSet; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; @@ -194,16 +196,16 @@ pub(crate) fn semantic_source_columns<'py>( return Ok(columns.clone()); } let mut required: BTreeSet = BTreeSet::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; let analysis_off: bool = matches!( *config_node_of(&model_values(&model)?, SQL_ANALYSIS_KEY)?, ConfigNode::Bool(false) ); - if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis_off { + if !selected_keys.contains(model_key(&model)?)? || analysis_off { continue; } - for reference in model_reads::references(&model)?.try_iter()? { + for reference in references(&model)?.try_iter()? { let reference = reference?; if native_field(&reference, "ref_kind")?.eq("source")? { required.insert(native_field(&reference, "ref_name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 02cf6f7df8..34783f37ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -1,20 +1,6 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - by_name, frozenset, model_key, native, text, tuple, -}; -use crate::bindings::_helpers::planner_scope::loaders::build_upstream_intermediate_source_map; use crate::bindings::_helpers::planner_warehouse::constants::{ ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, @@ -43,6 +29,22 @@ use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::model_reads::references::references; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_python_project::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; +use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_phases::main::by_name::by_name; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::model_key::model_key; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::build_upstream_intermediate_source_map::build_upstream_intermediate_source_map; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. @@ -82,9 +84,11 @@ pub(crate) fn destination_nodes<'py>( ) -> PyResult>> { let mut nodes: Vec<_> = Vec::new(); let collections: [Bound<'py, PyAny>; 3] = [ - project_reads::models(project)?, - project_reads::seeds(project)?, - project_reads::functions(project)?, + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)?, + sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, + sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + project, + )?, ]; for collection in collections { for node in collection.try_iter()? { @@ -95,7 +99,10 @@ pub(crate) fn destination_nodes<'py>( } fn selected_node(selected: &Bound<'_, PyAny>, node: &Bound<'_, PyAny>) -> PyResult { - Ok(selected.is_none() || selected.contains(resource_reads::key(node)?)?) + Ok(selected.is_none() + || selected.contains( + sqlbuild_python_project::compiled_views::main::resource_reads::key::key(node)?, + )?) } fn relevant_state_keys<'py>( @@ -125,7 +132,7 @@ fn resolve_database<'py>( if !selected_node(selected, &node)? { continue; } - let database = native_field(&resource_reads::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -142,7 +149,7 @@ fn target_schemas( if !selected_node(selected, &node)? { continue; } - let schema = native_field(&resource_reads::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -169,7 +176,7 @@ fn model_upstream_names( selected_names: &HashSet, ) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); - for reference in model_reads::references(model)?.try_iter()? { + for reference in references(model)?.try_iter()? { let reference = reference?; let kind = native_field(&reference, "ref_kind")?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -207,21 +214,24 @@ fn metadata_name_filter( let models = by_name(project, "models")?; let seeds = by_name(project, "seeds")?; let sources = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { + let entry = source_entry(&source?)?; sources.set_item(native_field(&entry, "name")?, entry)?; } for key in selected.try_iter()? { let name = native_field(&key?, "name")?; if let Some(model) = models.get_item(&name)? { - names.insert(native_field(&model_reads::destination(&model)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "name")?.extract()?); names.extend(model_upstream_names( &model, (&models, &seeds), &selected_names, )?); } else if let Some(seed) = seeds.get_item(&name)? { - names.insert(native_field(&seed_reads::destination(&seed)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?, "name")?.extract()?); } else if let Some(entry) = sources.get_item(&name)? { names.insert(source_table_name(&entry)?); } @@ -288,10 +298,17 @@ fn prefetch_schemas<'py>( for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); } - for source in project_reads::sources(&request.project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project_reads::sources::sources( + &request.project, + )? + .try_iter()? + { let source = source?; - let entry = source_reads::source_entry(&source)?; - if (!relevant.is_none() && !relevant.contains(source_reads::key(&source)?)?) + let entry = source_entry(&source)?; + if (!relevant.is_none() + && !relevant.contains( + sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + )?) || !native_field(&entry, "expression")?.is_none() || native_field(&entry, "schema")?.is_none() { @@ -391,8 +408,8 @@ fn gather_relations<'py>( let logical = PyDict::new(py); for node in destination_nodes(&request.project)? { logical.set_item( - location_identity(&resource_reads::destination(&node)?)?, - resource_reads::name(&node)?, + location_identity(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?)?, + sqlbuild_python_project::compiled_views::main::resource_reads::name::name(&node)?, )?; } let result = PyDict::new(py); @@ -442,7 +459,10 @@ fn gather_columns<'py>( let py = request.project.py(); let mut node_names: HashSet = HashSet::new(); for node in destination_nodes(&request.project)? { - node_names.insert(resource_reads::name(&node)?.extract()?); + node_names.insert( + sqlbuild_python_project::compiled_views::main::resource_reads::name::name(&node)? + .extract()?, + ); } let mut needed: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for (logical, relation) in relations.iter() { @@ -483,9 +503,12 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> .collect::>()?; let mut all_selected = true; let mut any_migration = false; - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model = model?; - if !names.contains(&model_reads::model_name(&model)?) { + if !names.contains(&model_name(&model)?) { all_selected = false; } if config_node_of(&model_values(&model)?, MIGRATE_FROM_CONFIG_KEY)?.kind() == NodeKind::Str @@ -857,11 +880,11 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( if let Some(model) = models.get_item(&name)? { result.set_item( name, - crate::bindings::_helpers::planner_phases::objects::replace( + sqlbuild_python_project::planner_phases::main::replace::replace( &snapshot, &[( "target_relation", - native_field(&model_reads::destination(&model)?, "qualified_name")?, + native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "qualified_name")?, )], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index f8eef566ff..6210af027b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -1,9 +1,6 @@ //! Warehouse source relation listing and column gathering for planning. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::observations::source_capture::capture_source_columns; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, }; @@ -15,6 +12,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; use std::collections::{BTreeSet, HashSet}; fn entries_of<'py>( @@ -23,9 +25,9 @@ fn entries_of<'py>( ) -> PyResult>> { match source_entries.filter(|entries| !entries.is_none()) { Some(entries) => entries.try_iter()?.collect(), - None => project_reads::sources(project)? + None => sources(project)? .try_iter()? - .map(|source| source_reads::source_entry(&source?)) + .map(|source| source_entry(&source?)) .collect(), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b93aaa112b..f64f63abd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -2,15 +2,6 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - optional_text, planner_error, replace, text, -}; use crate::bindings::_helpers::planner_warehouse::constants::{ MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, }; @@ -27,12 +18,29 @@ use sqlbuild_python_front::authored_values::main::render_project_var_text::rende use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::audit_reads::references::references; +use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::compiled_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; +use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::source_reads::source_file::source_file; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::text::text; use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; if !native_field(&entry, "loader")?.is_none() { names.insert(native_field(&entry, "name")?.extract()?); } @@ -91,26 +99,25 @@ fn selected_managed_refs( managed: &BTreeSet, ) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model = model?; - if !selected.contains(model_reads::model_key(&model)?)? { + if !selected.contains(model_key(&model)?)? { continue; } - names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); + names.extend(sql_source_reads(&query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } - for function in project_reads::functions(project)?.try_iter()? { + for function in functions(project)?.try_iter()? { let function = function?; - if selected.contains(function_reads::key(&function)?)? { - names.extend(sql_source_reads( - &function_reads::body_sql(&function)?, - managed, - )?); + if selected.contains( + sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + )? { + names.extend(sql_source_reads(&body_sql(&function)?, managed)?); } } - for audit in project_reads::audits(project)?.try_iter()? { + for audit in audits(project)?.try_iter()? { let audit = audit?; - let deps = audit_reads::scope_deps(&audit)?; + let deps = scope_deps(&audit)?; if deps.is_truthy()? { let mut any = false; for dep in deps.try_iter()? { @@ -123,14 +130,14 @@ fn selected_managed_refs( continue; } } - for reference in audit_reads::references(&audit)?.try_iter()? { + for reference in references(&audit)?.try_iter()? { let reference = reference?; let name: String = native_field(&reference, "ref_name")?.extract()?; if native_field(&reference, "ref_kind")?.eq("source")? && managed.contains(&name) { names.insert(name); } } - names.extend(sql_source_reads(&audit_reads::sql_body(&audit)?, managed)?); + names.extend(sql_source_reads(&sql_body(&audit)?, managed)?); } Ok(names) } @@ -173,9 +180,9 @@ fn source_target_name<'py>( } fn raw_source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let declared = source_reads::source_entry(source)?; + let declared = source_entry(source)?; let name = native_field(&declared, "name")?; - for entry in native_field(&source_reads::source_file(source)?, "source_entries")?.try_iter()? { + for entry in native_field(&source_file(source)?, "source_entries")?.try_iter()? { let entry = entry?; if native_field(&entry, "name")?.eq(&name)? { return Ok(entry); @@ -306,10 +313,9 @@ pub(crate) fn build_source_read_map<'py>( let target = resolve_target(project_config, local_config, target_name)?; let effective_vars = native_field(project, "effective_vars")?; let result = source_map.copy()?; - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let source = source?; - let name: String = - native_field(&source_reads::source_entry(&source)?, "name")?.extract()?; + let name: String = native_field(&source_entry(&source)?, "name")?.extract()?; if !selected.contains(&name) { continue; } @@ -339,8 +345,8 @@ pub(crate) fn with_declared_source_reads<'py>( return Ok(source_map.clone()); } let result = source_map.copy()?; - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; let name: String = native_field(&entry, "name")?.extract()?; if missing.contains(&name) { result.set_item(name, entry)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 61c4672fa8..629dc90bda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -1,13 +1,9 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; -use crate::bindings::_helpers::planner_phases::objects::{native, replace, text, tuple}; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -25,6 +21,11 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeSet; /// One warning per reason that left configured source freshness unknown. @@ -72,7 +73,7 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { for node in destination_nodes(project)? { - let database = native_field(&resource_reads::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -86,7 +87,7 @@ fn state_schemas( ) -> PyResult> { let mut schemas: BTreeSet<_> = BTreeSet::new(); for node in destination_nodes(project)? { - let schema = native_field(&resource_reads::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -100,7 +101,7 @@ fn state_schemas( let Some(model) = data.model(scope.py(), name) else { continue; }; - let schema = native_field(&model_reads::destination(&model)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 96362f5a61..be4a7dbf1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -1,13 +1,8 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -27,6 +22,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::discovery::main::project::yaml_entry_line::yaml_entry_line; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::source_reads::source_file::source_file; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { @@ -80,13 +81,14 @@ fn read_relation<'py>( } fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { - let file = source_reads::source_file(source)?; + let file = source_file(source)?; let path: String = native_field(&file, "relative_path")? .call_method0("as_posix")? .extract()?; let line = yaml_entry_line( &native_field(&file, "contents")?.extract::()?, - &source_reads::name(source)?.extract::()?, + &sqlbuild_python_project::compiled_views::main::source_reads::name::name(source)? + .extract::()?, ); Ok(match line { None => path, @@ -106,13 +108,18 @@ pub(crate) fn check_selected_source_tables_exist<'py>( let read_map = native_field(relations, "source_read_map")?; let listed = native_field(relations, "listed_source_names")?; let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { let source = source?; - let name: String = source_reads::name(&source)?.extract()?; + let name: String = + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + .extract()?; let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { continue; }; - let declared = source_reads::source_entry(&source)?; + let declared = source_entry(&source)?; let entry = mapping_get_or(&read_map, &name, &declared)?; if !external_table(&declared)? || !external_table(&entry)? { continue; @@ -162,7 +169,10 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let role = native_field(&probe, "role")?; missing.push(MissingSourceTable { - source_name: source_reads::name(&source)?.extract()?, + source_name: sqlbuild_python_project::compiled_views::main::source_reads::name::name( + &source, + )? + .extract()?, relation: String::new(), read_by, unreadable: status.eq("missing_or_unreadable")?, @@ -183,9 +193,15 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + .try_iter()? + { let source = source?; - sources.set_item(source_reads::name(&source)?, source)?; + sources.set_item( + sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + source, + )?; } for item in missing.iter_mut().take(10) { if let Some(source) = sources.get_item(&item.source_name)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index b1adf98e3f..3565911981 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,10 +1,5 @@ //! Validate reliable watermark column contracts before warehouse inspection. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, @@ -15,6 +10,12 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::text::text; use std::collections::HashMap; const CONTRACT_KEY: &str = "contract"; @@ -75,7 +76,8 @@ pub(crate) fn validate_watermark_column<'py>( let py = model.py(); let kind = native_field(reference, "ref_kind")?; let ref_name: String = native_field(reference, "ref_name")?.extract()?; - let model_name: String = model_reads::model_name(model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name(model)?; let lowered = |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; let (names, types) = if kind.eq("ref")? { @@ -112,7 +114,7 @@ pub(crate) fn validate_watermark_column<'py>( let Some(source) = sources.get_item(&ref_name)? else { return Ok(()); }; - let entry = source_reads::source_entry(&source)?; + let entry = source_entry(&source)?; if !native_field(&entry, "contract")?.eq("enforced")? { return Ok(()); } @@ -121,7 +123,7 @@ pub(crate) fn validate_watermark_column<'py>( let Some(seed) = seeds.get_item(&ref_name)? else { return Ok(()); }; - declared_columns(&native_field(&seed_reads::schema_entry(&seed)?, "columns")?)? + declared_columns(&native_field(&schema_entry(&seed)?, "columns")?)? } else { return Ok(()); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index e21f9cb61c..d43b3032de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -8,17 +8,16 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - attached_columns, attached_schema, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, SourceViews, read_source_view, -}; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::enum_member_value::enum_member_value; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use sqlbuild_python_project::compiled_views::main::model_input_view::attached_columns::attached_columns; +use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::compiled_views::types::AttachedSource; +use sqlbuild_python_project::compiled_views::types::SourceViews; const UNKNOWN_TYPE: &str = "UNKNOWN"; const NOT_NULL_AUDIT_NAME: &str = "not_null"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index e72461f44f..98360c9e15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -17,7 +17,6 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_model_config::templates::models::Scalar; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, }; @@ -34,6 +33,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::authored_values::main::project_var_text::project_var_text; use sqlbuild_python_front::model_config::main::config::record_reads::record_reads; +use sqlbuild_python_project::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 64cb409ce9..95d50398cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -6,10 +6,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index be9ac7a81a..c2e3ccf08d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -9,13 +9,13 @@ use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { document diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index bfab8b233a..bfa7a3382d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,7 +36,6 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ ClickHouseLocation, ClickHouseStoreOpener, missing_database_reason, @@ -53,6 +52,7 @@ use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_rep use crate::bindings::_helpers::recall::sinks::InvocationEvents; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs index a9c5c43c85..020327e3c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -4,13 +4,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compiled_views::relation_keys::entry_relation_key_row; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::relation_keys::entry_relation_key_row::entry_relation_key_row; /// `CompilerDiagnostic` positions. const DIAGNOSTIC_PHASE_FIELD: usize = 0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index ea894c5e18..42514a58ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,9 +1,6 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. use crate::bindings::_helpers::freshness::observation::normalize_observation; -use crate::bindings::_helpers::lineage::lineage_facts::{ - facts_eq, facts_getitem, facts_iter, facts_len, facts_projected, facts_resource_name, -}; use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; @@ -73,14 +70,20 @@ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; use crate::bindings::_helpers::{ - analysis, analysis_session, build_progress, compile_command, compiled_views, compute_logs, - dbt_events, dbt_interop, event_exporting, execution_history, graph, planner_entries, - planner_migrations, planner_phases, project_assembly, result_output, rules, + analysis, analysis_session, build_progress, compile_command, compute_logs, dbt_events, + dbt_interop, event_exporting, execution_history, planner_entries, planner_migrations, + planner_phases, project_assembly, result_output, rules, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; use sqlbuild_python_base::records::types::RecordFinish; +use sqlbuild_python_project::lineage::main::facts_eq::facts_eq; +use sqlbuild_python_project::lineage::main::facts_getitem::facts_getitem; +use sqlbuild_python_project::lineage::main::facts_iter::facts_iter; +use sqlbuild_python_project::lineage::main::facts_len::facts_len; +use sqlbuild_python_project::lineage::main::facts_projected::facts_projected; +use sqlbuild_python_project::lineage::main::facts_resource_name::facts_resource_name; /// Compiler records whose bound values are normalized or validated natively. pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ @@ -359,7 +362,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ build_progress::getters::GETTERS, compile_command::getters::GETTERS, sqlbuild_python_base::compile_context::constants::GETTERS, - compiled_views::getters::GETTERS, + sqlbuild_python_project::compiled_views::constants::GETTERS, compute_logs::getters::GETTERS, dbt_events::getters::GETTERS, dbt_interop::getters::GETTERS, @@ -367,7 +370,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_front::discovery::constants::GETTERS, event_exporting::getters::GETTERS, execution_history::getters::GETTERS, - graph::getters::GETTERS, + sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, planner_entries::getters::GETTERS, planner_migrations::getters::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index f93c691b4e..fb0b307779 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -4,12 +4,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::graph::project_graph::{ResourceRow, project_graph}; use crate::bindings::_helpers::record_behaviours::compiled::compile_input_error; use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::declarations::constant_declaration_sql::constant_declaration_sql; use sqlbuild_python_front::render::main::declarations::render_enum_member_value::render_enum_member_value; +use sqlbuild_python_project::graph::main::project_graph::project_graph::project_graph; +use sqlbuild_python_project::graph::models::ResourceRow; /// The parameter name through which a macro receives its context. const MACRO_CONTEXT_PARAMETER_NAME: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs index 3b5ab27c60..6ab2e79409 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs @@ -6,10 +6,10 @@ use pyo3::types::{ }; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_project::graph::models::PyNodeSet; /// `PythonNodeGraph.selected_sql_refs(selected_names=...)`: SQL references of the selected closure. pub(crate) fn graph_selected_sql_refs( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs index f183f2d284..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs @@ -1,5 +1 @@ -pub(crate) mod explicit_references; -pub(crate) mod reference_facts; pub(crate) mod registration; -pub(crate) mod relations; -pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs index cacffa8feb..5391badc54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs @@ -1,6 +1,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::references::relations::register(module)?; - crate::bindings::_helpers::references::source_relations::register(module) + sqlbuild_python_project::references::main::register_relations::register_relations(module)?; + sqlbuild_python_project::references::main::register_source_relations::register_source_relations( + module, + ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs index 0bbf05581d..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs @@ -1,2 +1 @@ -pub(crate) mod fitting; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs index 7a489501cf..240f77f73a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs @@ -2,8 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::relation_naming::fitting; +use sqlbuild_python_project::relation_naming::main::register_fitting::register_fitting; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - fitting::register(module) + register_fitting(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 8af26d5baa..545808be50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -18,7 +18,6 @@ use sqlbuild_cli::diff_report::types::DiffNamedValue; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::diff_report::command::target_max_full_rows; use crate::bindings::_helpers::diff_report::facts::diff_facts; @@ -78,7 +77,10 @@ fn models_named<'py>( names: &[String], ) -> PyResult)>> { let mut models: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model: Bound<'py, PyAny> = model?; let name: String = native_field(&model, "name")?.extract()?; if names.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs index 85d1a7c0ba..4e0c1ad6b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs @@ -7,12 +7,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString}; use sqlbuild_analysis::review_command::models::NodeComparison; -use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; /// The columns of a changed model a comparison proved changed, or `None` when every column may. fn changed_columns(comparison: Option<&NodeComparison>) -> Option> { @@ -44,9 +45,12 @@ pub(crate) fn affected_columns( ) -> PyResult>> { let py: Python<'_> = project.py(); let mut models: BTreeMap> = BTreeMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in + sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? + .try_iter()? + { let model: Bound<'_, PyAny> = model?; - let name: String = model_reads::model_name(&model)?; + let name: String = model_name(&model)?; if affected.contains(&name) { models.insert(name, model); } @@ -72,7 +76,7 @@ pub(crate) fn affected_columns( }; let columns: Vec = match changed_columns(comparisons.get(name)) { Some(columns) => columns, - None => model_reads::inferred_names(model)?.unwrap_or_default(), + None => inferred_names(model)?.unwrap_or_default(), }; pending.extend(columns.into_iter().map(|column| (name.clone(), column))); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index 0a225d96f2..7c093111ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -24,12 +24,7 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::project_wi use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::execution_history::conversion::optional_text; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -41,6 +36,12 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; use sqlbuild_python_front::discovery::main::project::configured_review::configured_review; +use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const BASE_REF_ALIAS: &str = "@base"; const UDF_KIND: &str = "udf"; @@ -213,12 +214,15 @@ fn key_columns(value: &Bound<'_, PyAny>) -> PyResult> { /// One model as a review node, critical by its `critical` key or one of the critical tags. fn model_node(model: &Bound<'_, PyAny>, critical_tags: &[String]) -> PyResult { - let values: Bound<'_, PyAny> = native_field(&model_reads::config(model)?, "values")?; + let values: Bound<'_, PyAny> = native_field( + &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + "values", + )?; let tags: Vec = strings(&config_value(&values, "tags")?)?; let marked: Bound<'_, PyAny> = config_value(&values, "critical")?; let materialized: Bound<'_, PyAny> = config_value(&values, "materialized")?; Ok(ReviewNode { - name: model_reads::model_name(model)?, + name: model_name(model)?, kind: MODEL_KIND.to_owned(), critical: (!marked.is_none() && marked.is_truthy()?) || tags.iter().any(|tag| critical_tags.contains(tag)), @@ -377,10 +381,16 @@ fn review_graph( stateful: BTreeSet::new(), databases: BTreeMap::new(), }; - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed: Bound<'_, PyAny> = seed?; - let name: String = seed_reads::name(&seed)?.extract()?; - if let Some(database) = destination_database(&seed_reads::destination(&seed)?)? { + let name: String = + sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + .extract()?; + if let Some(database) = destination_database( + &sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( + &seed, + )?, + )? { project_graph.databases.insert(name.clone(), database); } project_graph @@ -397,7 +407,7 @@ fn review_graph( ); } let mut dependencies: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; let node: ReviewNode = model_node(&model, critical_tags)?; if node.materialized == INCREMENTAL_MATERIALIZATION @@ -405,13 +415,17 @@ fn review_graph( { project_graph.stateful.insert(node.name.clone()); } - if let Some(database) = destination_database(&model_reads::destination(&model)?)? { + if let Some(database) = destination_database( + &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + &model, + )?, + )? { project_graph.databases.insert(node.name.clone(), database); } project_graph .keys .insert(node.name.clone(), node_key(MODEL_KIND, &node.name)); - dependencies.push((node.name.clone(), model_reads::dep_rows(&model)?)); + dependencies.push((node.name.clone(), dep_rows(&model)?)); project_graph.graph.nodes.insert(node.name.clone(), node); } let mut sources: BTreeMap> = BTreeMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index c0cb55ff21..9fe21fe6b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -13,7 +13,6 @@ use sqlbuild_rules::models::RulesConfig; use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSpec, OptionValue}; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules as verify_rule_sources; @@ -27,6 +26,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::main::fields::path_posix::path_posix; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs index 93ca3aed85..d77bdcda7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs @@ -12,17 +12,18 @@ use pyo3::types::{ use serde_json::{Map, Value, json}; use sqlbuild_discovery::models::LineColumnSpan; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::enum_columns; -use crate::bindings::_helpers::compiled_views::records::{ - compiled_audit_record, compiled_function_record, compiled_seed_record, compiled_source_record, -}; use crate::bindings::_helpers::rules::custom_host_columns::PayloadColumn; use crate::bindings::_helpers::rules::custom_hosts::pickled; use crate::bindings::_helpers::rules::fact_replay::{ ModelEntry, project_attribute, project_models, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::model_reads::enum_columns::enum_columns; +use sqlbuild_python_project::compiled_views::main::records::compiled_audit_record::compiled_audit_record; +use sqlbuild_python_project::compiled_views::main::records::compiled_function_record::compiled_function_record; +use sqlbuild_python_project::compiled_views::main::records::compiled_seed_record::compiled_seed_record; +use sqlbuild_python_project::compiled_views::main::records::compiled_source_record::compiled_source_record; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; /// The first bytes of every payload file; hosts reject any other. pub(crate) const CUSTOM_HOST_PAYLOAD_MAGIC: &[u8; 8] = b"SQBHOST1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index f51c455187..cb1d9f364d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -21,11 +21,11 @@ use sqlbuild_rules::custom_rule_cache::models::{ }; use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResults}; -use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; /// One custom rule's cache identity from its metadata, source fingerprint and settings. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 6e9e3d08a3..4947b57411 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -17,10 +17,6 @@ use sqlbuild_rules::engine::main::start_speculative_rows::start_speculative_rows use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::records::{ - compiled_project_facts, record_model_analyses, -}; use crate::bindings::_helpers::rules::catalogue::references_custom_rules; use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; use crate::bindings::_helpers::rules::rule_plan::{rule_run_plan, selects_row_rules}; @@ -28,6 +24,9 @@ use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::records::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::records::record_model_analyses::record_model_analyses; const CANCELLED: &str = "native rules evaluation was cancelled"; const BUILD_PHASE: &str = "early_rules_build_ms"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 2b8ca2857c..20d454a6ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -22,10 +22,6 @@ use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; -use crate::bindings::_helpers::compiled_views::relation_keys::model_key_row; -use crate::bindings::_helpers::compiled_views::relation_keys::{ - RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, -}; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; @@ -40,6 +36,12 @@ use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interru use sqlbuild_python_base::native_threads::models::Joined; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_project::compiled_views::main::relation_keys::model_key_row::model_key_row; +use sqlbuild_python_project::compiled_views::main::relation_keys::one_row_seed_names::one_row_seed_names; +use sqlbuild_python_project::compiled_views::main::relation_keys::relation_keys::relation_keys; +use sqlbuild_python_project::compiled_views::main::relation_keys::source_key_input::source_key_input; +use sqlbuild_python_project::compiled_views::models::RelationKeyRow; +use sqlbuild_python_project::compiled_views::types::RelationKeys; /// The lint thresholds used when the project leaves them unset. const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 564ae8ea7d..5a6e1e1415 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -13,12 +13,6 @@ use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_pa use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; use crate::bindings::_helpers::rules::fact_encoder::{ Encoded, FactEncoder, FactFailure, error_digest, value_digest, }; @@ -31,6 +25,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as record_field_or_attribute; use sqlbuild_python_base::records::main::fields::record_spec::record_spec; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::models::NativeCompiledAudit; +use sqlbuild_python_project::compiled_views::models::NativeCompiledFunction; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::compiled_views::models::NativeCompiledSeed; +use sqlbuild_python_project::compiled_views::models::NativeCompiledSource; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; const MODEL_RECORD: &str = "sqlbuild._native.Model"; const MATERIALIZED: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index bdbb81f607..6365bd65e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -17,9 +17,6 @@ use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compiled_views::relation_keys::{ - RelationKeys, project_relation_keys, -}; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::fix_facts::compile_facts; @@ -43,6 +40,8 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_project::compiled_views::main::relation_keys::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::types::RelationKeys; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 6b085a37ce..07a6fa78f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -5,10 +5,10 @@ use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::models::InterpolationSite; use sqlbuild_rules::lint_text::types::PreparedSql; -use crate::bindings::_helpers::compiled_views::relation_keys; -use crate::bindings::models::Positional; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::Positional; +use sqlbuild_python_project::compiled_views::main::relation_keys::register_relation_keys::register_relation_keys; #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { @@ -63,5 +63,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; - relation_keys::register(module) + register_relation_keys(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 794d7a499b..314baf9875 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -6,16 +6,15 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputRecord, NativeModelInput, -}; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::types::SpanRow; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::compiled_views::types::ModelInputRecord; const FILE_PATH_ATTRIBUTE: &str = "file_path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index cd3b67bc3c..145f1d66f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -18,7 +18,6 @@ use sqlbuild_rules::lint_text::models::{ use crate::bindings::_helpers::rules::lint_pipeline::{ LintViolationRow, NATIVE_ENGINE, ViolationClasses, project_relative_path, }; -use crate::bindings::models::Positional; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -27,6 +26,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::Positional; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index cce37d1f3f..50404e7c88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -25,11 +25,6 @@ use sqlbuild_rules::models::{ RowsEvaluation, RowsRequest, RulesConfig, }; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; -use crate::bindings::_helpers::compiled_views::records::declaration_rows; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; @@ -48,6 +43,11 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::scopes::main::reports::metadata_projection_value::metadata_projection_value; +use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::records::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::records::declaration_rows::declaration_rows; +use sqlbuild_python_project::compiled_views::models::NativeCompiledAudit; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 5a0e5b77ef..c952fad812 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -16,7 +16,6 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::rules::catalogue::{ resolved_ignore_selectors, rule_catalogue, select_rules, }; @@ -53,6 +52,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use sqlbuild_rules::models::RowsRequest; use std::sync::Arc; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 4c6143f6ab..a0d61a71c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -7,7 +7,6 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use std::path::Path; -use crate::bindings::_helpers::compiled_views::records::project_with; use crate::bindings::_helpers::rules::catalogue::{ references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, verify_custom_rules, @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; const MODEL_RESOURCE_TYPE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c0f7a4e6fb..9cabbfb701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -19,11 +19,6 @@ use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_liter use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, SqlRuleInputs, SqlRuleKey, -}; -use crate::bindings::_helpers::compiled_views::model_reads::output_proven; -use crate::bindings::_helpers::compiled_views::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::early_sql_lint::{ @@ -53,6 +48,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::model_reads::output_proven::output_proven; +use sqlbuild_python_project::compiled_views::main::relation_keys::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::compiled_views::models::SqlRuleInputs; +use sqlbuild_python_project::compiled_views::models::SqlRuleKey; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs index 4e2021bb60..d0ac6723af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -10,12 +10,12 @@ use sqlbuild_rules::models::{ SqlTestParameterFact, SqlTestParameterValueFact, TestedResource, }; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::records::plain_typed_value; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::compiled_views::main::records::plain_typed_value::plain_typed_value; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 11983dd763..e7c239a10b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -27,7 +27,6 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_project_conn use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; use sqlbuild_python_front::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 8f9d283296..73a9a0da16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_ use crate::bindings::_helpers::compile_driver::project::compile_adapter_context; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -31,6 +30,7 @@ use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_na use sqlbuild_python_front::scopes::main::index::project_fingerprint::project_fingerprint; use sqlbuild_python_front::scopes::main::index::read_scope_index_cache::read_scope_index_cache; use sqlbuild_python_front::scopes::main::index::write_scope_index_cache::write_scope_index_cache; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 74fc45deaa..7fa2339aa4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invo use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_seed_command_plan; use crate::bindings::_helpers::planner_entries::command_plans::read_selected_seed_fingerprints; @@ -39,6 +38,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::compiled_project::types::VerifiedRun; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index d4b551c8d9..bddd0a01f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -8,9 +8,6 @@ use sqlbuild_analysis::semantic_checks::models::{ CompletedDiagnostic, CompletionModel, CompletionRequest, SemanticDiagnostic, SemanticLocation, }; -use crate::bindings::_helpers::compiled_views::compiled_model_view::LineageInput; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_names, lineage_inputs}; -use crate::bindings::_helpers::compiled_views::records::with_binding_diagnostics; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::metadata::{ShapeRows, shape_rows}; use crate::bindings::_helpers::semantic_checks::session_facts::{ @@ -20,10 +17,14 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::{ StageShared, diagnostic_ids, id_of, items, lineage_outputs, }; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::model_reads::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::records::with_binding_diagnostics::with_binding_diagnostics; +use sqlbuild_python_project::compiled_views::types::LineageInput; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; const RECOVERED_CODE: &str = "B002"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs index 186498a7f4..aa627d60a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -3,17 +3,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use crate::bindings::_helpers::analysis::binding_catalog::ignores_quoted_case; -use crate::bindings::_helpers::analysis::schema_request::{ - SchemaRequestParts, schema_validation_request, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; use crate::bindings::_helpers::semantic_checks::resource_sql::resource_sql_diagnostics_of; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::analysis::main::ignores_quoted_case::ignores_quoted_case; +use sqlbuild_python_project::analysis::main::schema_validation_request::schema_validation_request; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::analysis::models::SchemaRequestParts; +use sqlbuild_python_project::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use sqlbuild_python_project::semantic_shapes::main::declared_types::declared_types; +use sqlbuild_python_project::semantic_shapes::main::function_names::function_names; /// The column names a metadata config value lists: a name, or the names of a list or tuple. pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index cbbae49101..e639693b61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -13,15 +13,12 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::binding_validated; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::inputs::{ column_names, resource_sql_diagnostics, sql_test_columns, }; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::items; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_project::models::ModelMetadataTexts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -32,6 +29,9 @@ use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::model_config::models::AuditInstance; use sqlbuild_python_front::model_config::types::HeaderValue; use sqlbuild_python_front::model_config::types::PyConfigNode; +use sqlbuild_python_project::compiled_views::main::model_reads::binding_validated::binding_validated; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index fd8b1f3ed5..d94db0ea49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -13,9 +13,6 @@ use sqlbuild_analysis::semantic_checks::models::{ ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, }; -use crate::bindings::_helpers::compiled_views::audit_views::audit_reference_rows; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::models::ProjectCatalog; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -26,6 +23,9 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::compiled_views::main::audit_views::audit_reference_rows::audit_reference_rows; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs index 467c2e4eb5..011a26181c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs @@ -3,7 +3,7 @@ use sqlbuild_analysis::assembly::analysis_session::main::finished_model_facts::finished_model_facts; use sqlbuild_analysis::assembly::analysis_session::models::{ColumnFact, SessionModelFacts}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; /// The facts the finished `session` kept for model `name`, which the payload left out. pub(crate) fn session_model_facts<'a>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index 4db00e85c4..6f968d24fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -6,8 +6,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::model_reads::output_proven; use crate::bindings::_helpers::rules::early_builtin_rules::start_early_rules; use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; @@ -15,10 +13,12 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::{ StageShared, items, recovered_types, }; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::compiled_views::main::model_reads::output_proven::output_proven; const NO_CATALOG_MESSAGE: &str = "NativeCompilerError: native semantic completion: the project has no analysis catalog"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 4bfe9c7345..b5ca21d054 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::records::project_with; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -11,6 +10,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 40e167e5d6..907d18bf37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -16,11 +16,6 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_checks::types::RevisedBinding; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::analysis::binding_catalog::schema_validations; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::compiled_model_view::LineageInput; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_names, lineage_inputs}; -use crate::bindings::_helpers::compiled_views::records::with_type_recovery; use crate::bindings::_helpers::semantic_checks::constants::SCHEMA_REVALIDATIONS; use crate::bindings::_helpers::semantic_checks::failures::{internal_error, semantic_error}; use crate::bindings::_helpers::semantic_checks::inputs::revalidation_request; @@ -28,7 +23,6 @@ use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -36,6 +30,13 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::main::schema_validations::schema_validations; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::model_reads::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::records::with_type_recovery::with_type_recovery; +use sqlbuild_python_project::compiled_views::types::LineageInput; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; const UNCHANGED: &str = "unchanged"; const UNKNOWN_RECOVERED_TYPE: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs deleted file mode 100644 index d1ffa807bf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Stage-work names the semantic shape stage counts. - -/// Model queries analysed again against inspected source columns. -pub(crate) const SOURCE_REBINDING_ANALYSES: &str = "source_rebinding_analyses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs index 11ccbb1fdc..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs @@ -1,7 +1 @@ -pub(crate) mod constants; -pub(crate) mod coverage; -pub(crate) mod declarations; -pub(crate) mod delta_shapes; pub(crate) mod registration; -pub(crate) mod shapes; -pub(crate) mod source_bindings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs index 5116b8f3a8..76d860d2db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_shapes::{coverage, shapes, source_bindings}; +use sqlbuild_python_project::semantic_shapes::main::register_coverage::register_coverage; +use sqlbuild_python_project::semantic_shapes::main::register_shapes::register_shapes; +use sqlbuild_python_project::semantic_shapes::main::register_source_bindings::register_source_bindings; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - shapes::register(module)?; - coverage::register(module)?; - source_bindings::register(module) + register_shapes(module)?; + register_coverage(module)?; + register_source_bindings(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index e9ef355756..3a84bf7a30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -14,11 +14,6 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; -use crate::bindings::_helpers::graph::python_node_graph::{ - python_node_graph, run_selection_from_inputs, -}; -use crate::bindings::_helpers::graph::python_selection::python_node_names; -use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; use sqlbuild_python_base::build_progress::main::terminal::flush::flush; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::native_host::main::flush_console::flush_console; @@ -27,6 +22,10 @@ use sqlbuild_python_base::native_host::main::python_executable::python_executabl use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; +use sqlbuild_python_project::graph::main::python_selection::python_node_names::python_node_names; +use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; /// The prefix of a Python test module's file name. const TEST_MODULE_PREFIX: &str = "test_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index a721a2c9df..f887567e21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -40,7 +40,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; @@ -70,6 +69,8 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::compiled_project::types::VerifiedRun; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 477491e529..fd4313cdfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -12,7 +12,6 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -20,6 +19,7 @@ use sqlbuild_python_base::records::main::fields::path_posix_text::path_posix_tex use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; const MACRO_TOKEN: char = '@'; const INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index f06ce7b343..0fba97583d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -70,7 +70,11 @@ impl<'py> Classes<'py> { assembly: record_class(py, RecordId::NativeSqlTestAssembly)?, failure: record_class(py, RecordId::NativeSqlTestFailure)?, location: record_class(py, RecordId::SourceLocation)?, - code: PyString::new(py, super::constants::SQL_TEST_HELPER_REFERENCE_CODE).into_any(), + code: PyString::new( + py, + sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_HELPER_REFERENCE_CODE, + ) + .into_any(), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs deleted file mode 100644 index 7921a369a4..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! SQL-test plan resource and severity decisions. - -pub(super) const UDF: &str = "udf"; -pub(super) const TABLE_FN: &str = "table_fn"; -pub(super) const ERROR_SEVERITY: &str = "error"; -pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ - ("model", "__ref__"), - ("source", "__source__"), - ("seed", "__seed__"), - ("sql_test", "__expected__"), -]; -pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; -pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; -pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; -pub(crate) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; -pub(crate) const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; -pub(crate) const DEFAULT_INTEGER_START: &str = "-1000000000000000"; -pub(crate) const DEFAULT_INTEGER_END: &str = "1000000000000000"; -pub(super) const CURSOR_START_KEY: &str = "cursor_start"; -pub(super) const CURSOR_END_KEY: &str = "cursor_end"; -pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; -/// The diagnostic code of a SQL test that references a helper it may not read. -pub(crate) const SQL_TEST_HELPER_REFERENCE_CODE: &str = "P013"; -/// The scan store namespace of expanded SQL-test CTEs. -pub(crate) const SQL_TEST_CTE_SCAN_ALGORITHM: &str = "expanded-sql-test-ctes-v1"; -/// The scan store namespace of the models SQL tests expect. -pub(crate) const SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: &str = "sql-test-expected-models-v1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index ac3d60ae0a..a2b5a62391 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,17 +1,3 @@ pub(crate) mod assembly; pub(crate) mod compiled_tests; -pub(crate) mod constants; -mod cursor_window; -mod fixture_validation; -pub(crate) mod models; -pub(crate) mod plan_assembly; -pub(crate) mod plan_driver; -mod plan_entries; -mod plan_errors; -mod plan_fixtures; -mod plan_rendering; -mod plan_rows; -mod plan_values; -pub(crate) mod planning; pub(crate) mod registration; -mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs deleted file mode 100644 index 75520b0694..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Typed requests of SQL-test planning. - -use pyo3::prelude::{Bound, PyAny}; - -/// How one batch of SQL tests is planned. -#[derive(Debug)] -pub(crate) struct SqlTestPlanOptions<'py> { - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) render_sql: bool, - pub(crate) include_plan: bool, - pub(crate) artifact: bool, -} - -/// How assembling SQL-test plan results validates fixtures. -#[derive(Debug)] -pub(crate) struct SqlTestAssembly<'py> { - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) validate_fixtures: bool, - pub(crate) fixture_planning_context: Bound<'py, PyAny>, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 8c7176fe72..5d1dc7f379 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -1,28 +1,15 @@ //! Register the native SQL test planning and assembly functions. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; use crate::bindings::_helpers::sql_test_glue::compiled_tests; use sqlbuild_python_front::sql_test_glue::main::register_authored_ctes::register_authored_ctes; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - super::plan_assembly::plan_sql_tests, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::plan_sql_tests_natively, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::plan_compiled_sql_test_artifacts, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::sql_test_plan_error_messages, - module - )?)?; + module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_function::plan_sql_tests_function(module)?)?; + module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_natively_function::plan_sql_tests_natively_function(module)?)?; + module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_compiled_sql_test_artifacts_function::plan_compiled_sql_test_artifacts_function(module)?)?; + module.add_function(sqlbuild_python_project::sql_test_glue::main::sql_test_plan_error_messages_function::sql_test_plan_error_messages_function(module)?)?; register_authored_ctes(module)?; compiled_tests::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 51ec6fea5f..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,2 +1 @@ -pub(crate) mod nullability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index 5c16c08005..90771ac3c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,11 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::type_system::nullability; use sqlbuild_python_base::type_system::main::register_normalization::register_normalization; +use sqlbuild_python_project::type_system::main::register_nullability::register_nullability; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_normalization(module)?; - nullability::register(module)?; + register_nullability(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 2b824dfc40..7a6e4e8166 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::{ references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; -use crate::bindings::models; use sqlbuild_python_base::boundary::main::register_expected_errors::register_expected_errors; use sqlbuild_python_base::boundary::main::register_native_compiler_error::register_native_compiler_error; use sqlbuild_python_base::boundary::main::register_oracles::register_oracles; @@ -19,6 +18,9 @@ use sqlbuild_python_base::records::main::registry::capture::capture; use sqlbuild_python_base::records::main::registry::register_records::register_records; use sqlbuild_python_base::records::main::registry::verify_attributes::verify_attributes; use sqlbuild_python_base::records::main::registry::verify_tables::verify_tables; +use sqlbuild_python_project::analysis::models::BindingPositions; +use sqlbuild_python_project::analysis::models::CompactAnalysisJob; +use sqlbuild_python_project::analysis::models::ProjectCatalog; #[pymodule] pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -32,7 +34,9 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { native_host::registration::register(module)?; rules::registration::register(module)?; cli::registration::register(module)?; - analysis::normalization::register(module)?; + sqlbuild_python_project::analysis::main::register_normalization::register_normalization( + module, + )?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; @@ -50,9 +54,9 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { register_native_compiler_error(module)?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/mod.rs b/crates/sqlbuild-python/src/bindings/mod.rs index 072cc6b98b..668a17d0aa 100644 --- a/crates/sqlbuild-python/src/bindings/mod.rs +++ b/crates/sqlbuild-python/src/bindings/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod _helpers; pub mod main; -pub(crate) mod models; #[cfg(test)] mod tests; diff --git a/fensu.toml b/fensu.toml index c15a410de7..5ede471c66 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1449,11 +1449,11 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_ reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs"] reason = "Each builds one staying-Python object from its own keyword set." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs", "crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs"] reason = "Each calls a class from its own lookup, a native attribute or a host entry, with keyword pairs." [[dupes.allowlist]] @@ -1469,11 +1469,11 @@ paths = ["crates/sqlbuild-python-base/src/boundary/models.rs"] reason = "Each fills a missing keyword argument with its own default, None or False." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs"] reason = "Each compiled view reads its own field set from the Python boundary." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs"] reason = "Each callback or worker report forwards one distinct build or check event with its own payload." [[dupes.allowlist]] @@ -1676,7 +1676,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 169791 } +thresholds = { max_crate_code_lines = 147515 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -1816,7 +1816,7 @@ reason = "ProjectCatalog implements Debug manually because Polyglot SchemaValida [[targets.rust.rule_exceptions]] rule = "FPRSS201" -path = "crates/sqlbuild-python/src/bindings/models.rs" +path = "crates/sqlbuild-python-project/src/analysis/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." [[targets.rust.rule_exceptions]] @@ -1955,6 +1955,7 @@ crate_names = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2158,14 +2159,60 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=1", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=5", "crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", + "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=1", + "crates/sqlbuild-python-project/src/analysis/models.rs=1", + "crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs=3", + "crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs=1", + "crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs=1", + "crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs=4", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs=9", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs=7", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs=14", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs=1", + "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs=6", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs=15", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs=1", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=16", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs=1", + "crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs=2", + "crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs=5", + "crates/sqlbuild-python-project/src/graph/models.rs=2", + "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=3", + "crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs=19", + "crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs=1", + "crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs=2", + "crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs=2", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs=6", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs=3", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs=1", + "crates/sqlbuild-python-project/src/planner_selection/models.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs=2", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs=4", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs=3", + "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=19", + "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=5", + "crates/sqlbuild-python-project/src/references/_helpers/relations.rs=2", + "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs=2", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs=1", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs=3", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs=13", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs=8", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=1", @@ -2204,7 +2251,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", @@ -2247,23 +2293,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", @@ -2328,10 +2361,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", @@ -2345,13 +2374,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs=4", @@ -2390,7 +2412,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=11", @@ -2430,8 +2451,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", @@ -2464,7 +2483,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=2", @@ -2483,19 +2501,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", @@ -2532,11 +2542,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", @@ -2595,20 +2600,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", "crates/sqlbuild-python/src/bindings/models.rs=0", ] @@ -2682,9 +2676,34 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/scopes/models.rs=11", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", "crates/sqlbuild-python-front/src/state_rows/models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python-project/src/analysis/models.rs=9", + "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs=4", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs=5", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs=7", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs=24", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs=5", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs=26", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs=4", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/models.rs=44", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=12", + "crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs=4", + "crates/sqlbuild-python-project/src/graph/models.rs=10", + "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=2", + "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs=1", + "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs=6", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/models.rs=30", + "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=2", + "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=3", @@ -2728,17 +2747,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs=33", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", @@ -2759,7 +2767,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", @@ -2767,10 +2774,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", @@ -2791,7 +2794,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", @@ -2812,7 +2814,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=56", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", @@ -2824,12 +2825,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", @@ -2840,8 +2836,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=4", @@ -2874,9 +2868,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=4", "crates/sqlbuild-python/src/bindings/models.rs=0", ] @@ -2885,25 +2876,10 @@ layer_order = ["infra", "front", "project", "plan|compile", "output", "commands" module_layers = [ "*/registration=registry", "analysis=compile", - "analysis/binding_catalog=project", - "analysis/conversions=project", - "analysis/normalization=project", - "analysis/normalization_results=project", - "analysis/relation_names=project", - "analysis/schema_request=project", "analysis_session=compile", - "analysis_session/fixture_columns=project", - "analysis_session/fixture_inferences=project", - "analysis_session/models=project", - "analysis_session/nullability_rules=project", - "analysis_session/profile_inputs=project", - "analysis_session/query_columns=project", - "analysis_session/session=project", - "analysis_session/stage_functions=project", "attachments=registry", "audit_command=commands", "auditing=compile", - "auditing/render=project", "authored_values=registry", "build_command=commands", "build_progress=output", @@ -2914,7 +2890,6 @@ module_layers = [ "clone=commands", "command_runtime=commands", "compile_attachment=compile", - "compile_attachment/cursor_roles=project", "compile_command=compile", "compile_delta=compile", "compile_driver=compile", @@ -2922,12 +2897,12 @@ module_layers = [ "compile_driver/plan_result=commands", "compile_driver/static_context=commands", "compile_outputs=compile", - "compiled_project=project", - "compiled_views=project", + "compiled_project=registry", + "compiled_views=registry", "compute_logs=output", "contract_command=commands", "consumption=plan", - "contracts=project", + "contracts=registry", "cost_command=commands", "cost_report=output", "cost_telemetry=output", @@ -2946,12 +2921,12 @@ module_layers = [ "event_exporting=output", "execution_history=front", "exported_constants=registry", - "fingerprints=project", + "fingerprints=registry", "format_command=commands", "freshness=plan", "freshness/freshness_command=commands", "frontier=compile", - "graph=project", + "graph=registry", "hand_built=commands", "hand_built/compiled_model=compile", "init_command=commands", @@ -2960,7 +2935,6 @@ module_layers = [ "lifecycle_events=plan", "lineage=compile", "lineage/lineage_command=commands", - "lineage/lineage_facts=project", "load=commands", "location_command=commands", "macro_bridge=registry", @@ -2968,9 +2942,7 @@ module_layers = [ "microbatches=plan", "model_config=registry", "native_host=front", - "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", - "native_host/orchestration_triggers=project", "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "observations=plan", @@ -2980,31 +2952,16 @@ module_layers = [ "planner_changes=plan", "planner_entries=plan", "planner_identity=plan", - "planner_identity/graph=project", - "planner_identity/hashing=project", "planner_migrations=plan", "planner_phases=plan", "planner_phases/models=project", - "planner_phases/objects=project", "planner_phases/scopes=project", "planner_registration=registry", "planner_resolution=plan", "planner_scenario=plan", - "planner_scope=project", - "planner_selection=project", + "planner_scope=registry", + "planner_selection=registry", "planner_strategy=plan", - "planner_strategy/constants=project", - "planner_strategy/fixture_completion=project", - "planner_strategy/fixture_context=project", - "planner_strategy/fixture_metadata=project", - "planner_strategy/fixture_preparation=project", - "planner_strategy/fixture_reads=project", - "planner_strategy/fixture_selection=project", - "planner_strategy/fixture_sql=project", - "planner_strategy/fixture_types=project", - "planner_strategy/fixture_values=project", - "planner_strategy/models=project", - "planner_strategy/types=project", "planner_warehouse=plan", "presentation_text=registry", "project_assembly=compile", @@ -3016,8 +2973,8 @@ module_layers = [ "refactor=front", "refactor/offline_compile=commands", "refactor/refactor_command=commands", - "references=project", - "relation_naming=project", + "references=registry", + "relation_naming=registry", "render=registry", "result_output=output", "review_command=commands", @@ -3028,25 +2985,12 @@ module_layers = [ "scopes/scope_loading=compile", "seed_command=commands", "semantic_checks=compile", - "semantic_shapes=project", + "semantic_shapes=registry", "sql_test_command=commands", "sql_test_glue=compile", - "sql_test_glue/constants=infra", - "sql_test_glue/cursor_window=project", - "sql_test_glue/fixture_validation=project", - "sql_test_glue/models=project", - "sql_test_glue/plan_assembly=project", - "sql_test_glue/plan_driver=project", - "sql_test_glue/plan_entries=project", - "sql_test_glue/plan_errors=project", - "sql_test_glue/plan_fixtures=project", - "sql_test_glue/plan_rendering=project", - "sql_test_glue/plan_rows=project", - "sql_test_glue/plan_values=project", - "sql_test_glue/planning=project", "sqltext=registry", "state_rows=registry", - "type_system=project", + "type_system=registry", "workspace_commands=commands", ] pending_budgets = [ @@ -3088,7 +3032,7 @@ paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "c reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3168,7 +3112,7 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 77f5de4f70..3c7b02624c 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -287,6 +287,7 @@ "crates/sqlbuild-python/src/", "crates/sqlbuild-python-base/src/", "crates/sqlbuild-python-front/src/", + "crates/sqlbuild-python-project/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 2c5ea422f0..904f3f4ce6 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -5,6 +5,7 @@ SCANNED_SOURCE_ROOTS: tuple[str, ...] = ( "crates/sqlbuild-python-base/src", "crates/sqlbuild-python-front/src", + "crates/sqlbuild-python-project/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", From 34827b8f5dfa594ef465da4565f46e4e2c750eaa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:59:27 +0000 Subject: [PATCH 04/10] wip: lay out the project crate's areas and pass fensu check Groups the moved helper modules into buckets, splits the snapshot and resource views into their own areas, keeps the native_host edge, and records Debug and record-tuple exceptions. fensu check passes; new duplicate clusters are not allowlisted yet. Not tested. --- crates/README.md | 7 +- .../src/analysis/_helpers/native_classes.rs | 22 - .../src/analysis/_helpers/normalization.rs | 2 +- .../analysis/main/normalize_analysis_sql.rs | 2 +- .../src/analysis/models.rs | 21 + .../src/analysis_session/_helpers/session.rs | 17 +- .../src/analysis_session/models.rs | 33 + .../_helpers/cursor_roles.rs | 2 +- .../_helpers/attachment_inputs.rs | 6 +- .../src/compiled_project/_helpers/mod.rs | 8 - .../src/compiled_project/_helpers/project.rs | 39 +- .../src/compiled_project/constants.rs | 8 +- .../main/attachment_inputs/mod.rs | 1 - .../main/{project => }/config_values.rs | 0 .../main/{project => }/declarations.rs | 0 .../{schema_facts => }/entry_schema_fact.rs | 0 .../main/{schema_facts => }/family_fact.rs | 0 .../main/{job_catalog => }/graph_with_jobs.rs | 0 .../main/{attachment_inputs => }/install.rs | 0 .../compiled_project/main/job_catalog/mod.rs | 5 - .../main/{job_catalog => }/job_names.rs | 0 .../main/{project => }/materialization.rs | 0 .../src/compiled_project/main/mod.rs | 30 +- .../main/{project => }/model_facts.rs | 2 +- .../main/{job_catalog => }/needed_jobs.rs | 0 .../main/{values => }/plain_value.rs | 0 .../src/compiled_project/main/project/mod.rs | 6 - .../{schema_facts => }/python_column_fact.rs | 0 .../{schema_facts => }/python_column_facts.rs | 0 .../main/{rows => }/reference_row_tuple.rs | 0 .../main/{project => }/register_project.rs | 0 .../src/compiled_project/main/rows/mod.rs | 1 - .../compiled_project/main/schema_facts/mod.rs | 4 - .../src/compiled_project/main/snapshot/mod.rs | 1 - .../main/snapshot_boundary/mod.rs | 1 - .../main/snapshot_declarations/mod.rs | 2 - .../main/snapshot_models/mod.rs | 1 - .../main/snapshot_resources/mod.rs | 1 - .../main/snapshot_roles/mod.rs | 1 - .../main/{project => }/text_value.rs | 0 .../main/{job_catalog => }/triggered_jobs.rs | 0 .../src/compiled_project/main/values/mod.rs | 1 - .../main/{job_catalog => }/with_job_scope.rs | 0 .../src/compiled_project/mod.rs | 2 +- .../src/compiled_project/models.rs | 1027 +---------------- .../src/compiled_project/types.rs | 9 - .../compiled_model_analysis.rs | 19 +- .../compiled_model_view.rs | 24 +- .../_helpers/compiled_models/mod.rs | 4 + .../{ => compiled_models}/model_reads.rs | 0 .../{ => compiled_models}/relation_keys.rs | 10 +- .../src/compiled_views/_helpers/mod.rs | 23 +- .../{ => model_inputs}/inputs_view.rs | 11 +- .../_helpers/model_inputs/mod.rs | 2 + .../{ => model_inputs}/model_input_view.rs | 20 +- .../_helpers/{ => project}/getters.rs | 16 +- .../compiled_views/_helpers/project/mod.rs | 5 + .../_helpers/{ => project}/project_host.rs | 48 +- .../_helpers/{ => project}/project_reads.rs | 0 .../_helpers/{ => project}/project_view.rs | 11 +- .../_helpers/{ => project}/records.rs | 4 +- .../src/compiled_views/constants.rs | 40 +- .../main/compiled_models/audit_key_rows.rs | 12 + .../binding_validated.rs | 2 +- .../config.rs | 2 +- .../declared_rows.rs | 2 +- .../compiled_models/defaulted_destination.rs | 17 + .../dep_rows.rs | 2 +- .../{model_reads => compiled_models}/deps.rs | 2 +- .../destination.rs | 2 +- .../dynamic_columns_declared.rs | 2 +- .../entry_relation_key_row.rs | 2 +- .../enum_columns.rs | 2 +- .../inferred_columns.rs | 2 +- .../inferred_names.rs | 2 +- .../inferred_rows.rs | 2 +- .../lineage_facts.rs | 2 +- .../lineage_inputs.rs | 2 +- .../{model_reads => compiled_models}/mod.rs | 12 + .../main/compiled_models/model_graph_row.rs | 13 + .../model_key.rs | 2 +- .../model_key_row.rs | 2 +- .../model_name.rs | 2 +- .../name_object.rs | 2 +- .../main/compiled_models/object_key.rs | 17 + .../one_row_seed_names.rs | 2 +- .../output_proven.rs | 2 +- .../project_relation_keys.rs | 2 +- .../proven_inputs.rs | 2 +- .../query_sql.rs | 2 +- .../references.rs | 2 +- .../register_relation_keys.rs | 2 +- .../relation_keys.rs | 2 +- .../main/compiled_models/relation_location.rs | 17 + .../relative_path.rs | 2 +- .../schema_entry.rs | 2 +- .../source_key_input.rs | 2 +- .../star_flags.rs | 2 +- .../unchecked_columns.rs | 2 +- .../src/compiled_views/main/mod.rs | 17 +- .../attached_columns.rs | 2 +- .../attached_schema.rs | 2 +- .../input_analysis_sql.rs | 2 +- .../input_model_name.rs | 2 +- .../input_reference_rows.rs | 2 +- .../input_sql_macro_checks.rs | 2 +- .../{model_input_view => model_inputs}/mod.rs | 2 + .../main/model_inputs/reference_tuple.rs | 19 + .../main/model_inputs/view_classes.rs | 17 + .../main/{project_reads => project}/audits.rs | 2 +- .../{project_reads => project}/collection.rs | 2 +- .../compile_inputs_view.rs | 2 +- .../compiled_audit_record.rs | 2 +- .../compiled_function_record.rs | 2 +- .../compiled_model_record.rs | 2 +- .../models.rs => project/compiled_models.rs} | 4 +- .../compiled_project_facts.rs | 2 +- .../compiled_seed_record.rs | 2 +- .../compiled_source_record.rs | 2 +- .../{records => project}/declaration_rows.rs | 2 +- .../declaration_rows_function.rs | 2 +- .../{project_reads => project}/functions.rs | 2 +- .../main/{records => project}/inputs_with.rs | 2 +- .../main/{records => project}/mod.rs | 7 + .../{records => project}/plain_typed_value.rs | 2 +- .../main/{records => project}/project_with.rs | 2 +- .../record_model_analyses.rs | 2 +- .../{records => project}/register_records.rs | 2 +- .../main/{records => project}/replace.rs | 2 +- .../main/{project_reads => project}/seeds.rs | 2 +- .../{project_reads => project}/sources.rs | 2 +- .../{project_reads => project}/sql_tests.rs | 2 +- .../with_binding_diagnostics.rs | 2 +- .../with_type_recovery.rs | 2 +- .../compiled_views/main/project_reads/mod.rs | 7 - .../compiled_views/main/relation_keys/mod.rs | 7 - .../src/compiled_views/mod.rs | 2 +- .../src/compiled_views/models.rs | 728 ++---------- .../src/compiled_views/types.rs | 49 - .../_helpers/contract_diagnostics.rs | 4 +- .../src/graph/_helpers/mod.rs | 12 +- .../graph/_helpers/{ => nodes}/algorithms.rs | 0 .../_helpers/{ => nodes}/execution_gates.rs | 0 .../src/graph/_helpers/{ => nodes}/keys.rs | 0 .../src/graph/_helpers/nodes/mod.rs | 3 + .../graph/_helpers/{ => project}/getters.rs | 0 .../src/graph/_helpers/project/mod.rs | 3 + .../_helpers/{ => project}/project_graph.rs | 0 .../src/graph/_helpers/{ => project}/views.rs | 2 +- .../src/graph/_helpers/python_nodes/mod.rs | 3 + .../{ => python_nodes}/python_lifecycle.rs | 0 .../{ => python_nodes}/python_node_graph.rs | 4 +- .../{ => python_nodes}/python_selection.rs | 2 +- .../src/graph/constants.rs | 2 +- .../src/graph/main/algorithms/mod.rs | 4 - .../src/graph/main/execution_gates/mod.rs | 3 - .../src/graph/main/keys/mod.rs | 1 - .../src/graph/main/mod.rs | 11 +- .../main/{algorithms => nodes}/closure.rs | 2 +- .../main/{algorithms => nodes}/inverted.rs | 2 +- .../src/graph/main/nodes/mod.rs | 8 + .../model_python_hook_names.rs | 2 +- .../graph/main/{keys => nodes}/native_key.rs | 2 +- .../register_algorithms.rs | 2 +- .../register_execution_gates.rs | 2 +- .../{execution_gates => nodes}/sql_ref_key.rs | 2 +- .../transitive_closure_many.rs | 2 +- .../{views => project}/compiled_graph_keys.rs | 2 +- .../main/{views => project}/edge_dict.rs | 2 +- .../graph_from_view_mappings.rs | 2 +- .../src/graph/main/{views => project}/mod.rs | 2 + .../native_graph_from_edges.rs | 2 +- .../project_graph.rs | 2 +- .../project_lineage_views.rs | 2 +- .../register_project_graph.rs | 2 +- .../main/{views => project}/register_views.rs | 2 +- .../src/graph/main/project_graph/mod.rs | 2 - .../src/graph/main/python_lifecycle/mod.rs | 2 - .../src/graph/main/python_node_graph/mod.rs | 3 - .../build_python_sql_run_lifecycle.rs | 2 +- .../src/graph/main/python_nodes/mod.rs | 8 + .../python_node_graph.rs | 2 +- .../python_node_names.rs | 2 +- .../register_python_lifecycle.rs | 2 +- .../register_python_node_graph.rs | 2 +- .../register_python_selection.rs | 2 +- .../resolve_python_check_excludes.rs | 2 +- .../run_selection_from_inputs.rs | 2 +- .../src/graph/main/python_selection/mod.rs | 3 - .../src/graph/models.rs | 20 +- crates/sqlbuild-python-project/src/lib.rs | 2 + .../src/planner_identity/_helpers/graph.rs | 22 +- .../src/planner_phases/_helpers/objects.rs | 4 +- .../src/planner_scope/_helpers/loaders.rs | 2 +- .../src/planner_scope/_helpers/retained.rs | 2 +- .../src/planner_scope/_helpers/scope.rs | 11 +- .../src/planner_scope/_helpers/sources.rs | 8 +- .../src/planner_scope/_helpers/values.rs | 6 +- .../src/planner_scope/models.rs | 2 +- .../_helpers/model_selectors.rs | 10 +- .../planner_selection/_helpers/selectors.rs | 6 +- .../planner_selection/_helpers/unit_tests.rs | 2 +- .../{ => fixtures}/fixture_completion.rs | 35 +- .../{ => fixtures}/fixture_context.rs | 29 +- .../{ => fixtures}/fixture_metadata.rs | 10 +- .../{ => fixtures}/fixture_preparation.rs | 6 +- .../_helpers/{ => fixtures}/fixture_reads.rs | 15 +- .../{ => fixtures}/fixture_selection.rs | 8 +- .../_helpers/{ => fixtures}/fixture_sql.rs | 0 .../_helpers/{ => fixtures}/fixture_types.rs | 2 +- .../_helpers/{ => fixtures}/fixture_values.rs | 0 .../planner_strategy/_helpers/fixtures/mod.rs | 9 + .../src/planner_strategy/_helpers/mod.rs | 11 +- .../src/planner_strategy/_helpers/types.rs | 4 - .../main/complete_empty_fixture_sql.rs | 2 +- .../main/complete_fixtures.rs | 4 +- .../main/complete_typed_null_columns.rs | 2 +- .../main/count_chain_resolution.rs | 2 +- .../planner_strategy/main/planning_context.rs | 2 +- .../main/register_fixture_completion.rs | 2 +- .../main/register_fixture_context.rs | 2 +- .../main/register_fixture_reads.rs | 2 +- .../src/planner_strategy/models.rs | 82 +- .../src/planner_strategy/types.rs | 3 + .../src/project_snapshot/_helpers/mod.rs | 1 + .../project_snapshot/_helpers/snapshot/mod.rs | 8 + .../_helpers/snapshot}/snapshot.rs | 4 +- .../_helpers/snapshot}/snapshot_boundary.rs | 10 +- .../snapshot}/snapshot_declarations.rs | 0 .../_helpers/snapshot}/snapshot_execution.rs | 0 .../_helpers/snapshot}/snapshot_models.rs | 6 +- .../_helpers/snapshot}/snapshot_resources.rs | 18 +- .../_helpers/snapshot}/snapshot_roles.rs | 0 .../_helpers/snapshot}/snapshot_worker.rs | 0 .../main}/compile_declarations.rs | 2 +- .../main}/compile_project_roles.rs | 2 +- .../src/project_snapshot/main/mod.rs | 7 + .../main}/model_nodes.rs | 2 +- .../main}/python_declarations.rs | 2 +- .../main}/register_snapshot.rs | 2 +- .../main}/relative_path.rs | 2 +- .../main}/resource_nodes.rs | 2 +- .../src/project_snapshot/mod.rs | 6 + .../src/project_snapshot/models.rs | 991 ++++++++++++++++ .../src/project_snapshot/types.rs | 13 + .../_helpers/explicit_references.rs | 2 +- .../_helpers/audit_reads.rs | 2 +- .../_helpers/audit_views.rs | 23 +- .../_helpers/function_reads.rs | 2 +- .../_helpers/function_views.rs | 29 +- .../src/resource_views/_helpers/mod.rs | 9 + .../_helpers/resource_reads.rs | 10 +- .../_helpers/seed_reads.rs | 2 +- .../_helpers/seed_views.rs | 20 +- .../_helpers/source_reads.rs | 2 +- .../_helpers/source_views.rs | 31 +- .../main/audit_reads/always_run.rs | 2 +- .../main/audit_reads/attached_column_name.rs | 2 +- .../main/audit_reads/attached_target_kind.rs | 2 +- .../main/audit_reads/attached_target_name.rs | 2 +- .../main/audit_reads/definition_name.rs | 2 +- .../main/audit_reads/description.rs | 2 +- .../main/audit_reads/evaluation_mode.rs | 2 +- .../main/audit_reads/evidence_limit.rs | 2 +- .../main/audit_reads/evidence_sql.rs | 2 +- .../main/audit_reads/key.rs | 2 +- .../main/audit_reads/measurement_contract.rs | 2 +- .../main/audit_reads/meta.rs | 2 +- .../main/audit_reads/minimum_samples.rs | 2 +- .../main/audit_reads/mod.rs | 0 .../main/audit_reads/name.rs | 2 +- .../main/audit_reads/references.rs | 2 +- .../main/audit_reads/run_scope.rs | 2 +- .../main/audit_reads/scope_deps.rs | 2 +- .../main/audit_reads/severity.rs | 2 +- .../main/audit_reads/sql_body.rs | 2 +- .../main/audit_reads/thresholds.rs | 2 +- .../main/audit_views/audit_record.rs | 6 +- .../main/audit_views/audit_reference_rows.rs | 2 +- .../main/audit_views/mod.rs | 0 .../main/function_reads/arguments.rs | 2 +- .../main/function_reads/body_sql.rs | 2 +- .../main/function_reads/deps.rs | 2 +- .../main/function_reads/destination.rs | 2 +- .../main/function_reads/entry_point.rs | 2 +- .../function_reads/fingerprint_destination.rs | 2 +- .../main/function_reads/key.rs | 2 +- .../main/function_reads/language.rs | 2 +- .../main/function_reads/mod.rs | 0 .../main/function_reads/name.rs | 2 +- .../main/function_reads/packages.rs | 2 +- .../main/function_reads/relative_path.rs | 2 +- .../main/function_reads/return_columns.rs | 2 +- .../main/function_reads/returns.rs | 2 +- .../main/function_reads/runtime_version.rs | 2 +- .../main/function_reads/source_file_path.rs | 2 +- .../main/function_views/function_record.rs | 6 +- .../main/function_views/mod.rs | 0 .../src/resource_views/main/mod.rs | 9 + .../main/resource_reads/destination.rs | 2 +- .../main/resource_reads/graph_row.rs | 2 +- .../main/resource_reads/key.rs | 2 +- .../main/resource_reads/mod.rs | 0 .../main/resource_reads/name.rs | 2 +- .../main/resource_reads/references.rs | 2 +- .../main/resource_reads/schema_entry.rs | 2 +- .../main/seed_reads/destination.rs | 2 +- .../main/seed_reads/external.rs | 2 +- .../main/seed_reads/key.rs | 2 +- .../main/seed_reads/mod.rs | 0 .../main/seed_reads/name.rs | 2 +- .../main/seed_reads/schema_entry.rs | 2 +- .../main/seed_reads/seed_file.rs | 2 +- .../main/seed_views/mod.rs | 0 .../main/seed_views/paired_seeds.rs | 4 +- .../main/source_reads/key.rs | 2 +- .../main/source_reads/mod.rs | 0 .../main/source_reads/name.rs | 2 +- .../main/source_reads/source_entry.rs | 2 +- .../main/source_reads/source_file.rs | 2 +- .../main/source_views/mod.rs | 0 .../main/source_views/read_source_view.rs | 4 +- .../main/source_views/replaced.rs | 2 +- .../main/source_views/retain_sources.rs | 6 +- .../src/resource_views/mod.rs | 6 + .../src/resource_views/models.rs | 653 +++++++++++ .../src/resource_views/types.rs | 53 + .../src/semantic_shapes/_helpers/coverage.rs | 4 +- .../src/semantic_shapes/_helpers/shapes.rs | 10 +- .../_helpers/{ => inputs}/cursor_window.rs | 4 +- .../{ => inputs}/fixture_validation.rs | 0 .../src/sql_test_glue/_helpers/inputs/mod.rs | 2 + .../src/sql_test_glue/_helpers/mod.rs | 14 +- .../src/sql_test_glue/_helpers/plan/mod.rs | 9 + .../_helpers/{ => plan}/plan_assembly.rs | 10 +- .../_helpers/{ => plan}/plan_driver.rs | 35 +- .../_helpers/{ => plan}/plan_entries.rs | 13 +- .../_helpers/{ => plan}/plan_errors.rs | 2 +- .../_helpers/{ => plan}/plan_fixtures.rs | 8 +- .../_helpers/{ => plan}/plan_rendering.rs | 16 +- .../_helpers/{ => plan}/plan_rows.rs | 0 .../_helpers/{ => plan}/plan_values.rs | 0 .../_helpers/{ => plan}/planning.rs | 4 +- .../main/assemble_sql_test_plans.rs | 2 +- ...an_compiled_sql_test_artifacts_function.rs | 2 +- .../main/plan_sql_tests_function.rs | 2 +- .../main/plan_sql_tests_natively_function.rs | 2 +- .../src/sql_test_glue/main/planned_rows.rs | 2 +- .../main/sql_test_model_chain_names_by_key.rs | 2 +- ..._test_model_chain_names_by_key_function.rs | 2 +- .../sql_test_plan_error_messages_function.rs | 2 +- .../src/sql_test_glue/mod.rs | 1 + .../src/sql_test_glue/{_helpers => }/types.rs | 5 +- .../src/bindings/_helpers/analysis/queries.rs | 8 +- .../analysis_session/model_analyses.rs | 10 +- .../_helpers/audit_command/command.rs | 4 +- .../_helpers/auditing/registration.rs | 3 +- .../_helpers/build_command/command.rs | 4 +- .../_helpers/build_command/execution.rs | 2 +- .../_helpers/build_command/invocation.rs | 2 +- .../_helpers/build_command/lifecycle.rs | 4 +- .../build_progress/build_callbacks.rs | 2 +- .../change_selection/definition_snapshot.rs | 36 +- .../_helpers/change_selection/selection.rs | 14 +- .../bindings/_helpers/check/check_command.rs | 10 +- .../src/bindings/_helpers/check/check_core.rs | 2 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../_helpers/compile_driver/assembly.rs | 4 +- .../compile_driver/assembly_analysis.rs | 4 +- .../compile_driver/assembly_models.rs | 2 +- .../bindings/_helpers/compile_driver/diff.rs | 6 +- .../_helpers/compile_driver/inputs.rs | 4 +- .../_helpers/compile_driver/phases.rs | 2 +- .../_helpers/compile_driver/plan_result.rs | 13 +- .../_helpers/compile_driver/project.rs | 2 +- .../compile_driver/python_plan_entries.rs | 4 +- .../_helpers/compile_driver/static_context.rs | 2 +- .../compile_driver/target_defaults.rs | 6 +- .../_helpers/compiled_project/registration.rs | 10 +- .../_helpers/compiled_views/registration.rs | 5 +- .../bindings/_helpers/consumption/facts.rs | 38 +- .../bindings/_helpers/consumption/policy.rs | 2 +- .../_helpers/freshness/freshness_command.rs | 2 +- .../bindings/_helpers/graph/registration.rs | 23 +- .../_helpers/input_loops/audit_loop.rs | 4 +- .../_helpers/input_loops/function_loop.rs | 4 +- .../_helpers/input_loops/loop_classes.rs | 4 +- .../_helpers/input_loops/model_loop.rs | 10 +- .../_helpers/input_loops/model_loop_header.rs | 2 +- .../_helpers/input_loops/model_loop_host.rs | 8 +- .../_helpers/input_loops/model_loop_schema.rs | 2 +- .../_helpers/input_loops/registration.rs | 2 +- .../input_loops/resource_attachment.rs | 14 +- .../input_loops/resource_diagnostics.rs | 8 +- .../_helpers/input_loops/source_loop.rs | 6 +- .../_helpers/input_loops/sql_test_loop.rs | 2 +- .../_helpers/janitor/janitor_old_names.rs | 4 +- .../_helpers/lineage/project_lineage.rs | 2 +- .../_helpers/lineage/project_schema.rs | 4 +- .../_helpers/location_command/command.rs | 4 +- .../_helpers/location_command/snapshot.rs | 12 +- .../location_command/snapshot_providers.rs | 2 +- .../_helpers/location_command/worker.rs | 10 +- .../src/bindings/_helpers/manifest/builder.rs | 2 +- .../native_host/orchestration_sensors.rs | 2 +- .../native_host/warehouse_metadata.rs | 2 +- .../_helpers/observations/publication.rs | 20 +- .../_helpers/planner_changes/detect.rs | 27 +- .../_helpers/planner_changes/renames.rs | 8 +- .../_helpers/planner_entries/clone_plans.rs | 95 +- .../_helpers/planner_entries/command_plans.rs | 42 +- .../_helpers/planner_entries/display_plan.rs | 16 +- .../_helpers/planner_entries/model_entries.rs | 6 +- .../_helpers/planner_entries/plan_output.rs | 53 +- .../_helpers/planner_entries/relations.rs | 14 +- .../_helpers/planner_identity/identities.rs | 18 +- .../_helpers/planner_identity/metadata.rs | 42 +- .../planner_migrations/column_candidates.rs | 6 +- .../planner_migrations/column_entries.rs | 10 +- .../planner_migrations/column_objects.rs | 4 +- .../planner_migrations/column_overlay.rs | 6 +- .../planner_migrations/column_planning.rs | 4 +- .../planner_migrations/compatibility.rs | 4 +- .../_helpers/planner_migrations/discovery.rs | 2 +- .../planner_migrations/discovery_events.rs | 7 +- .../planner_migrations/discovery_models.rs | 17 +- .../planner_migrations/discovery_requests.rs | 6 +- .../planner_migrations/fingerprint.rs | 2 +- .../_helpers/planner_migrations/matching.rs | 6 +- .../planner_migrations/model_entries.rs | 4 +- .../planner_migrations/model_handover.rs | 6 +- .../planner_migrations/model_planning.rs | 8 +- .../planner_migrations/old_name_entries.rs | 4 +- .../planner_migrations/old_name_objects.rs | 4 +- .../planner_migrations/old_name_planning.rs | 14 +- .../_helpers/planner_migrations/old_names.rs | 20 +- .../planner_migrations/planning_origin.rs | 26 +- .../_helpers/planner_phases/buildability.rs | 8 +- .../_helpers/planner_phases/full_refresh.rs | 8 +- .../_helpers/planner_phases/identities.rs | 4 +- .../_helpers/planner_phases/retention.rs | 36 +- .../planner_phases/warehouse_state.rs | 2 +- .../planner_resolution/cursor_model.rs | 8 +- .../planner_resolution/cursor_overrides.rs | 2 +- .../planner_resolution/cursor_ranges.rs | 2 +- .../_helpers/planner_resolution/locations.rs | 4 +- .../planner_resolution/orchestration.rs | 8 +- .../bindings/_helpers/planner_scenario/cli.rs | 10 +- .../_helpers/planner_scenario/entries.rs | 10 +- .../_helpers/planner_scenario/execution.rs | 33 +- .../_helpers/planner_scenario/graph.rs | 16 +- .../_helpers/planner_scenario/relations.rs | 11 +- .../planner_strategy/audit_entries.rs | 44 +- .../_helpers/planner_strategy/audits.rs | 12 +- .../planner_strategy/cursor_inputs.rs | 4 +- .../_helpers/planner_strategy/lineage.rs | 6 +- .../planner_strategy/microbatch_limits.rs | 2 +- .../planner_strategy/model_entries.rs | 28 +- .../_helpers/planner_strategy/registration.rs | 6 +- .../_helpers/planner_strategy/strategy.rs | 7 +- .../_helpers/planner_warehouse/cursors.rs | 23 +- .../planner_warehouse/semantic_sources.rs | 8 +- .../_helpers/planner_warehouse/snapshot.rs | 51 +- .../planner_warehouse/source_columns.rs | 4 +- .../planner_warehouse/source_deferral.rs | 31 +- .../planner_warehouse/source_freshness.rs | 6 +- .../planner_warehouse/source_tables.rs | 22 +- .../_helpers/planner_warehouse/watermarks.rs | 8 +- .../project_assembly/declared_relations.rs | 12 +- .../project_assembly/native_resources.rs | 2 +- .../project_assembly/syntax_checks.rs | 2 +- .../bindings/_helpers/query_command/dag.rs | 2 +- .../src/bindings/_helpers/recall/session.rs | 2 +- .../_helpers/record_behaviours/compiled.rs | 2 +- .../_helpers/record_behaviours/macros.rs | 2 +- .../_helpers/references/registration.rs | 8 +- .../_helpers/review_command/comparison.rs | 6 +- .../_helpers/review_command/impact.rs | 10 +- .../_helpers/review_command/planning.rs | 24 +- .../_helpers/rules/custom_host_payload.rs | 10 +- .../_helpers/rules/custom_rule_cache.rs | 2 +- .../_helpers/rules/early_builtin_rules.rs | 4 +- .../bindings/_helpers/rules/early_sql_lint.rs | 8 +- .../bindings/_helpers/rules/fact_replay.rs | 8 +- .../bindings/_helpers/rules/fix_session.rs | 2 +- .../src/bindings/_helpers/rules/lint.rs | 2 +- .../src/bindings/_helpers/rules/rows.rs | 8 +- .../src/bindings/_helpers/rules/rule_plan.rs | 2 +- .../src/bindings/_helpers/rules/sql_rules.rs | 4 +- .../_helpers/rules/sql_test_objects.rs | 4 +- .../bindings/_helpers/seed_command/command.rs | 4 +- .../_helpers/semantic_checks/completion.rs | 6 +- .../_helpers/semantic_checks/metadata.rs | 2 +- .../_helpers/semantic_checks/resource_sql.rs | 2 +- .../_helpers/semantic_checks/stage.rs | 2 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../_helpers/semantic_checks/type_recovery.rs | 6 +- .../_helpers/sql_test_command/python_tests.rs | 6 +- .../_helpers/sql_test_command/test_command.rs | 4 +- .../_helpers/sql_test_glue/registration.rs | 12 +- .../src/bindings/main/register.rs | 5 +- fensu.toml | 141 ++- tests/native_field_fallbacks.txt | 56 +- 503 files changed, 3516 insertions(+), 3366 deletions(-) delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/config_values.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/declarations.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{schema_facts => }/entry_schema_fact.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{schema_facts => }/family_fact.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{job_catalog => }/graph_with_jobs.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{attachment_inputs => }/install.rs (100%) delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs rename crates/sqlbuild-python-project/src/compiled_project/main/{job_catalog => }/job_names.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/materialization.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/model_facts.rs (83%) rename crates/sqlbuild-python-project/src/compiled_project/main/{job_catalog => }/needed_jobs.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{values => }/plain_value.rs (100%) delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs rename crates/sqlbuild-python-project/src/compiled_project/main/{schema_facts => }/python_column_fact.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{schema_facts => }/python_column_facts.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{rows => }/reference_row_tuple.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/register_project.rs (100%) delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs rename crates/sqlbuild-python-project/src/compiled_project/main/{project => }/text_value.rs (100%) rename crates/sqlbuild-python-project/src/compiled_project/main/{job_catalog => }/triggered_jobs.rs (100%) delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs rename crates/sqlbuild-python-project/src/compiled_project/main/{job_catalog => }/with_job_scope.rs (100%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => compiled_models}/compiled_model_analysis.rs (97%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => compiled_models}/compiled_model_view.rs (93%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => compiled_models}/model_reads.rs (100%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => compiled_models}/relation_keys.rs (97%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => model_inputs}/inputs_view.rs (97%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => model_inputs}/model_input_view.rs (94%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => project}/getters.rs (98%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => project}/project_host.rs (53%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => project}/project_reads.rs (100%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => project}/project_view.rs (98%) rename crates/sqlbuild-python-project/src/compiled_views/_helpers/{ => project}/records.rs (99%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/binding_validated.rs (83%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/config.rs (80%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/declared_rows.rs (87%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/dep_rows.rs (81%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/deps.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/destination.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/dynamic_columns_declared.rs (84%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/entry_relation_key_row.rs (89%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/enum_columns.rs (81%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/inferred_columns.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/inferred_names.rs (84%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/inferred_rows.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/lineage_facts.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/lineage_inputs.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/mod.rs (62%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/model_key.rs (80%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/model_key_row.rs (84%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/model_name.rs (78%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/name_object.rs (81%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/one_row_seed_names.rs (85%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/output_proven.rs (83%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/project_relation_keys.rs (85%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/proven_inputs.rs (85%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/query_sql.rs (80%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/references.rs (81%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/register_relation_keys.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/relation_keys.rs (90%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/relative_path.rs (81%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/schema_entry.rs (83%) rename crates/sqlbuild-python-project/src/compiled_views/main/{relation_keys => compiled_models}/source_key_input.rs (85%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/star_flags.rs (83%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_reads => compiled_models}/unchecked_columns.rs (84%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/attached_columns.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/attached_schema.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/input_analysis_sql.rs (86%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/input_model_name.rs (85%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/input_reference_rows.rs (86%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/input_sql_macro_checks.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{model_input_view => model_inputs}/mod.rs (78%) create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/audits.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/collection.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compile_inputs_view.rs (91%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_audit_record.rs (88%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_function_record.rs (88%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_model_record.rs (88%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads/models.rs => project/compiled_models.rs} (60%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_project_facts.rs (89%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_seed_record.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/compiled_source_record.rs (88%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/declaration_rows.rs (89%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/declaration_rows_function.rs (83%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/functions.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/inputs_with.rs (89%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/mod.rs (78%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/plain_typed_value.rs (86%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/project_with.rs (89%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/record_model_analyses.rs (90%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/register_records.rs (84%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/replace.rs (86%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/seeds.rs (81%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/sources.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{project_reads => project}/sql_tests.rs (82%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/with_binding_diagnostics.rs (87%) rename crates/sqlbuild-python-project/src/compiled_views/main/{records => project}/with_type_recovery.rs (90%) delete mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs rename crates/sqlbuild-python-project/src/graph/_helpers/{ => nodes}/algorithms.rs (100%) rename crates/sqlbuild-python-project/src/graph/_helpers/{ => nodes}/execution_gates.rs (100%) rename crates/sqlbuild-python-project/src/graph/_helpers/{ => nodes}/keys.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs rename crates/sqlbuild-python-project/src/graph/_helpers/{ => project}/getters.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs rename crates/sqlbuild-python-project/src/graph/_helpers/{ => project}/project_graph.rs (100%) rename crates/sqlbuild-python-project/src/graph/_helpers/{ => project}/views.rs (99%) create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs rename crates/sqlbuild-python-project/src/graph/_helpers/{ => python_nodes}/python_lifecycle.rs (100%) rename crates/sqlbuild-python-project/src/graph/_helpers/{ => python_nodes}/python_node_graph.rs (98%) rename crates/sqlbuild-python-project/src/graph/_helpers/{ => python_nodes}/python_selection.rs (99%) delete mode 100644 crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/graph/main/keys/mod.rs rename crates/sqlbuild-python-project/src/graph/main/{algorithms => nodes}/closure.rs (91%) rename crates/sqlbuild-python-project/src/graph/main/{algorithms => nodes}/inverted.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs rename crates/sqlbuild-python-project/src/graph/main/{execution_gates => nodes}/model_python_hook_names.rs (90%) rename crates/sqlbuild-python-project/src/graph/main/{keys => nodes}/native_key.rs (89%) rename crates/sqlbuild-python-project/src/graph/main/{algorithms => nodes}/register_algorithms.rs (86%) rename crates/sqlbuild-python-project/src/graph/main/{execution_gates => nodes}/register_execution_gates.rs (86%) rename crates/sqlbuild-python-project/src/graph/main/{execution_gates => nodes}/sql_ref_key.rs (89%) rename crates/sqlbuild-python-project/src/graph/main/{algorithms => nodes}/transitive_closure_many.rs (92%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/compiled_graph_keys.rs (91%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/edge_dict.rs (89%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/graph_from_view_mappings.rs (90%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/mod.rs (75%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/native_graph_from_edges.rs (91%) rename crates/sqlbuild-python-project/src/graph/main/{project_graph => project}/project_graph.rs (88%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/project_lineage_views.rs (91%) rename crates/sqlbuild-python-project/src/graph/main/{project_graph => project}/register_project_graph.rs (86%) rename crates/sqlbuild-python-project/src/graph/main/{views => project}/register_views.rs (86%) delete mode 100644 crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs rename crates/sqlbuild-python-project/src/graph/main/{python_lifecycle => python_nodes}/build_python_sql_run_lifecycle.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs rename crates/sqlbuild-python-project/src/graph/main/{python_node_graph => python_nodes}/python_node_graph.rs (87%) rename crates/sqlbuild-python-project/src/graph/main/{python_selection => python_nodes}/python_node_names.rs (89%) rename crates/sqlbuild-python-project/src/graph/main/{python_lifecycle => python_nodes}/register_python_lifecycle.rs (84%) rename crates/sqlbuild-python-project/src/graph/main/{python_node_graph => python_nodes}/register_python_node_graph.rs (84%) rename crates/sqlbuild-python-project/src/graph/main/{python_selection => python_nodes}/register_python_selection.rs (84%) rename crates/sqlbuild-python-project/src/graph/main/{python_selection => python_nodes}/resolve_python_check_excludes.rs (89%) rename crates/sqlbuild-python-project/src/graph/main/{python_node_graph => python_nodes}/run_selection_from_inputs.rs (91%) delete mode 100644 crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_completion.rs (86%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_context.rs (90%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_metadata.rs (82%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_preparation.rs (93%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_reads.rs (92%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_selection.rs (87%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_sql.rs (100%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_types.rs (98%) rename crates/sqlbuild-python-project/src/planner_strategy/_helpers/{ => fixtures}/fixture_values.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot.rs (92%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_boundary.rs (95%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_declarations.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_execution.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_models.rs (96%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_resources.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_roles.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_project/_helpers => project_snapshot/_helpers/snapshot}/snapshot_worker.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_declarations => project_snapshot/main}/compile_declarations.rs (92%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_roles => project_snapshot/main}/compile_project_roles.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_models => project_snapshot/main}/model_nodes.rs (92%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_declarations => project_snapshot/main}/python_declarations.rs (90%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot => project_snapshot/main}/register_snapshot.rs (83%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_boundary => project_snapshot/main}/relative_path.rs (78%) rename crates/sqlbuild-python-project/src/{compiled_project/main/snapshot_resources => project_snapshot/main}/resource_nodes.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/mod.rs create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/models.rs create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/types.rs rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/audit_reads.rs (99%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/audit_views.rs (94%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/function_reads.rs (99%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/function_views.rs (94%) create mode 100644 crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/resource_reads.rs (95%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/seed_reads.rs (98%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/seed_views.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/source_reads.rs (97%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/_helpers/source_views.rs (83%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/always_run.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/attached_column_name.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/attached_target_kind.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/attached_target_name.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/definition_name.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/description.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/evaluation_mode.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/evidence_limit.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/evidence_sql.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/key.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/measurement_contract.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/meta.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/minimum_samples.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/name.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/references.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/run_scope.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/scope_deps.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/severity.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/sql_body.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_reads/thresholds.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_views/audit_record.rs (67%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_views/audit_reference_rows.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/audit_views/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/arguments.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/body_sql.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/deps.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/destination.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/entry_point.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/fingerprint_destination.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/key.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/language.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/name.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/packages.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/relative_path.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/return_columns.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/returns.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/runtime_version.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_reads/source_file_path.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_views/function_record.rs (67%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/function_views/mod.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/resource_views/main/mod.rs rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/destination.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/graph_row.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/key.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/name.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/references.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/resource_reads/schema_entry.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/destination.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/external.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/key.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/name.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/schema_entry.rs (88%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_reads/seed_file.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_views/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/seed_views/paired_seeds.rs (85%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_reads/key.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_reads/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_reads/name.rs (86%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_reads/source_entry.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_reads/source_file.rs (87%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_views/mod.rs (100%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_views/read_source_view.rs (83%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_views/replaced.rs (89%) rename crates/sqlbuild-python-project/src/{compiled_views => resource_views}/main/source_views/retain_sources.rs (72%) create mode 100644 crates/sqlbuild-python-project/src/resource_views/mod.rs create mode 100644 crates/sqlbuild-python-project/src/resource_views/models.rs create mode 100644 crates/sqlbuild-python-project/src/resource_views/types.rs rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => inputs}/cursor_window.rs (98%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => inputs}/fixture_validation.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs create mode 100644 crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_assembly.rs (94%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_driver.rs (90%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_entries.rs (95%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_errors.rs (98%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_fixtures.rs (93%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_rendering.rs (90%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_rows.rs (100%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/plan_values.rs (100%) rename crates/sqlbuild-python-project/src/sql_test_glue/_helpers/{ => plan}/planning.rs (98%) rename crates/sqlbuild-python-project/src/sql_test_glue/{_helpers => }/types.rs (56%) diff --git a/crates/README.md b/crates/README.md index 3b6a0de5bd..d6edeeaef9 100644 --- a/crates/README.md +++ b/crates/README.md @@ -26,18 +26,19 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, SQL-test/scenario extraction and compiler-side assembly, and Snowflake cost statement classification and query-tag parsing. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. Its caches and rule fingerprints key on the build identity the binary installs at startup. | | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | -| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | +| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python-project` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | | `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs index bd172affb0..cfa44b0871 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs @@ -6,8 +6,6 @@ use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_analysis::semantic_validation::types::Relations; -use std::collections::HashMap; - #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] pub(crate) struct CatalogInput { @@ -19,15 +17,6 @@ pub(crate) struct CatalogInput { pub(crate) relations: Relations, } -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub struct NormalizationInput { - pub(crate) sql: String, - pub(crate) dialect: String, - pub(crate) stubs: HashMap, - pub(crate) placeholders: HashMap, -} - #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] pub(crate) struct PositionInput { @@ -50,17 +39,6 @@ impl From for validation::CatalogInput { } } -impl From for validation::NormalizationInput { - fn from(input: NormalizationInput) -> Self { - Self { - sql: input.sql, - dialect: input.dialect, - stubs: input.stubs, - placeholders: input.placeholders, - } - } -} - impl From for validation::PositionInput { fn from(input: PositionInput) -> Self { Self { diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs index d856ce81fd..960787e244 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs @@ -12,7 +12,7 @@ use sqlbuild_python_base::types::CompilerDetach; #[pyfunction] pub(crate) fn normalize_analysis_sql( py: Python<'_>, - request: crate::analysis::_helpers::native_classes::NormalizationInput, + request: crate::analysis::models::NormalizationInput, ) -> PyResult { py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( diff --git a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs index ca75cbea4b..14458feef5 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs @@ -8,7 +8,7 @@ use pyo3::prelude::Python; #[inline] pub fn normalize_analysis_sql( py: Python<'_>, - request: crate::analysis::_helpers::native_classes::NormalizationInput, + request: crate::analysis::models::NormalizationInput, ) -> PyResult { normalization::normalize_analysis_sql(py, request) } diff --git a/crates/sqlbuild-python-project/src/analysis/models.rs b/crates/sqlbuild-python-project/src/analysis/models.rs index 4930ec7c5a..6eaf134b27 100644 --- a/crates/sqlbuild-python-project/src/analysis/models.rs +++ b/crates/sqlbuild-python-project/src/analysis/models.rs @@ -32,6 +32,7 @@ use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_base::types::FromPositional; use sqlbuild_python_base::types::IntoPositional; +use std::collections::HashMap; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; @@ -293,6 +294,15 @@ pub struct SchemaRequestParts<'py> { pub quoted_identifiers_ignore_case: bool, } +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub struct NormalizationInput { + pub sql: String, + pub dialect: String, + pub stubs: HashMap, + pub placeholders: HashMap, +} + #[pyclass(module = "sqlbuild._native")] #[derive(Debug)] pub struct ProjectCatalog { @@ -319,6 +329,17 @@ pub struct BindingPositions { pub inner: validation::BindingPositions, } +impl From for validation::NormalizationInput { + fn from(input: NormalizationInput) -> Self { + Self { + sql: input.sql, + dialect: input.dialect, + stubs: input.stubs, + placeholders: input.placeholders, + } + } +} + /// A record of another crate exchanged with Python as its positional tuple. #[derive(Clone, Debug)] pub struct Positional(pub T); diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs index 67f3f2d22b..d5d180b943 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs @@ -5,6 +5,7 @@ use crate::analysis_session::models::FamilyRow; use crate::analysis_session::models::ModelRow; use crate::analysis_session::models::NativeModelAnalysisSession; use crate::analysis_session::models::PivotRequestRow; +use crate::analysis_session::models::ProofRow; use crate::analysis_session::models::RequestRow; use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::pyfunction; @@ -40,7 +41,7 @@ use sqlbuild_python_base::cache::main::open_store::open_store; use sqlbuild_python_base::types::CompilerDetach; /// `(name, type, nullability)`. -type ColumnRow = (String, Option, String); +pub(crate) type ColumnRow = (String, Option, String); positional_record! { into; /// One lineage column: output column, transform and confidence codes and its sources. @@ -72,20 +73,6 @@ positional_record! { } } -positional_record! { - into; - /// One dynamic pivot proof. - pub struct ProofRow { - output_proven: bool, - fixed_columns: Vec, - /// `(family, inferred type)` of each family. - families: Vec<(String, Option)>, - input_relations: Vec, - failure_reason: Option, - bare_dynamic_pivot: bool, - } -} - /// One model's pivot SQL and declared families. pub(crate) type PivotModelRow = (String, Vec); diff --git a/crates/sqlbuild-python-project/src/analysis_session/models.rs b/crates/sqlbuild-python-project/src/analysis_session/models.rs index d2c5105c74..011c23a073 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/models.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/models.rs @@ -4,6 +4,7 @@ use crate::analysis::models::Positional; use crate::analysis_session::_helpers::fixture_columns::infer_fixture_column_facts; use crate::analysis_session::_helpers::nullability_rules::RuleFailure; use crate::analysis_session::_helpers::nullability_rules::raised; +use crate::analysis_session::_helpers::session::ColumnRow; use crate::analysis_session::_helpers::session::ContractRow; use crate::analysis_session::_helpers::session::FinishRow; use crate::analysis_session::_helpers::session::OutcomeRow; @@ -209,6 +210,38 @@ pub struct RequestRow { pub models: Vec, } +/// One dynamic pivot proof. +#[derive(Debug)] +pub struct ProofRow { + pub output_proven: bool, + pub fixed_columns: Vec, + /// `(family, inferred type)` of each family. + pub families: Vec<(String, Option)>, + pub input_relations: Vec, + pub failure_reason: Option, + pub bare_dynamic_pivot: bool, +} + +impl<'py> pyo3::IntoPyObject<'py> for ProofRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.output_proven, + self.fixed_columns, + self.families, + self.input_relations, + self.failure_reason, + self.bare_dynamic_pivot, + ), + py, + ) + } +} + /// The dialect, relation facts and families by table every proof reads, and the models. #[derive(Debug)] pub struct PivotRequestRow { diff --git a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs index b016e02993..3f42923150 100644 --- a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use crate::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; +use crate::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs index c639b790aa..8f9a7a6353 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs @@ -19,9 +19,9 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeFunctionInput; -use crate::compiled_views::models::NativeSourceInput; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSourceInput; const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs index 6a8ba26c5c..0a33885dcf 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -3,12 +3,4 @@ pub(crate) mod job_catalog; pub(crate) mod project; pub(crate) mod rows; pub(crate) mod schema_facts; -pub(crate) mod snapshot; -pub(crate) mod snapshot_boundary; -pub(crate) mod snapshot_declarations; -pub(crate) mod snapshot_execution; -pub(crate) mod snapshot_models; -pub(crate) mod snapshot_resources; -pub(crate) mod snapshot_roles; -pub(crate) mod snapshot_worker; pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs index 844d904600..0014567ecb 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs @@ -3,11 +3,11 @@ use crate::compiled_project::constants::CONTRACT_KEY; use crate::compiled_project::constants::INCREMENTAL_MODE_KEY; use crate::compiled_project::models::DeclarationPy; +use crate::compiled_project::models::ModelInputPy; use crate::compiled_project::models::NativeCompiledProject; use sqlbuild_python_base::positional_record; use std::collections::BTreeMap; -use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use serde_json::Value; @@ -17,18 +17,18 @@ use sqlbuild_analysis::compiled_project::models::{ use crate::compiled_project::_helpers::values::object_value; use crate::compiled_project::_helpers::values::plain_value; -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeCompiledAudit; -use crate::compiled_views::models::NativeCompiledFunction; use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeCompiledSeed; -use crate::compiled_views::models::NativeCompiledSource; -use crate::compiled_views::models::NativeFunctionInput; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; -use crate::compiled_views::models::NativeSeedInput; -use crate::compiled_views::models::NativeSourceInput; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; pub(crate) const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; pub(crate) const ATTACHMENT_LOOP_UNSET: &str = @@ -43,28 +43,19 @@ pub(crate) type NamedType = (String, Option); positional_record! { extract; /// A model's name, relative path, compiled and authored queries, and authored SQL file. + #[derive(Debug)] pub(crate) struct ModelTextsPy { - name: String, - relative_path: String, - query_sql: String, - authored_query_sql: String, - authored_sql: String, + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) query_sql: String, + pub(crate) authored_query_sql: String, + pub(crate) authored_sql: String, } } /// Encoded config values and the encoder's error, if any. pub(crate) type ConfigValues = (BTreeMap, Option); -/// One model as the compile-input stage produced it. -#[derive(FromPyObject)] -pub struct ModelInputPy<'py>( - ModelTextsPy, - (Bound<'py, PyDict>, Vec, Option), - Vec<(String, String, Option)>, - Vec, - (Vec>, Vec>), -); - pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { let ModelInputPy( ModelTextsPy { diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index 3402663812..d72ce96a40 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -1,13 +1,13 @@ //! Public constants of compiled project. -pub(crate) const SKIP_ACTION: &str = "skip"; +pub const SKIP_ACTION: &str = "skip"; -pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub const NO_CHANGE_REASON: &str = "no_change"; -pub(crate) const REPORT_ATTEMPTS: usize = 3; +pub const REPORT_ATTEMPTS: usize = 3; /// The HTTP status a control server answers while its store is busy; reports retry it. -pub(crate) const STORE_BUSY_STATUS: u16 = 503; +pub const STORE_BUSY_STATUS: u16 = 503; /// Selector kinds that can name a job: `job:` and a bare name. pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs deleted file mode 100644 index e5b1d65b77..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod install; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs b/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/config_values.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/declarations.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/entry_schema_fact.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/family_fact.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/graph_with_jobs.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs b/crates/sqlbuild-python-project/src/compiled_project/main/install.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/attachment_inputs/install.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/install.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs deleted file mode 100644 index dd6ef73702..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod graph_with_jobs; -pub mod job_names; -pub mod needed_jobs; -pub mod triggered_jobs; -pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/job_names.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs b/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/materialization.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index 5a36812609..1a34adb72d 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -1,12 +1,18 @@ -pub mod attachment_inputs; -pub mod job_catalog; -pub mod project; -pub mod rows; -pub mod schema_facts; -pub mod snapshot; -pub mod snapshot_boundary; -pub mod snapshot_declarations; -pub mod snapshot_models; -pub mod snapshot_resources; -pub mod snapshot_roles; -pub mod values; +pub mod config_values; +pub mod declarations; +pub mod entry_schema_fact; +pub mod family_fact; +pub mod graph_with_jobs; +pub mod install; +pub mod job_names; +pub mod materialization; +pub mod model_facts; +pub mod needed_jobs; +pub mod plain_value; +pub mod python_column_fact; +pub mod python_column_facts; +pub mod reference_row_tuple; +pub mod register_project; +pub mod text_value; +pub mod triggered_jobs; +pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs index 8d164ea87a..850e339ddc 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/project/model_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs @@ -1,7 +1,7 @@ //! Model facts. use crate::compiled_project::_helpers::project; -use crate::compiled_project::_helpers::project::ModelInputPy; +use crate::compiled_project::models::ModelInputPy; use pyo3::prelude::PyResult; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/needed_jobs.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/values/plain_value.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs deleted file mode 100644 index 0173ee2d17..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/project/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod config_values; -pub mod declarations; -pub mod materialization; -pub mod model_facts; -pub mod register_project; -pub mod text_value; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_fact.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/python_column_facts.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/rows/reference_row_tuple.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs b/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/register_project.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs deleted file mode 100644 index 8aaca236c8..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/rows/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod reference_row_tuple; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs deleted file mode 100644 index cd7255834c..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/schema_facts/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod entry_schema_fact; -pub mod family_fact; -pub mod python_column_fact; -pub mod python_column_facts; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs deleted file mode 100644 index 13bea31dc0..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod register_snapshot; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs deleted file mode 100644 index 4205cea05f..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod relative_path; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs deleted file mode 100644 index f30195d675..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod compile_declarations; -pub mod python_declarations; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs deleted file mode 100644 index 8a1804d90e..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod model_nodes; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs deleted file mode 100644 index 2b4f92fed1..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod resource_nodes; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs deleted file mode 100644 index 6dbdce4266..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod compile_project_roles; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/project/text_value.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/triggered_jobs.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs deleted file mode 100644 index a9728688b6..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/main/values/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod plain_value; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs b/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_catalog/with_job_scope.rs rename to crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/mod.rs index 67c7ce0488..4af01a464c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/mod.rs @@ -1,4 +1,4 @@ -//! The compiled project: its rows, schema facts, job catalog and the snapshot written and read across processes. +//! The compiled project: its rows, attachment inputs, schema facts, values and job catalog. pub(crate) mod _helpers; pub mod constants; diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs index 8a360d9455..55b3e55438 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -8,55 +8,42 @@ use crate::compiled_project::_helpers::attachment_inputs::ResourceDiagnosticsRun use crate::compiled_project::_helpers::attachment_inputs::resource_attachment; use crate::compiled_project::_helpers::project::ATTACHMENT_LOOP_UNSET; use crate::compiled_project::_helpers::project::INPUT_CLASSES_UNSET; +use crate::compiled_project::_helpers::project::ModelTextsPy; use crate::compiled_project::_helpers::project::NamedType; -use crate::compiled_project::_helpers::snapshot_execution::Completion; -use crate::compiled_project::_helpers::snapshot_execution::is_function; -use crate::compiled_project::_helpers::snapshot_execution::poisoned; -use crate::compiled_project::_helpers::snapshot_execution::severity; -use crate::compiled_project::_helpers::snapshot_worker::WorkerOwner; -use crate::compiled_project::_helpers::snapshot_worker::unavailable; -use crate::compiled_project::constants::NO_CHANGE_REASON; -use crate::compiled_project::constants::REPORT_ATTEMPTS; -use crate::compiled_project::constants::SKIP_ACTION; -use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; -use crate::compiled_project::constants::STORE_BUSY_STATUS; use crate::compiled_project::types::AttachedSqlTests; use crate::compiled_project::types::LoaderRow; -use crate::compiled_project::types::NodeKeys; use crate::compiled_project::types::ResourceDiagnostics; use crate::compiled_project::types::SourceRow; use crate::compiled_project::types::SourceSettings; use crate::compiled_project::types::TestCaseRow; -use crate::compiled_views::main::audit_views::audit_record::audit_record; -use crate::compiled_views::main::function_views::function_record::function_record; -use crate::compiled_views::main::seed_views::paired_seeds::paired_seeds; use crate::compiled_views::models::CompiledViewClasses; -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeCompiledAudit; -use crate::compiled_views::models::NativeCompiledFunction; use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeCompiledSeed; -use crate::compiled_views::models::NativeCompiledSource; -use crate::compiled_views::models::NativeFunctionInput; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeProjectView; -use crate::compiled_views::models::NativeSeedInput; -use crate::compiled_views::models::NativeSourceInput; use crate::compiled_views::models::PyProjectHost; -use crate::compiled_views::models::SeedNamespace; -use crate::compiled_views::types::AttachedAudit; -use crate::compiled_views::types::AttachedFunction; -use crate::compiled_views::types::AuditRecord; -use crate::compiled_views::types::FunctionRecord; use crate::compiled_views::types::ModelInputRecord; -use crate::compiled_views::types::SeedRecord; -use crate::compiled_views::types::SeedViews; -use crate::compiled_views::types::SourceRecord; +use crate::resource_views::main::audit_views::audit_record::audit_record; +use crate::resource_views::main::function_views::function_record::function_record; +use crate::resource_views::main::seed_views::paired_seeds::paired_seeds; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::AuditRecord; +use crate::resource_views::types::FunctionRecord; +use crate::resource_views::types::SeedRecord; +use crate::resource_views::types::SeedViews; +use crate::resource_views::types::SourceRecord; use pyo3::FromPyObject; use pyo3::exceptions::PyIndexError; -use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::exceptions::PyRuntimeError; -use pyo3::exceptions::PyValueError; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; @@ -69,7 +56,6 @@ use pyo3::pymethods; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; use pyo3::types::PyString; -use pyo3::types::PyStringMethods; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; @@ -78,39 +64,15 @@ use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; use sqlbuild_analysis::compiled_project::models::DynamicProofFact; use sqlbuild_analysis::compiled_project::models::ModelAnalysisFact; use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; -use sqlbuild_core::models::ResourceKind; -use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; -use sqlbuild_host::models::SqlTestOutcome; -use sqlbuild_host::types::ResultEnum; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::runs::models::NodeOutcome; -use sqlbuild_orchestration::runs::models::RunChange; -use sqlbuild_orchestration::runs::models::RunCommand; -use sqlbuild_orchestration::runs::models::RunExecution; -use sqlbuild_orchestration::runs::models::RunOutcome; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use sqlbuild_orchestration::snapshots::models::SnapshotRecord; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; -use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_project::models::CompiledProject; use sqlbuild_project::types::InferredRow; use sqlbuild_project::types::ObjectKeyRow; use sqlbuild_project::types::SourceNamespace; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::cache::models::SqlTestScanStore; -use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_front::render::models::ConstantRendering; -use std::collections::BTreeMap; -use std::collections::BTreeSet; use std::collections::HashMap; -use std::collections::VecDeque; -use std::sync::Arc; use std::sync::Mutex; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; -use uuid::Uuid; /// A function's paths, stem, discovered file, header values, body and SQL file contents. #[derive(Debug)] @@ -596,6 +558,16 @@ impl<'py> pyo3::FromPyObject<'py> for AnalysisPy { } } +/// One model as the compile-input stage produced it. +#[derive(Debug, FromPyObject)] +pub struct ModelInputPy<'py>( + pub(crate) ModelTextsPy, + pub(crate) (Bound<'py, PyDict>, Vec, Option), + pub(crate) Vec<(String, String, Option)>, + pub(crate) Vec, + pub(crate) (Vec>, Vec>), +); + /// Name, relative path, members, value, value type and rendering of one declaration. #[derive(Debug, FromPyObject)] pub struct DeclarationPy<'py>( @@ -1062,942 +1034,3 @@ impl<'py> pyo3::IntoPyObject<'py> for ReferenceTuple { ) } } - -/// A data-only publication; this object never contains adapters, connections or user callables. -#[pyclass(frozen, module = "sqlbuild._native")] -#[derive(Debug)] -pub struct NativeOrchestrationSnapshot { - pub input: SnapshotInput, -} - -impl NativeOrchestrationSnapshot { - /// Native callers retain transport errors so they can report an uncertain acknowledgement. - pub fn publication( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - ) -> PyResult> { - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Ok(result) - }) - } -} - -#[pymethods] -impl NativeOrchestrationSnapshot { - /// Serialize only at the public boundary; native transports can use the retained input. - pub(crate) fn to_json(&self, py: Python<'_>) -> PyResult { - py.detach(|| serde_json::to_string(&self.input)) - .map_err(|error| PyValueError::new_err(error.to_string())) - } - - /// Publish retained data without projecting it through Python, releasing the GIL during IO. - pub(crate) fn publish( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - ) -> PyResult { - let record = self - .publication(py, server_url, bearer)? - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - py.detach(|| { - serde_json::to_string(&record) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Retain the verified claim and its independent native heartbeat in the worker interpreter. - pub(crate) fn start_worker( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - identity: (String, String), - ) -> PyResult { - NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) - } -} - -/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. -#[derive(Debug)] -pub struct WorkerBuild { - pub worker: Py, - pub(crate) execution: RunExecution, - pub(crate) completed: Mutex>, - pub(crate) checks: Mutex>>, - /// The snapshot's audits and SQL tests, each with the node keys it checks. - pub(crate) attached: BTreeMap>, - /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. - pub(crate) blocked: BTreeMap>, - /// How many of each function's planned tests have still to report; a function is built before its tests run. - pub(crate) untested: Mutex>, - /// A built function's result, held until its tests report so a failing one can fail it instead. - pub(crate) held: Mutex>, - /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. - pub(crate) upstream: BTreeMap>, - /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. - pub(crate) causes: Mutex>, - /// The worst attached outcome of each node, for commands that only run tests or audits. - pub(crate) checked: Mutex>, - /// Whether the command returned, having run every check it planned, rather than being refused or raising. - pub(crate) returned: AtomicBool, - /// Why the worker refused the run before executing anything, reported as the run's status reason. - pub(crate) refusal: Mutex>, -} - -impl WorkerBuild { - pub fn new( - worker: &Bound<'_, NativeOrchestrationWorker>, - (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), - ) -> PyResult> { - let state = Arc::new(Self { - execution: worker.get().execution(worker.py())?, - worker: worker.clone().unbind(), - completed: Mutex::default(), - checks: Mutex::default(), - attached, - blocked, - untested: Mutex::default(), - held: Mutex::default(), - upstream, - causes: Mutex::default(), - checked: Mutex::default(), - returned: AtomicBool::new(false), - refusal: Mutex::default(), - }); - for key in state.execution.skipped_nodes.keys() { - state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; - } - Ok(state) - } - - pub fn command(&self) -> RunCommand { - self.execution.claim.request.command - } - - /// The server run this build executes. - pub fn run_id(&self) -> uuid::Uuid { - self.execution.claim.request.run_id - } - - /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. - pub(crate) fn checks_only(&self) -> bool { - matches!(self.command(), RunCommand::Test | RunCommand::Audit) - } - - /// Record that the command returned instead of being refused or raising. - pub fn command_returned(&self) { - self.returned.store(true, Ordering::SeqCst); - } - - pub fn has_work(&self) -> bool { - self.execution - .claim - .request - .selection - .iter() - .any(|key| !self.execution.skipped_nodes.contains_key(key)) - } - - /// Record why the run is refused; the terminal report carries it as the run's status reason. - pub fn refuse(&self, reason: &str) -> PyResult<()> { - *self.refusal.lock().map_err(|_| poisoned())? = Some(reason.to_owned()); - Ok(()) - } - - pub fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { - if self.worker.bind(py).get().cancel_requested(py) { - return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); - } - Ok(()) - } - - /// Reject writers outside the frozen write set before any executor connection. - pub fn enforce_writers(&self, py: Python<'_>, keys: &[String]) -> Result<(), pyo3::PyErr> { - self.check_cancelled(py)?; - match keys - .iter() - .find(|key| !self.execution.claim.request.write_set.contains(*key)) - { - Some(key) => Err(PyValueError::new_err(format!( - "planned writer {key} is outside the frozen run write set" - ))), - None => Ok(()), - } - } - - /// Reject an implicitly expanded write set before any executor connection or schema change. - pub fn enforce_plan_writers(&self, pipeline: &Bound<'_, PyAny>) -> Result<(), pyo3::PyErr> { - self.check_cancelled(pipeline.py())?; - let plan = field(pipeline, "plan_output")?; - self.count_planned_tests(&plan)?; - for collection in [ - "model_entries", - "seed_entries", - "function_entries", - "source_load_entries", - ] { - for entry in field(&plan, collection)?.try_iter()? { - let entry = entry?; - let identity = field(&entry, "key")?; - let kind = field(&identity, "resource_type")? - .str()? - .to_str()? - .to_owned(); - let key = format!("{kind}:{}", field(&identity, "name")?.str()?); - if !self.execution.claim.request.write_set.contains(&key) { - return Err(PyValueError::new_err(format!( - "planned writer {key} is outside the frozen run write set" - ))); - } - if kind == ResourceKind::Model.as_str() - && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION - { - let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { - NodeOutcome::Success - } else { - NodeOutcome::Skipped - }; - self.complete(pipeline.py(), &key, (outcome, None, (None, None)))?; - } - } - } - Ok(()) - } - - /// Count each function's tests this build plans, so a built function waits only for tests that will report. - pub(crate) fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { - let mut untested = self.untested.lock().map_err(|_| poisoned())?; - for entry in field(plan, "test_entries")?.try_iter()? { - let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); - for node in self.blocked.get(&test).into_iter().flatten() { - if is_function(node) { - *untested.entry(node.clone()).or_default() += 1; - } - } - } - Ok(()) - } - - /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. - pub fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { - for key in keys { - self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; - } - Ok(()) - } - - /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. - pub fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { - for (key, why) in nodes { - self.complete( - py, - key, - ( - NodeOutcome::Skipped, - None, - (None, Some(format!("held back: {why}"))), - ), - )?; - } - Ok(()) - } - - /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. - pub(crate) fn complete( - &self, - py: Python<'_>, - key: &str, - (outcome, rows_affected, (error_code, error_message)): ( - NodeOutcome, - Option, - (Option, Option), - ), - ) -> PyResult<()> { - if !self.execution.claim.request.selection.contains(key) { - return Ok(()); - } - if self - .completed - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - return Ok(()); - } - let skipped_because_of = match outcome { - NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { - Some(cause) => Some(cause.clone()), - None if !self.checks_only() => self.failed_upstream(key)?, - None => None, - }, - _ => None, - }; - self.worker.bind(py).get().change( - py, - &format!("node-finished/{key}"), - RunChange::NodeFinished { - node_key: key.into(), - outcome, - rows_affected, - error_code, - skipped_because_of: skipped_because_of.clone(), - error_message: error_message.as_deref().map(redact_error_text), - }, - )?; - let cause = match outcome { - NodeOutcome::Failed => Some(key.to_owned()), - _ => skipped_because_of, - }; - if let Some(cause) = cause { - self.causes - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), cause); - } - self.completed - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), outcome); - Ok(()) - } - - pub fn start( - &self, - py: Python<'_>, - name: &str, - resource_kind: &Bound<'_, PyAny>, - ) -> PyResult<()> { - self.check_cancelled(py)?; - let kind = field(resource_kind, "value")?.extract::()?; - let kind = match kind.as_str() { - "table" | "view" | "snapshot" | "custom" => "model", - kind => kind, - }; - self.start_key(py, &format!("{kind}:{name}")) - } - - /// Report a selected node as started unless its outcome is final, such as a planner skip. - pub fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { - self.check_cancelled(py)?; - if self.execution.claim.request.selection.contains(key) - && !self - .completed - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - self.worker.bind(py).get().change( - py, - &format!("node-started/{key}"), - RunChange::NodeStarted { - node_key: key.into(), - }, - )?; - } - Ok(()) - } - - /// Report a selected node's outcome when the command reports no executor result for it. - pub fn finish_key(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { - self.complete(py, key, (outcome, None, (None, None))) - } - - /// A test or audit the snapshot records has started; a selected SQL test reports as a node. - pub fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { - self.check_cancelled(py)?; - let Some(nodes) = self.attached.get(key) else { - return Ok(()); - }; - if self.execution.claim.request.selection.contains(key) { - return self.start_key(py, key); - } - if self - .checks - .lock() - .map_err(|_| poisoned())? - .contains_key(key) - { - return Ok(()); - } - self.worker.bind(py).get().change( - py, - &format!("check-started/{key}"), - RunChange::CheckStarted { - node_key: key.into(), - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), None); - if self.checks_only() { - for node in nodes { - self.start_key(py, node)?; - } - } - Ok(()) - } - - /// A test or audit finished; a check the snapshot does not record is not reported. - pub fn finish_attached(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { - let Some(nodes) = self.attached.get(key) else { - return Ok(()); - }; - if outcome != NodeOutcome::Skipped { - self.start_attached(py, key)?; - } - let outcome = if outcome == NodeOutcome::Failed - && self.worker.bind(py).get().refresh_cancellation(py)? - { - NodeOutcome::Cancelled - } else { - outcome - }; - if self.execution.claim.request.selection.contains(key) { - self.complete(py, key, (outcome, None, (None, None)))?; - } else { - self.finish_check(py, key, outcome)?; - } - if outcome == NodeOutcome::Skipped { - return Ok(()); - } - let mut checked = self.checked.lock().map_err(|_| poisoned())?; - for node in nodes { - let worst = checked.get(node).map_or(outcome, |previous| { - if severity(outcome) > severity(*previous) { - outcome - } else { - *previous - } - }); - checked.insert(node.clone(), worst); - } - Ok(()) - } - - /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. - pub(crate) fn failed_upstream(&self, key: &str) -> PyResult> { - let causes = self.causes.lock().map_err(|_| poisoned())?; - let mut seen: BTreeSet<&str> = BTreeSet::from([key]); - let mut pending: VecDeque<&str> = VecDeque::from([key]); - while let Some(node) = pending.pop_front() { - for upstream in self.upstream.get(node).into_iter().flatten() { - if let Some(cause) = causes.get(upstream) { - return Ok(Some(cause.clone())); - } - if seen.insert(upstream) { - pending.push_back(upstream); - } - } - } - Ok(None) - } - - /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. - pub(crate) fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { - let py = result.py(); - let outcome: String = field(result, "outcome")?.str()?.extract()?; - let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); - let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; - let name: String = field(result, "test_name")?.extract()?; - let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); - let error: Option = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message: Option = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - let node = match (passed, cancelled) { - (true, _) => NodeOutcome::Success, - (false, true) => NodeOutcome::Cancelled, - (false, false) => NodeOutcome::Failed, - }; - self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; - if !passed && !cancelled { - for model in self.blocked.get(&key).into_iter().flatten() { - self.complete( - py, - model, - (NodeOutcome::Failed, None, (error.clone(), message.clone())), - )?; - } - } - self.release_tested(py, &key) - } - - /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. - pub(crate) fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { - let mut released: Vec<(String, Completion)> = Vec::new(); - { - let mut untested = self.untested.lock().map_err(|_| poisoned())?; - let mut held = self.held.lock().map_err(|_| poisoned())?; - for node in self.blocked.get(test).into_iter().flatten() { - let Some(remaining) = untested.get_mut(node) else { - continue; - }; - *remaining = remaining.saturating_sub(1); - if *remaining == 0 - && let Some(completion) = held.remove(node) - { - released.push((node.clone(), completion)); - } - } - } - for (node, completion) in released { - self.complete(py, &node, completion)?; - } - Ok(()) - } - - /// Publish a node's result, or hold a built function's while its tests have still to report. - pub(crate) fn complete_or_hold( - &self, - py: Python<'_>, - key: &str, - completion: Completion, - ) -> PyResult<()> { - let waiting = !self.checks_only() - && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) - && self - .untested - .lock() - .map_err(|_| poisoned())? - .get(key) - .is_some_and(|remaining| *remaining > 0); - if waiting { - self.held - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), completion); - return Ok(()); - } - self.complete(py, key, completion) - } - - /// Existing native progress callbacks publish selected-node evidence without Python relays. - pub fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { - if key.starts_with(SQL_TEST_KEY_PREFIX) { - return self.finished_test(result); - } - if self.execution.claim.request.selection.contains(key) { - let status: String = field(result, "status")?.str()?.extract()?; - let outcome = match status.as_str() { - "success" => { - if result.hasattr("warning_messages")? - && field(result, "warning_messages")?.is_truthy()? - { - NodeOutcome::Warn - } else { - NodeOutcome::Success - } - } - "failed" => { - if self - .worker - .bind(result.py()) - .get() - .refresh_cancellation(result.py())? - { - NodeOutcome::Cancelled - } else { - NodeOutcome::Failed - } - } - "skipped" => NodeOutcome::Skipped, - _ => { - return Err(PyValueError::new_err(format!( - "unsupported worker node outcome {status}" - ))); - } - }; - let rows = match result.getattr_opt("rows_affected")? { - Some(value) => value.extract()?, - None => None, - }; - let error = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; - } - Ok(()) - } - - pub fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { - self.check_cancelled(py)?; - let key = format!("check:{name}"); - self.worker.bind(py).get().change( - py, - &format!("check-started/{key}"), - RunChange::CheckStarted { - node_key: key.clone(), - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key, None); - Ok(()) - } - - /// A completion without our actual-execution start callback is a blocked check, not a failure. - pub fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { - let name: String = field(result, "node_name")?.extract()?; - let key = format!("check:{name}"); - let started = self - .checks - .lock() - .map_err(|_| poisoned())? - .contains_key(&key); - let outcome = if !started { - NodeOutcome::Skipped - } else if field(result, "passed")?.is_truthy()? { - NodeOutcome::Success - } else if field(result, "warned")?.is_truthy()? { - NodeOutcome::Warn - } else if self - .worker - .bind(result.py()) - .get() - .refresh_cancellation(result.py())? - { - NodeOutcome::Cancelled - } else { - NodeOutcome::Failed - }; - self.finish_check(result.py(), &key, outcome) - } - - pub(crate) fn finish_check( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { - self.worker.bind(py).get().change( - py, - &format!("check-finished/{key}"), - RunChange::CheckFinished { - node_key: key.into(), - outcome, - }, - )?; - self.checks - .lock() - .map_err(|_| poisoned())? - .insert(key.into(), Some(outcome)); - Ok(()) - } - - /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. - pub fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { - let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; - let checks_only = self.checks_only(); - if checks_only && !cancelled { - let checked = self.checked.lock().map_err(|_| poisoned())?.clone(); - let returned = self.returned.load(Ordering::SeqCst); - for key in &self.execution.claim.request.selection { - let seen = checked.get(key).copied(); - let outcome = if returned { - Some(seen.unwrap_or(NodeOutcome::Success)) - } else { - seen.filter(|outcome| { - matches!(outcome, NodeOutcome::Failed | NodeOutcome::Cancelled) - }) - }; - if let Some(outcome) = outcome { - self.complete(py, key, (outcome, None, (None, None)))?; - } - } - } - let held: Vec<(String, Completion)> = - std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) - .into_iter() - .collect(); - for (key, completion) in held { - self.complete(py, &key, completion)?; - } - let returned = self.returned.load(Ordering::SeqCst); - let build = self.command().is_build(); - let source_needs_no_run = |key: &str| build && key.starts_with("source:"); - for key in &self.execution.claim.request.selection { - let outcome = if cancelled { - NodeOutcome::Cancelled - } else if returned && source_needs_no_run(key) { - NodeOutcome::Success - } else { - NodeOutcome::Skipped - }; - self.complete(py, key, (outcome, None, (None, None)))?; - } - let outcome = { - let completed = self.completed.lock().map_err(|_| poisoned())?; - let outcomes: Vec = completed.values().copied().collect(); - let good = completed.iter().any(|(key, value)| { - matches!(value, NodeOutcome::Success | NodeOutcome::Warn) - && !source_needs_no_run(key) - }); - let bad = outcomes.iter().any(|value| { - matches!( - value, - NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled - ) - }); - if cancelled { - RunOutcome::Cancelled - } else if bad && good { - RunOutcome::Partial - } else if bad || exit_code != 0 { - RunOutcome::Failed - } else if outcomes.contains(&NodeOutcome::Warn) { - RunOutcome::Warn - } else { - RunOutcome::Success - } - }; - Ok(outcome) - } - - /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. - pub fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { - let outcome = self.outcome(py, exit_code)?; - let unfinished: Vec = self - .checks - .lock() - .map_err(|_| poisoned())? - .iter() - .filter(|(_, outcome)| outcome.is_none()) - .map(|(key, _)| key.clone()) - .collect(); - for key in unfinished { - self.finish_check( - py, - &key, - if outcome == RunOutcome::Cancelled { - NodeOutcome::Cancelled - } else { - NodeOutcome::Skipped - }, - )?; - } - let reason: String = self - .refusal - .lock() - .map_err(|_| poisoned())? - .clone() - .unwrap_or_else(|| "worker executor completed and closed its connections".into()); - self.worker.bind(py).get().change( - py, - "worker-finished", - RunChange::Finished { - outcome, - reason, - exit_code: Some(i64::from(exit_code)), - }, - )?; - Ok(outcome) - } -} - -/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. -#[pyclass(frozen, module = "sqlbuild._native")] -#[derive(Debug)] -pub struct NativeOrchestrationWorker { - pub(crate) owner: Mutex>, -} - -impl NativeOrchestrationWorker { - /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. - pub(crate) fn with_owner( - &self, - py: Python<'_>, - work: impl FnOnce(&WorkerOwner) -> PyResult + Send, - ) -> PyResult { - py.detach(|| { - let owner = self.owner.lock().map_err(|_| unavailable())?; - work(owner.as_ref().ok_or_else(unavailable)?) - }) - } - - /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. - pub fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - if owner.session.cancel_requested() { - return Ok(true); - } - owner - .runtime - .block_on(owner.client.heartbeat()) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Native execution reads the retained request, never caller-provided selection or settings. - pub fn execution( - &self, - py: Python<'_>, - ) -> PyResult { - self.with_owner(py, |owner| Ok(owner.session.execution.clone())) - } - - /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. - pub fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { - self.with_owner(py, |owner| { - let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); - owner.runtime.block_on(async { - let mut attempts = 0; - loop { - attempts += 1; - match owner.client.report(event_id, change.clone()).await { - Ok(_) => return Ok(()), - Err(error) - if attempts < REPORT_ATTEMPTS - && match &error { - WorkerError::Transport(_) => true, - WorkerError::Rejected(status) => { - status.as_u16() == STORE_BUSY_STATUS - } - _ => false, - } => - { - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - } - Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), - } - } - }) - }) - } - - /// Verify this retained baseline before the caller applies run settings or opens a warehouse. - pub fn start( - snapshot: &NativeOrchestrationSnapshot, - py: Python<'_>, - (server_url, bearer): (&str, &str), - (run_id, claim_id): (String, String), - ) -> PyResult { - let identity = ( - Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - ); - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let connected = runtime.block_on(async { - let client = connect_worker(server_url, bearer, identity)?; - let session = start_worker(&client, &snapshot.input).await?; - Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) - }); - match connected { - Ok((client, session)) => Ok(NativeOrchestrationWorker { - owner: Mutex::new(Some(WorkerOwner { - runtime, - client, - session, - })), - }), - Err(error) => { - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Err(PyRuntimeError::new_err(error.to_string())) - } - } - }) - } -} - -#[pymethods] -impl NativeOrchestrationWorker { - /// The frozen request supplies selection and settings only after baseline verification. - pub(crate) fn execution_json(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - serde_json::to_string(&owner.session.execution) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Cancellation remains sticky; a closed worker cannot authorize further work. - pub fn cancel_requested(&self, py: Python<'_>) -> bool { - py.detach(|| { - self.owner.lock().map_or(true, |owner| { - owner - .as_ref() - .is_none_or(|owner| owner.session.cancel_requested()) - }) - }) - } - - /// Retry an uncertain report with the identical event ID and change, never a new identity. - pub(crate) fn report( - &self, - py: Python<'_>, - event_id: &str, - change_json: &str, - ) -> PyResult { - let event_id = - Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; - let change: RunChange = serde_json::from_str(change_json) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - self.with_owner(py, |owner| { - owner - .runtime - .block_on(owner.client.report(event_id, change)) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - pub(crate) fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { - if slf.owner.lock().map_err(|_| unavailable())?.is_none() { - return Err(unavailable()); - } - Ok(slf) - } - - pub(crate) fn __exit__( - &self, - py: Python<'_>, - _kind: &pyo3::Bound<'_, pyo3::PyAny>, - _value: &pyo3::Bound<'_, pyo3::PyAny>, - _traceback: &pyo3::Bound<'_, pyo3::PyAny>, - ) -> PyResult<()> { - self.close(py) - } - - /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. - pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { - py.detach(|| { - let mut owner = self.owner.lock().map_err(|_| unavailable())?; - if let Some(active) = owner.take() { - active.close(); - } - Ok(()) - }) - } -} - -impl Drop for NativeOrchestrationWorker { - /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. - fn drop(&mut self) { - if let Ok(owner) = self.owner.get_mut() - && let Some(owner) = owner.take() - { - owner.close(); - } - } -} diff --git a/crates/sqlbuild-python-project/src/compiled_project/types.rs b/crates/sqlbuild-python-project/src/compiled_project/types.rs index d57b5bdd68..2a472bb98f 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/types.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/types.rs @@ -1,11 +1,8 @@ //! Public traits and type aliases of compiled project. -use crate::compiled_project::models::WorkerBuild; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; -use std::collections::BTreeMap; -use std::sync::Arc; /// One discovered source: its file's path, the discovered file and the entry. pub type SourceRow = (String, Py, Py); @@ -28,9 +25,3 @@ pub type ResourceDiagnostics<'py> = ( Vec>, Vec>, ); - -/// Node keys by snapshot node key, such as the models each test checks. -pub type NodeKeys = BTreeMap>; - -/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. -pub type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs similarity index 97% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs index b3da6dd247..dd703c2c3f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs @@ -1,12 +1,12 @@ //! A compiled model's analysis, kept from the native session and projected on demand. +use crate::compiled_views::models::AnalysisProjections; use sqlbuild_python_base::positional_record; use std::borrow::Cow; use std::collections::HashMap; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; @@ -62,23 +62,6 @@ pub(crate) type SessionArgument = (Py, usize); /// What one model's analysis left for its compiled model. pub(crate) type AnalysisRecord = project_models::AnalysisRecord; -/// Python projections of the analysis, each built once on first access. -pub struct AnalysisProjections { - lineage: PyOnceLock>, - binding: PyOnceLock>, - proof: PyOnceLock>, -} - -impl Default for AnalysisProjections { - fn default() -> Self { - Self { - lineage: PyOnceLock::new(), - binding: PyOnceLock::new(), - proof: PyOnceLock::new(), - } - } -} - fn inferred_row(column: &ColumnFact) -> InferredRow { ( column.name.clone(), diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs similarity index 93% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs index 1697599d39..250b283471 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs @@ -1,5 +1,6 @@ //! A compiled model, read from the native compiled project that owns it. +use crate::compiled_views::models::CompiledProjections; use crate::compiled_views::models::CompiledViewClasses; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::types::LineageInput; @@ -16,10 +17,10 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; -use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisRecord; -use crate::compiled_views::_helpers::compiled_model_analysis::ProofArgument; -use crate::compiled_views::_helpers::compiled_model_analysis::SessionArgument; -use crate::compiled_views::_helpers::compiled_model_analysis::analysis_record; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::ProofArgument; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::SessionArgument; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::analysis_record; use crate::compiled_views::models::PyProjectHost; use crate::compiled_views::types::ModelInputRecord; use crate::graph::models::ResourceRow; @@ -46,21 +47,6 @@ pub(crate) type DestinationRow = (Option, Option, String); /// What assembly and the semantic checks finished for one compiled model. pub(crate) type CompiledRecord = project_models::CompiledRecord; -/// A compiled model's Python attribute projections, each built once on first access. -pub struct CompiledProjections { - pub(crate) key: PyOnceLock>, - pub(crate) deps: PyOnceLock>, -} - -impl Default for CompiledProjections { - fn default() -> Self { - Self { - key: PyOnceLock::new(), - deps: PyOnceLock::new(), - } - } -} - /// What later stages changed for one view of a model; the owner's record stays as assembled. #[derive(Debug, Default)] pub(crate) struct ModelOverlay { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs new file mode 100644 index 0000000000..8f5cc23169 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod compiled_model_analysis; +pub(crate) mod compiled_model_view; +pub(crate) mod model_reads; +pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/model_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs similarity index 97% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs index 7e66f2f7a3..885491e727 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs @@ -14,13 +14,13 @@ use sqlbuild_discovery::project_validation::models::CsvSettings; use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; -use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; -use crate::compiled_views::_helpers::model_input_view::model_relation_key_row; -use crate::compiled_views::_helpers::source_views::SourceKeyFacts; -use crate::compiled_views::_helpers::source_views::read_source_view; +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeModelInput; -use crate::compiled_views::types::SourceViews; +use crate::resource_views::main::source_views::read_source_view::read_source_view; +use crate::resource_views::models::SourceKeyFacts; +use crate::resource_views::types::SourceViews; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs index b0e7c674c7..e0b40801cf 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs @@ -1,20 +1,3 @@ -pub(crate) mod audit_reads; -pub(crate) mod audit_views; -pub(crate) mod compiled_model_analysis; -pub(crate) mod compiled_model_view; -pub(crate) mod function_reads; -pub(crate) mod function_views; -pub(crate) mod getters; -pub(crate) mod inputs_view; -pub(crate) mod model_input_view; -pub(crate) mod model_reads; -pub(crate) mod project_host; -pub(crate) mod project_reads; -pub(crate) mod project_view; -pub(crate) mod records; -pub(crate) mod relation_keys; -pub(crate) mod resource_reads; -pub(crate) mod seed_reads; -pub(crate) mod seed_views; -pub(crate) mod source_reads; -pub(crate) mod source_views; +pub(crate) mod compiled_models; +pub(crate) mod model_inputs; +pub(crate) mod project; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs similarity index 97% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs index 7f80eaccfc..ffb2686b6d 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs @@ -9,7 +9,6 @@ const RUN_ID_FIELD: &str = "run_id"; const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; /// Compile inputs held as Python objects: user code, discovery or an integration owns them. -#[derive(Debug)] pub(crate) struct BoundaryInputs { /// The discovered project config. pub(crate) project_config: Py, @@ -40,7 +39,6 @@ pub(crate) struct BoundaryInputs { } /// Compiler-derived inputs still held as Python objects until they are owner facts. -#[derive(Debug)] pub(crate) struct CompilerHandles { /// Effective settings. pub(crate) effective_settings: Py, @@ -65,7 +63,6 @@ pub(crate) struct CompilerHandles { } /// The values of one compile inputs view. -#[derive(Debug)] pub(crate) struct InputsFields { pub(crate) run_id: String, pub(crate) effective_target_name: Option, @@ -75,6 +72,14 @@ pub(crate) struct InputsFields { pub(crate) compiler: CompilerHandles, } +impl std::fmt::Debug for InputsFields { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputsFields") + .finish_non_exhaustive() + } +} + impl InputsFields { /// The fields named in `fields`, each required. pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs new file mode 100644 index 0000000000..c4dc7d398f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod inputs_view; +pub(crate) mod model_input_view; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs similarity index 94% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs index 8b1739a8e3..70eb8f8fd3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs @@ -1,23 +1,23 @@ //! A model's compile input, read from the native compiled project that owns it. use crate::compiled_views::models::AttachedSchema; +use crate::compiled_views::models::InputViewClasses; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::types::ModelInputRecord; use sqlbuild_project::models::ExpansionRecord; use std::sync::Arc; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; -use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use crate::compiled_project::main::python_column_facts::python_column_facts; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::relation_keys::relation_key_row; -use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::_helpers::compiled_models::relation_keys::relation_key_row; use crate::compiled_views::models::RelationKeyRow; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -29,18 +29,6 @@ use sqlbuild_python_front::render::types::SpanRow; const MODEL_RELATION_KIND: &str = "ref"; -/// The Python classes a model input's projected attributes are built with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct InputViewClasses<'py> { - pub(crate) compiled: CompiledViewClasses<'py>, - pub(crate) sql_reference: Bound<'py, PyAny>, - pub(crate) reference_kind: Bound<'py, PyAny>, - pub(crate) schema_model_entry: Bound<'py, PyAny>, - pub(crate) sql_expansion: Bound<'py, PyAny>, - pub(crate) expansion_span: Bound<'py, PyAny>, -} - /// Read the input record at `index` of `owner` together with `owner`. pub(crate) fn read_input_with_owner( owner: &Py, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs similarity index 98% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs index 0ab5be42f1..769e1eebee 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/getters.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs @@ -3,18 +3,18 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeCompiledAudit; -use crate::compiled_views::models::NativeCompiledFunction; use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeCompiledSeed; -use crate::compiled_views::models::NativeCompiledSource; -use crate::compiled_views::models::NativeFunctionInput; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; -use crate::compiled_views::models::NativeSeedInput; -use crate::compiled_views::models::NativeSourceInput; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; pub(crate) fn read_native_audit_input<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs new file mode 100644 index 0000000000..bd4765fc84 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod getters; +pub(crate) mod project_host; +pub(crate) mod project_reads; +pub(crate) mod project_view; +pub(crate) mod records; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs similarity index 53% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs index ea763d064d..061e60262f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_host.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs @@ -2,17 +2,13 @@ use std::fmt::{Debug, Formatter, Result as FmtResult}; -use crate::compiled_views::_helpers::audit_views::AuditProjections; -use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisProjections; -use crate::compiled_views::_helpers::compiled_model_view::CompiledProjections; -use crate::compiled_views::_helpers::function_views::FunctionProjections; -use crate::compiled_views::_helpers::seed_views::SeedHandles; -use crate::compiled_views::_helpers::source_views::SourceProjections; -use crate::compiled_views::models::AuditHandles; -use crate::compiled_views::models::FunctionHandles; +use crate::compiled_views::models::AnalysisProjections; +use crate::compiled_views::models::CompiledProjections; use crate::compiled_views::models::ModelInputHandles; -use crate::compiled_views::models::SchemaHandles; -use crate::compiled_views::models::SourceHandles; +use crate::resource_views::models::AuditProjections; +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::models::SourceProjections; impl Debug for ModelInputHandles { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { @@ -22,14 +18,6 @@ impl Debug for ModelInputHandles { } } -impl Debug for SchemaHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SchemaHandles") - .finish_non_exhaustive() - } -} - impl Debug for CompiledProjections { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { formatter @@ -46,14 +34,6 @@ impl Debug for AnalysisProjections { } } -impl Debug for SourceHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SourceHandles") - .finish_non_exhaustive() - } -} - impl Debug for SourceProjections { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { formatter @@ -62,14 +42,6 @@ impl Debug for SourceProjections { } } -impl Debug for FunctionHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("FunctionHandles") - .finish_non_exhaustive() - } -} - impl Debug for FunctionProjections { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { formatter @@ -78,14 +50,6 @@ impl Debug for FunctionProjections { } } -impl Debug for AuditHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("AuditHandles") - .finish_non_exhaustive() - } -} - impl Debug for AuditProjections { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { formatter diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/project_reads.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs similarity index 98% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs index 4d4b2e5fc0..eb60838df9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs @@ -21,6 +21,7 @@ const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; const NAME_ATTRIBUTE: &str = "name"; /// Project inputs held as Python objects: user code, discovery or an integration owns them. +#[derive(Debug)] pub(crate) struct BoundaryInputs { /// The resolved connection, adapter integration values. pub(crate) effective_connection: Py, @@ -55,6 +56,7 @@ pub(crate) struct BoundaryInputs { } /// Compiler-derived project parts still held as Python objects until they are owner facts. +#[derive(Debug)] pub(crate) struct CompilerHandles { /// Effective settings. pub(crate) settings: Py, @@ -81,6 +83,7 @@ pub(crate) struct CompilerHandles { } /// The project-level values of one compiled project view. +#[derive(Debug)] pub(crate) struct ProjectFields { pub(crate) run_id: String, pub(crate) effective_target_name: Option, @@ -93,14 +96,6 @@ pub(crate) struct ProjectFields { pub(crate) compiler: CompilerHandles, } -impl std::fmt::Debug for ProjectFields { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("ProjectFields") - .finish_non_exhaustive() - } -} - impl ProjectFields { /// The fields named in `fields`, each required. pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs similarity index 99% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs index 12a33aa75b..2481309065 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs @@ -9,8 +9,8 @@ use pyo3::types::{ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; -use crate::compiled_project::main::project::model_facts::model_facts; -use crate::compiled_project::main::schema_facts::entry_schema_fact::entry_schema_fact; +use crate::compiled_project::main::entry_schema_fact::entry_schema_fact; +use crate::compiled_project::main::model_facts::model_facts; use crate::compiled_project::models::AnalysisPy; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_project::models::ProofPy; diff --git a/crates/sqlbuild-python-project/src/compiled_views/constants.rs b/crates/sqlbuild-python-project/src/compiled_views/constants.rs index 0c64a3a13a..05bd22c3c7 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/constants.rs @@ -1,29 +1,29 @@ //! Public constants of compiled views. -use crate::compiled_views::_helpers::getters::read_native_audit_input; -use crate::compiled_views::_helpers::getters::read_native_compiled_audit; -use crate::compiled_views::_helpers::getters::read_native_compiled_function; -use crate::compiled_views::_helpers::getters::read_native_compiled_model; -use crate::compiled_views::_helpers::getters::read_native_compiled_seed; -use crate::compiled_views::_helpers::getters::read_native_compiled_source; -use crate::compiled_views::_helpers::getters::read_native_function_input; -use crate::compiled_views::_helpers::getters::read_native_inputs_view; -use crate::compiled_views::_helpers::getters::read_native_model_input; -use crate::compiled_views::_helpers::getters::read_native_project_view; -use crate::compiled_views::_helpers::getters::read_native_seed_input; -use crate::compiled_views::_helpers::getters::read_native_source_input; -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeCompiledAudit; -use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::_helpers::project::getters::read_native_audit_input; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_audit; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_function; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_model; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_seed; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_source; +use crate::compiled_views::_helpers::project::getters::read_native_function_input; +use crate::compiled_views::_helpers::project::getters::read_native_inputs_view; +use crate::compiled_views::_helpers::project::getters::read_native_model_input; +use crate::compiled_views::_helpers::project::getters::read_native_project_view; +use crate::compiled_views::_helpers::project::getters::read_native_seed_input; +use crate::compiled_views::_helpers::project::getters::read_native_source_input; use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeCompiledSeed; -use crate::compiled_views::models::NativeCompiledSource; -use crate::compiled_views::models::NativeFunctionInput; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; -use crate::compiled_views::models::NativeSeedInput; -use crate::compiled_views::models::NativeSourceInput; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs new file mode 100644 index 0000000000..3a8474c4a1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs @@ -0,0 +1,12 @@ +//! Each audit spec's `(definition name, column argument when it is text)`. + +use crate::compiled_views::_helpers::compiled_models::relation_keys; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each audit spec's `(definition name, column argument when it is text)`. +#[inline] +pub fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { + relation_keys::audit_key_rows(audits) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs index 55c38f638d..52eefbb1cb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/binding_validated.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs @@ -1,6 +1,6 @@ //! Whether the native session validated the model's bindings. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs similarity index 80% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs index 2b5aa18c10..d1e11779df 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/config.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs @@ -1,6 +1,6 @@ //! The model's `CompileModelConfig`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs index aefe69ce68..9031c277f2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/declared_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs @@ -1,6 +1,6 @@ //! Each declared column of the attached schema as `(name, type)`, `None` without a schema. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use crate::compiled_views::types::DeclaredRow; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs new file mode 100644 index 0000000000..8251d79449 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs @@ -0,0 +1,17 @@ +//! The destination with adapter defaults filled in; `render` renders its qualified name. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_project::models::DestinationFact; + +/// The destination with adapter defaults filled in; `render` renders its qualified name. +#[inline] +pub fn defaulted_destination( + current: &DestinationFact, + defaults: (Option, Option), + render: &Bound<'_, PyAny>, +) -> PyResult { + compiled_model_view::defaulted_destination(current, defaults, render) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs index f39e51333f..07c12f13f0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dep_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs @@ -1,6 +1,6 @@ //! `(resource type, name)` of each dependency. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs index 802de4e9c9..e343eecfd3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/deps.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs @@ -1,6 +1,6 @@ //! The model's dependency `CompiledObjectKey` tuple. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs index 86a11cee6a..61f210511a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs @@ -1,6 +1,6 @@ //! The model's destination `CompiledRelationLocation`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs similarity index 84% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs index 5477819682..4949ac2ff5 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/dynamic_columns_declared.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs @@ -1,6 +1,6 @@ //! Whether the attached schema declares dynamic column families. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs index b0f600536b..a0696b255b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/entry_relation_key_row.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs @@ -1,6 +1,6 @@ //! The key row of a seed, source or hand-built model from its config values and schema entry. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use crate::compiled_views::models::RelationKeyRow; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs index 524a0f1fcc..525272a9ef 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/enum_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs @@ -1,6 +1,6 @@ //! The model's enum columns by name. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs index f81ca7ebdd..96eb742b03 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs @@ -1,6 +1,6 @@ //! The model's `InferredColumn` tuple, or `None`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs similarity index 84% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs index bd2e6593ff..3d989f68a0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_names.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs @@ -1,6 +1,6 @@ //! The names of the inferred output columns, `None` when the shape is unknown. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs index 4a9f368c29..cca9756c1f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/inferred_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs @@ -1,6 +1,6 @@ //! Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs index 52b0bd4e51..63ed6d3f33 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs @@ -1,6 +1,6 @@ //! Each lineage output with its transform kind and confidence values, `None` without lineage. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use crate::compiled_views::models::LineageFactRow; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs index 8ab719d265..2d54843884 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/lineage_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs @@ -1,6 +1,6 @@ //! Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use crate::compiled_views::types::LineageInput; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs similarity index 62% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs index 96e507a09d..6379cf4731 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs @@ -1,24 +1,36 @@ +pub mod audit_key_rows; pub mod binding_validated; pub mod config; pub mod declared_rows; +pub mod defaulted_destination; pub mod dep_rows; pub mod deps; pub mod destination; pub mod dynamic_columns_declared; +pub mod entry_relation_key_row; pub mod enum_columns; pub mod inferred_columns; pub mod inferred_names; pub mod inferred_rows; pub mod lineage_facts; pub mod lineage_inputs; +pub mod model_graph_row; pub mod model_key; +pub mod model_key_row; pub mod model_name; pub mod name_object; +pub mod object_key; +pub mod one_row_seed_names; pub mod output_proven; +pub mod project_relation_keys; pub mod proven_inputs; pub mod query_sql; pub mod references; +pub mod register_relation_keys; +pub mod relation_keys; +pub mod relation_location; pub mod relative_path; pub mod schema_entry; +pub mod source_key_input; pub mod star_flags; pub mod unchecked_columns; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs new file mode 100644 index 0000000000..a80f2cd744 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs @@ -0,0 +1,13 @@ +//! The model's project graph row: its key, dependencies, tags and folder. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use crate::compiled_views::models::NativeCompiledModel; +use crate::graph::models::ResourceRow; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The model's project graph row: its key, dependencies, tags and folder. +#[inline] +pub fn model_graph_row(model: &NativeCompiledModel, py: Python<'_>) -> PyResult { + compiled_model_view::model_graph_row(model, py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs similarity index 80% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs index 844a3329d1..346592f8b8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_key.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs @@ -1,6 +1,6 @@ //! The model's `CompiledObjectKey`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs similarity index 84% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs index 59f9850952..85c69cbf6f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/model_key_row.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs @@ -1,6 +1,6 @@ //! The declared key facts of a model input or compiled model. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use crate::compiled_views::models::RelationKeyRow; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs similarity index 78% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs index 596bb0fcd7..db42985266 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/model_name.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs @@ -1,6 +1,6 @@ //! The model's name. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs index 08b1863df9..ff4e7b3a68 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/name_object.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs @@ -1,6 +1,6 @@ //! The model's name as a Python string. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs new file mode 100644 index 0000000000..2042400894 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs @@ -0,0 +1,17 @@ +//! Object key. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use crate::compiled_views::models::CompiledViewClasses; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Object key. +#[inline] +pub fn object_key( + classes: &CompiledViewClasses<'_>, + kind: &str, + name: &str, +) -> PyResult> { + compiled_model_view::object_key(classes, kind, name) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs index 86580705ae..db83a43784 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/one_row_seed_names.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs @@ -1,6 +1,6 @@ //! Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs index 9140f3c6d6..ff0f9c9203 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/output_proven.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs @@ -1,6 +1,6 @@ //! Whether the model's dynamic pivot proof proved its output columns. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs index 02e71c03f3..5bcb0b43c5 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/project_relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs @@ -1,6 +1,6 @@ //! Declared keys of a compiled project's models, seeds and sources. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use crate::compiled_views::types::RelationKeys; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs index 24c16211a8..60839a7a46 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/proven_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs @@ -1,6 +1,6 @@ //! The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs similarity index 80% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs index 93e054b538..451f3aac2e 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/query_sql.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs @@ -1,6 +1,6 @@ //! The model's compiled query SQL. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs index 68f59ec296..d8963ab406 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/references.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs @@ -1,6 +1,6 @@ //! The model's `CompileSqlReference` tuple. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs index 843dca6bc3..a62f815534 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/register_relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs @@ -1,6 +1,6 @@ //! Register the relation keys functions and classes. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs index e8b531ca28..f49e35676b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs @@ -1,6 +1,6 @@ //! Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use crate::compiled_views::models::RelationKeyRow; use crate::compiled_views::types::RelationKeys; use pyo3::prelude::Bound; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs new file mode 100644 index 0000000000..69e5897717 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs @@ -0,0 +1,17 @@ +//! The `CompiledRelationLocation` of a destination. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use crate::compiled_views::models::CompiledViewClasses; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_project::models::DestinationFact; + +/// The `CompiledRelationLocation` of a destination. +#[inline] +pub fn relation_location( + classes: &CompiledViewClasses<'_>, + destination: &DestinationFact, +) -> PyResult> { + compiled_model_view::relation_location(classes, destination) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs index cede5426b5..647befff8f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/relative_path.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs @@ -1,6 +1,6 @@ //! The model file's relative path. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs index 7c7e9881e9..cb964b3ba4 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs @@ -1,6 +1,6 @@ //! The model's attached `SchemaModelEntry`, or `None`. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs index 42c8554a90..58b5e86718 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/source_key_input.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs @@ -1,6 +1,6 @@ //! What `relation_keys` reads for a source: its native view itself, or its `source_entry`. -use crate::compiled_views::_helpers::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs index 21851ebf26..e0113cb1d2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/star_flags.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs @@ -1,6 +1,6 @@ //! `(has star, star resolved)` of the model's analysed query. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs similarity index 84% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs index ad414a8c0d..0563ef98ab 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_reads/unchecked_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs @@ -1,6 +1,6 @@ //! Output columns whose types semantic recovery forgot, in any order. -use crate::compiled_views::_helpers::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs index acf736cc6e..fe1d52175b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs @@ -1,14 +1,3 @@ -pub mod audit_reads; -pub mod audit_views; -pub mod function_reads; -pub mod function_views; -pub mod model_input_view; -pub mod model_reads; -pub mod project_reads; -pub mod records; -pub mod relation_keys; -pub mod resource_reads; -pub mod seed_reads; -pub mod seed_views; -pub mod source_reads; -pub mod source_views; +pub mod compiled_models; +pub mod model_inputs; +pub mod project; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs index ab2ee2cd25..19206139c3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs @@ -1,6 +1,6 @@ //! The declared column facts of `model_input`'s attached schema, empty without one. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs index 6bcb65e7e3..442cd1b07f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/attached_schema.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs @@ -1,6 +1,6 @@ //! The attached schema of `model_input`; a native input answers without building its entry. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use crate::compiled_views::models::AttachedSchema; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs index c1084c5926..aa8f388ae9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_analysis_sql.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs @@ -1,6 +1,6 @@ //! The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs index 3b4df41a90..ebf0a2f342 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_model_name.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs @@ -1,6 +1,6 @@ //! The name of `model_input`, its file stem; a native input answers from its record. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs index f8ab027bb2..9137bb66f6 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_reference_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs @@ -1,6 +1,6 @@ //! Each reference of `model_input` as `(kind, name, package, call argument count)`. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs index c08c043e93..de3213f3d9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/input_sql_macro_checks.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs @@ -1,6 +1,6 @@ //! What the analysed SQL of a native model input must show for its SQL macro calls to fit. -use crate::compiled_views::_helpers::model_input_view; +use crate::compiled_views::_helpers::model_inputs::model_input_view; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs similarity index 78% rename from crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs index 7c4abf2a27..e6822ef688 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_input_view/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs @@ -4,3 +4,5 @@ pub mod input_analysis_sql; pub mod input_model_name; pub mod input_reference_rows; pub mod input_sql_macro_checks; +pub mod reference_tuple; +pub mod view_classes; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs new file mode 100644 index 0000000000..0068f4eb7d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs @@ -0,0 +1,19 @@ +//! `CompileSqlReference` records of `rows`, built once into `cache`. + +use crate::compiled_views::_helpers::model_inputs::model_input_view; +use crate::compiled_views::models::InputViewClasses; +use pyo3::prelude::Py; +use pyo3::prelude::PyResult; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use sqlbuild_project::models::ReferenceRow; + +/// `CompileSqlReference` records of `rows`, built once into `cache`. +#[inline] +pub fn reference_tuple( + classes: &InputViewClasses<'_>, + rows: &[ReferenceRow], + cache: &PyOnceLock>, +) -> PyResult> { + model_input_view::reference_tuple(classes, rows, cache) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs new file mode 100644 index 0000000000..5e485dbf78 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs @@ -0,0 +1,17 @@ +//! The classes the owner's views project attributes with. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::model_inputs::model_input_view; +use crate::compiled_views::models::InputViewClasses; +use pyo3::prelude::Py; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The classes the owner's views project attributes with. +#[inline] +pub fn view_classes<'py>( + owner: &Py, + py: Python<'py>, +) -> PyResult> { + model_input_view::view_classes(owner, py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs index d62361d306..4bb78901c7 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/audits.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs @@ -1,6 +1,6 @@ //! The project's compiled audits. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs index d4109dbf56..6cc7667444 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/collection.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs @@ -1,6 +1,6 @@ //! The project's resource collection named `field`; any other field is read by attribute. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs index d89f15e85d..095a175f91 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compile_inputs_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs @@ -1,7 +1,7 @@ //! The native owner's inputs view when it holds every model input; else a record. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs index 4ac5081e9b..e33fba8e33 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_audit_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs @@ -1,6 +1,6 @@ //! The audit as a `CompiledAudit`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs index 55b535f477..22243f5df2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_function_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs @@ -1,6 +1,6 @@ //! The function as a `CompiledFunction`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs index 1aec08f0c2..62876a2a3a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_model_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs @@ -1,6 +1,6 @@ //! The model as a `CompiledModel`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs similarity index 60% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs index 3084971bb6..e061552284 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs @@ -1,12 +1,12 @@ //! The project's compiled models, in compile order. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; /// The project's compiled models, in compile order. #[inline] -pub fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub fn compiled_models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { project_reads::models(project) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs index ba6c6ac76b..fcc6df7b04 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_project_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs @@ -1,7 +1,7 @@ //! The compile's retained project, or one recorded from a hand-built project. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs index 784aa64fdb..6dd53396d3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_seed_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs @@ -1,6 +1,6 @@ //! The seed as a `CompiledSeed`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs index 001211900c..22765746a8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/compiled_source_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs @@ -1,6 +1,6 @@ //! The source as a `CompiledSource`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs index 1b23b33be0..c6725009b3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs @@ -1,6 +1,6 @@ //! Enum and constant declarations as native rows, in the given order. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs index 87b1d93dac..395c418e72 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/declaration_rows_function.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs @@ -1,6 +1,6 @@ //! `declaration_rows` as a Python function. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; /// `declaration_rows` as a Python function. #[inline] diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs index e828b81594..34b730bbe7 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/functions.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs @@ -1,6 +1,6 @@ //! The project's compiled functions. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs index c0bcd3fbe1..b632dbbea2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/inputs_with.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs @@ -1,6 +1,6 @@ //! The compile inputs with `changes`; a native view shares its owner's model inputs. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs similarity index 78% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs index 10dc87056b..f18ad74f48 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs @@ -1,17 +1,24 @@ +pub mod audits; +pub mod collection; pub mod compile_inputs_view; pub mod compiled_audit_record; pub mod compiled_function_record; pub mod compiled_model_record; +pub mod compiled_models; pub mod compiled_project_facts; pub mod compiled_seed_record; pub mod compiled_source_record; pub mod declaration_rows; pub mod declaration_rows_function; +pub mod functions; pub mod inputs_with; pub mod plain_typed_value; pub mod project_with; pub mod record_model_analyses; pub mod register_records; pub mod replace; +pub mod seeds; +pub mod sources; +pub mod sql_tests; pub mod with_binding_diagnostics; pub mod with_type_recovery; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs index c6e38b4743..e8d05bdfc1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/plain_typed_value.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs @@ -1,6 +1,6 @@ //! A typed SQL value as the plain value native consumers read. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs index b078c9acd5..db14f06169 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/project_with.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs @@ -1,6 +1,6 @@ //! The project with `changes`; a native view shares its owner's models. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs index bd5c8b11fe..477189486e 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/record_model_analyses.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs @@ -1,7 +1,7 @@ //! Transfer each assembled model's typed analysis to its retained facts; return how many. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs similarity index 84% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs index d431a08dc5..f9ff66dc81 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/register_records.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs @@ -1,6 +1,6 @@ //! Register the records functions and classes. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs index 112eec91fb..315b0e7cdc 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/replace.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs @@ -1,6 +1,6 @@ //! `dataclasses.replace(value, **changes)`. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs similarity index 81% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs index 2035f6fd86..bf99d00c84 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/seeds.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs @@ -1,6 +1,6 @@ //! The project's compiled seeds. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs index 8c142e37e9..433f25a2df 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sources.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs @@ -1,6 +1,6 @@ //! The project's compiled sources. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs similarity index 82% rename from crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs index 62008ee4de..9fb5eb97fb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/sql_tests.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs @@ -1,6 +1,6 @@ //! The project's compiled SQL tests. -use crate::compiled_views::_helpers::project_reads; +use crate::compiled_views::_helpers::project::project_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs index b666daaa7e..aa9b9f2a52 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_binding_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs @@ -1,6 +1,6 @@ //! The model with these final binding diagnostics. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs rename to crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs index 6a4253eeda..9494c5b900 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/records/with_type_recovery.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs @@ -1,6 +1,6 @@ //! The model with its recovered bindings and its poisoned outputs' types forgotten. -use crate::compiled_views::_helpers::records; +use crate::compiled_views::_helpers::project::records; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs deleted file mode 100644 index 6672cad240..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project_reads/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod audits; -pub mod collection; -pub mod functions; -pub mod models; -pub mod seeds; -pub mod sources; -pub mod sql_tests; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs deleted file mode 100644 index 29ef53105c..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_views/main/relation_keys/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod entry_relation_key_row; -pub mod model_key_row; -pub mod one_row_seed_names; -pub mod project_relation_keys; -pub mod register_relation_keys; -pub mod relation_keys; -pub mod source_key_input; diff --git a/crates/sqlbuild-python-project/src/compiled_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/mod.rs index 0cdb75ac22..937dcc3deb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/mod.rs @@ -1,4 +1,4 @@ -//! Python views of the compiled project: models, inputs, audits, functions, seeds, sources and their reads. +//! Python views of the compiled project: its models, model inputs, analysis, records and relation keys. pub(crate) mod _helpers; pub mod constants; diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index 5eac042e4e..75eb837deb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -1,69 +1,62 @@ //! Public records of compiled views. -use crate::compiled_project::main::rows::reference_row_tuple::reference_row_tuple; +use crate::compiled_project::main::reference_row_tuple::reference_row_tuple; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_project::models::ReferenceTuple; -use crate::compiled_views::_helpers::audit_views::AuditProjections; -use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisProjections; -use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisRecord; -use crate::compiled_views::_helpers::compiled_model_analysis::AnalysisViews; -use crate::compiled_views::_helpers::compiled_model_view::AuditedSchema; -use crate::compiled_views::_helpers::compiled_model_view::CompiledArguments; -use crate::compiled_views::_helpers::compiled_model_view::CompiledProjections; -use crate::compiled_views::_helpers::compiled_model_view::CompiledRecord; -use crate::compiled_views::_helpers::compiled_model_view::DestinationRow; -use crate::compiled_views::_helpers::compiled_model_view::MODEL_RESOURCE; -use crate::compiled_views::_helpers::compiled_model_view::ModelOverlay; -use crate::compiled_views::_helpers::compiled_model_view::NULLABLE; -use crate::compiled_views::_helpers::compiled_model_view::SQL_ANALYSIS_KEY; -use crate::compiled_views::_helpers::compiled_model_view::ViewProjections; -use crate::compiled_views::_helpers::compiled_model_view::casefolded_families; -use crate::compiled_views::_helpers::compiled_model_view::compiled_of; -use crate::compiled_views::_helpers::compiled_model_view::compiled_record; -use crate::compiled_views::_helpers::compiled_model_view::defaulted_destination; -use crate::compiled_views::_helpers::compiled_model_view::lineage_input; -use crate::compiled_views::_helpers::compiled_model_view::model_graph_row; -use crate::compiled_views::_helpers::compiled_model_view::model_name; -use crate::compiled_views::_helpers::compiled_model_view::object_key; -use crate::compiled_views::_helpers::compiled_model_view::relation_location; -use crate::compiled_views::_helpers::function_views::FunctionProjections; -use crate::compiled_views::_helpers::function_views::NOT_ASSEMBLED; -use crate::compiled_views::_helpers::function_views::read_function; -use crate::compiled_views::_helpers::inputs_view::InputsFields; -use crate::compiled_views::_helpers::model_input_view::InputViewClasses; -use crate::compiled_views::_helpers::model_input_view::attached_schema_name; -use crate::compiled_views::_helpers::model_input_view::authored_expansion; -use crate::compiled_views::_helpers::model_input_view::expansion_object; -use crate::compiled_views::_helpers::model_input_view::model_relation_key_row; -use crate::compiled_views::_helpers::model_input_view::query_sql_text; -use crate::compiled_views::_helpers::model_input_view::read_input; -use crate::compiled_views::_helpers::model_input_view::read_input_with_owner; -use crate::compiled_views::_helpers::model_input_view::references; -use crate::compiled_views::_helpers::model_input_view::rejected_opt_out; -use crate::compiled_views::_helpers::model_input_view::schema_entry; -use crate::compiled_views::_helpers::model_input_view::view_classes; -use crate::compiled_views::_helpers::project_view::CompilerHandles; -use crate::compiled_views::_helpers::project_view::ProjectFields; -use crate::compiled_views::_helpers::project_view::SCHEMA_ENTRY_ATTRIBUTE; -use crate::compiled_views::_helpers::project_view::resource_row; -use crate::compiled_views::_helpers::seed_views::SEED_RESOURCE; -use crate::compiled_views::_helpers::seed_views::SeedHandles; -use crate::compiled_views::_helpers::seed_views::read_seed; -use crate::compiled_views::_helpers::source_views::SOURCE_RESOURCE; -use crate::compiled_views::_helpers::source_views::SourceAssemblyRow; -use crate::compiled_views::_helpers::source_views::SourceProjections; -use crate::compiled_views::_helpers::source_views::read_source; -use crate::compiled_views::types::AttachedSource; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisViews; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::AuditedSchema; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledArguments; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DestinationRow; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::MODEL_RESOURCE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::ModelOverlay; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::NULLABLE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SQL_ANALYSIS_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::ViewProjections; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::casefolded_families; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_of; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_record; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::defaulted_destination; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::lineage_input; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_graph_row; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_name; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::object_key; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::relation_location; +use crate::compiled_views::_helpers::model_inputs::inputs_view::InputsFields; +use crate::compiled_views::_helpers::model_inputs::model_input_view::attached_schema_name; +use crate::compiled_views::_helpers::model_inputs::model_input_view::authored_expansion; +use crate::compiled_views::_helpers::model_inputs::model_input_view::expansion_object; +use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; +use crate::compiled_views::_helpers::model_inputs::model_input_view::query_sql_text; +use crate::compiled_views::_helpers::model_inputs::model_input_view::read_input; +use crate::compiled_views::_helpers::model_inputs::model_input_view::read_input_with_owner; +use crate::compiled_views::_helpers::model_inputs::model_input_view::references; +use crate::compiled_views::_helpers::model_inputs::model_input_view::rejected_opt_out; +use crate::compiled_views::_helpers::model_inputs::model_input_view::schema_entry; +use crate::compiled_views::_helpers::model_inputs::model_input_view::view_classes; +use crate::compiled_views::_helpers::project::project_view::CompilerHandles; +use crate::compiled_views::_helpers::project::project_view::ProjectFields; +use crate::compiled_views::_helpers::project::project_view::SCHEMA_ENTRY_ATTRIBUTE; +use crate::compiled_views::_helpers::project::project_view::TAGS_ATTRIBUTE; +use crate::compiled_views::_helpers::project::project_view::resource_row; use crate::compiled_views::types::DeclaredRow; use crate::compiled_views::types::LineageInput; use crate::compiled_views::types::ModelInputRecord; -use crate::compiled_views::types::SourceViews; -use crate::graph::main::project_graph::project_graph::project_graph; +use crate::graph::main::project::project_graph::project_graph; use crate::graph::models::NativeProjectGraph; use crate::graph::models::ResourceRow; +use crate::resource_views::models::AuditHandles; +use crate::resource_views::models::AuditProjections; +use crate::resource_views::models::FunctionHandles; +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::models::SourceHandles; +use crate::resource_views::models::SourceProjections; use pyo3::FromPyObject; -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyRuntimeError; use pyo3::exceptions::PyTypeError; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -91,7 +84,6 @@ use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_project::models::ModelMetadataTexts; use sqlbuild_project::types::InferredRow; -use sqlbuild_project::types::ObjectKeyRow; use sqlbuild_project::types::ProjectHost; use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; @@ -105,77 +97,20 @@ use std::fmt::Debug; use std::path::PathBuf; use std::sync::Arc; -/// An attached audit's user values. -pub struct AuditHandles { - /// The discovered (or built-in) audit file and block the SQL comes from. - pub audit_file: Py, - pub audit_block: Py, - /// The authored measurement policy specs, carrying user values. - pub thresholds: Py, - pub minimum_samples: Py, - pub evidence_limit: Py, - /// Usage records, the identities the scope index is built from. - pub usages: Py, -} - -/// One attached audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeAuditInput { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeAuditInput { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -/// One assembled audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeCompiledAudit { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeCompiledAudit { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } +/// Python projections of the analysis, each built once on first access. +pub struct AnalysisProjections { + pub(crate) lineage: PyOnceLock>, + pub(crate) binding: PyOnceLock>, + pub(crate) proof: PyOnceLock>, } -impl NativeCompiledAudit { - /// The `CompiledAudit` field values, in field order, without building the record. - pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.scope_deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.definition_name(py)?.into_bound_py_any(py)?, - self.audit_file(py)?.into_bound(py), - self.audit_block(py)?.into_bound(py), - self.sql_body(py)?.into_bound_py_any(py)?, - self.evaluation_mode(py)?.into_bound(py), - self.measurement_contract(py)?.into_bound(py), - self.thresholds(py)?.into_bound(py), - self.minimum_samples(py)?.into_bound(py), - self.measure_sql(py)?.into_bound_py_any(py)?, - self.evidence_sql(py)?.into_bound_py_any(py)?, - self.evidence_limit(py)?.into_bound(py), - self.references(py)?.into_bound(py).into_any(), - self.attached_target_kind(py)?, - self.attached_target_name(py)?.into_bound_py_any(py)?, - self.attached_column_name(py)?.into_bound_py_any(py)?, - self.severity(py)?, - self.run_scope(py)?.into_bound_py_any(py)?, - pyo3::types::PyBool::new(py, self.always_run(py)?) - .to_owned() - .into_any(), - self.description(py)?.into_bound_py_any(py)?, - self.meta(py)?.into_bound(py), - ]) +impl Default for AnalysisProjections { + fn default() -> Self { + Self { + lineage: PyOnceLock::new(), + binding: PyOnceLock::new(), + proof: PyOnceLock::new(), + } } } @@ -213,6 +148,21 @@ pub struct CompiledViewClasses<'py> { pub family_proof: Bound<'py, PyAny>, } +/// A compiled model's Python attribute projections, each built once on first access. +pub struct CompiledProjections { + pub(crate) key: PyOnceLock>, + pub(crate) deps: PyOnceLock>, +} + +impl Default for CompiledProjections { + fn default() -> Self { + Self { + key: PyOnceLock::new(), + deps: PyOnceLock::new(), + } + } +} + /// One compiled model, projected from the owner on demand. #[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] @@ -857,128 +807,6 @@ impl NativeCompiledModel { } } -/// An attached function's user values. -pub struct FunctionHandles { - /// The discovered function file. - pub function_file: Py, - /// Usage records, the identities the scope index is built from. - pub usages: Py, -} - -/// One attached function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeFunctionInput { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeFunctionInput { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -/// One assembled function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub struct NativeCompiledFunction { - pub(crate) owner: Py, - pub(crate) index: usize, - /// The destination and fingerprint destination with target defaults applied. - pub(crate) defaulted: Option<(DestinationFact, DestinationFact)>, - pub(crate) destination: PyOnceLock>, - pub(crate) fingerprint_destination: PyOnceLock>, -} - -impl std::fmt::Debug for NativeCompiledFunction { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("NativeCompiledFunction") - .finish_non_exhaustive() - } -} - -impl NativeCompiledFunction { - pub fn new(owner: Py, index: usize) -> Self { - Self::defaulted(owner, index, None) - } - - /// The function's project graph row: its key, the resources it reads and its tags. - pub fn graph_row(&self, py: Python<'_>) -> PyResult { - read_function(&self.owner, self.index, py, |function, _| { - let deps: &[ObjectKeyRow] = function - .deps - .as_deref() - .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(ResourceRow { - key: ( - function.attached.node_type().to_owned(), - function.attached.name.clone(), - ), - deps: deps.to_vec(), - tags: function.attached.tags.clone(), - folder: None, - }) - }) - } - - pub(crate) fn defaulted( - owner: Py, - index: usize, - defaulted: Option<(DestinationFact, DestinationFact)>, - ) -> Self { - Self { - owner, - index, - defaulted, - destination: PyOnceLock::new(), - fingerprint_destination: PyOnceLock::new(), - } - } - - /// The destination and fingerprint destination, with target defaults once applied. - pub fn current_destinations( - &self, - py: Python<'_>, - ) -> PyResult<(DestinationFact, DestinationFact)> { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_function(&self.owner, self.index, py, |function, _| { - Ok(( - function.attached.destination.clone(), - function.attached.fingerprint_destination.clone(), - )) - }) - } -} - -impl NativeCompiledFunction { - /// The `CompiledFunction` field values, in field order, without building the record. - pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.relative_path(py)?.into_bound(py), - self.arguments(py)?.into_bound(py).into_any(), - self.returns(py)?.into_bound_py_any(py)?, - self.body_sql(py)?.into_bound_py_any(py)?, - self.destination(py)?.into_bound(py), - self.fingerprint_destination(py)?.into_bound(py), - self.return_columns(py)?.into_bound(py).into_any(), - self.references(py)?.into_bound(py).into_any(), - self.language(py)?, - self.source_file_path(py)?.into_bound(py), - self.runtime_version(py)?.into_bound_py_any(py)?, - self.entry_point(py)?.into_bound_py_any(py)?, - self.packages(py)?.into_any(), - self.tags(py)?.into_any(), - self.description(py)?.into_bound_py_any(py)?, - ]) - } -} - /// One compile's attached inputs, its model inputs projected from the owner. #[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] @@ -1171,7 +999,20 @@ impl NativeInputsView { } } +/// The Python classes a model input's projected attributes are built with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct InputViewClasses<'py> { + pub compiled: CompiledViewClasses<'py>, + pub(crate) sql_reference: Bound<'py, PyAny>, + pub(crate) reference_kind: Bound<'py, PyAny>, + pub(crate) schema_model_entry: Bound<'py, PyAny>, + pub(crate) sql_expansion: Bound<'py, PyAny>, + pub(crate) expansion_span: Bound<'py, PyAny>, +} + /// A model schema's header columns and specs, which carry user-authored audit arguments. +#[derive(Debug)] pub struct SchemaHandles { pub columns: Arc, pub audits: Arc, @@ -1776,7 +1617,7 @@ impl NativeProjectView { Err(_) => { let tags: Bound<'_, PyAny> = native_field( &native_field(&seed, SCHEMA_ENTRY_ATTRIBUTE)?, - crate::compiled_views::_helpers::project_view::TAGS_ATTRIBUTE, + TAGS_ATTRIBUTE, )?; resource_row(&seed, Some(&tags))? } @@ -1787,10 +1628,7 @@ impl NativeProjectView { rows.push(match function.cast::() { Ok(native) => native.get().graph_row(py)?, Err(_) => { - let tags: Bound<'_, PyAny> = native_field( - &function, - crate::compiled_views::_helpers::project_view::TAGS_ATTRIBUTE, - )?; + let tags: Bound<'_, PyAny> = native_field(&function, TAGS_ATTRIBUTE)?; resource_row(&function, Some(&tags))? } }); @@ -1866,387 +1704,3 @@ impl<'py> pyo3::IntoPyObject<'py> for RelationKeyRow { ) } } - -/// `(database, schema, logical database, logical schema)` of a seed's destination. -#[derive(Debug)] -pub struct SeedNamespace { - pub database: Option, - pub schema: Option, - pub logical_database: Option, - pub logical_schema: Option, -} - -impl<'py> pyo3::FromPyObject<'py> for SeedNamespace { - fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { - let (database, schema, logical_database, logical_schema): ( - Option, - Option, - Option, - Option, - ) = pyo3::types::PyAnyMethods::extract(value)?; - Ok(Self { - database, - schema, - logical_database, - logical_schema, - }) - } -} - -/// A seed's name, CSV file, declaration and declaring schema file. -#[derive(Debug)] -pub struct SeedParts<'a> { - pub name: &'a str, - pub seed_file: &'a Py, - pub schema_entry: &'a Py, - pub schema_file: &'a Py, -} - -/// One attached seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeSeedInput { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeSeedInput { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSeedInput { - #[getter] - pub fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } -} - -/// One assembled seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub struct NativeCompiledSeed { - pub(crate) owner: Py, - pub(crate) index: usize, - /// The destination with target defaults applied, replacing the assembled one. - pub(crate) defaulted: Option, - pub(crate) destination: PyOnceLock>, -} - -impl std::fmt::Debug for NativeCompiledSeed { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("NativeCompiledSeed") - .finish_non_exhaustive() - } -} - -impl NativeCompiledSeed { - pub fn new(owner: Py, index: usize) -> Self { - Self { - owner, - index, - defaulted: None, - destination: PyOnceLock::new(), - } - } - - /// The seed's project graph row: its key and its declared tags. - pub fn graph_row(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, py| { - let mut tags: Vec = Vec::new(); - for tag in native_field( - seed.host.schema_entry.bind(py), - crate::compiled_views::_helpers::seed_views::TAGS_ATTRIBUTE, - )? - .try_iter()? - { - tags.push(tag?.str()?.extract()?); - } - Ok(ResourceRow { - key: (SEED_RESOURCE.to_owned(), seed.name.clone()), - deps: Vec::new(), - tags, - folder: None, - }) - }) - } - - pub fn current_destination(&self, py: Python<'_>) -> PyResult { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_seed(&self.owner, self.index, py, |seed, _| { - seed.destination - .clone() - .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) - }) - } -} - -#[pymethods] -impl NativeCompiledSeed { - /// The seed's planner-facing `CompiledObjectKey`. - #[getter] - pub fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_seed(&self.owner, self.index, py, |seed, py| { - seed.host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SEED_RESOURCE, &seed.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub fn name(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) - } - - #[getter] - pub fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } - - /// The seed's planner-facing `CompiledRelationLocation`. - #[getter] - pub fn destination(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.destination - .get_or_try_init(py, || { - relation_location(&classes.compiled, &self.current_destination(py)?) - }) - .map(|destination| destination.clone_ref(py)) - } - - /// This seed with the adapter's default schema and database filled into its destination. - pub fn with_target_defaults( - &self, - py: Python<'_>, - defaults: (Option, Option), - render: Bound<'_, PyAny>, - ) -> PyResult { - let defaulted: DestinationFact = - defaulted_destination(&self.current_destination(py)?, defaults, &render)?; - Ok(Self { - owner: self.owner.clone_ref(py), - index: self.index, - defaulted: Some(defaulted), - destination: PyOnceLock::new(), - }) - } - - #[getter(external)] - pub fn is_external(&self) -> bool { - false - } -} - -impl NativeCompiledSeed { - /// The `CompiledSeed` field values, in field order, without building the record. - pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.seed_file(py)?.into_bound(py), - self.schema_entry(py)?.into_bound(py), - self.schema_file(py)?.into_bound(py), - self.destination(py)?.into_bound(py), - pyo3::types::PyBool::new(py, self.is_external()) - .to_owned() - .into_any(), - ]) - } -} - -/// An attached source's user values. -pub struct SourceHandles { - /// The discovered source file. - pub source_file: Py, - /// The discovered entry; its spec carries user-authored metadata. - pub entry: Py, - /// Template-expanded user metadata. - pub meta: Py, - /// Template-expanded column specs, which carry user metadata and audit arguments. - pub columns: Py, - /// Template-expanded table audit specs, which carry user audit arguments. - pub audits: Py, - /// Usage records, the identities the scope index is built from. - pub usages: Py, -} - -/// One attached source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeSourceInput { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeSourceInput { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSourceInput { - #[getter] - pub fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.attached_entry(py) - }) - } - - #[getter] - pub fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } - - #[getter] - pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.usages.clone_ref(py)) - }) - } - - /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. - #[getter] - pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, py| { - let attached: &AttachedSource = &source.attached; - Ok(SourceAssemblyRow { - name: attached.name.clone(), - managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), - database: attached.database.clone(), - schema: attached.schema.clone(), - }) - }) - } -} - -/// One assembled source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub struct NativeCompiledSource { - pub(crate) owner: Py, - pub(crate) index: usize, -} - -impl NativeCompiledSource { - pub fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } - - /// The source's project graph row: its key, with no dependencies, tags or folder. - pub fn graph_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(ResourceRow { - key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - deps: Vec::new(), - tags: Vec::new(), - folder: None, - }) - }) - } -} - -#[pymethods] -impl NativeCompiledSource { - /// The source's planner-facing `CompiledObjectKey`. - #[getter] - pub fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_source(&self.owner, self.index, py, |source, py| { - source - .host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub fn name(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(source.attached.name.clone()) - }) - } - - #[getter] - pub fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.assembled_entry(py) - }) - } - - #[getter] - pub fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } -} - -impl NativeCompiledSource { - /// The `CompiledSource` field values, in field order, without building the record. - pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.source_entry(py)?.into_bound(py), - self.source_file(py)?.into_bound(py), - ]) - } -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/types.rs b/crates/sqlbuild-python-project/src/compiled_views/types.rs index e65215546d..5836b137ee 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/types.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/types.rs @@ -1,33 +1,14 @@ //! Public traits and type aliases of compiled views. -use crate::compiled_views::_helpers::source_views::SourceKeyFacts; use crate::compiled_views::models::PyProjectHost; -use crate::compiled_views::models::SeedNamespace; -use crate::compiled_views::models::SeedParts; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; use sqlbuild_project::models as project_models; -/// What the audit loop attached for one singular audit block or attached generic audit. -pub type AttachedAudit = project_models::AttachedAudit; - -/// One attached audit held by the compiled project. -pub type AuditRecord = project_models::AuditRecord; - /// `(name, type)` of a declared column. pub type DeclaredRow = (String, Option); /// `(output column, [(resource name, column name)])` of one lineage output. pub type LineageInput = (String, Vec<(String, String)>); -/// What the function loop attached for one SQL or Python function. -pub type AttachedFunction = project_models::AttachedFunction; - -/// One attached function held by the compiled project. -pub type FunctionRecord = project_models::FunctionRecord; - /// A model's attached header schema; the host keeps its columns, audits and dynamic columns. pub type SchemaRecord = project_models::SchemaRecord; @@ -36,33 +17,3 @@ pub type ModelInputRecord = project_models::ModelInputRecord; /// Declared keys by `kind:name` identity, in identity order. pub type RelationKeys = Vec<(String, Vec>)>; - -/// One seed CSV paired with its declaration; the discovered objects are discovery's. -pub type SeedRecord = project_models::SeedRecord; - -/// What a seed record reads and keeps through its Python values. -pub trait SeedViews { - /// The seed's name, CSV file, declaration and declaring schema file. - fn parts(&self) -> SeedParts<'_>; - - /// Keep the destination native project assembly resolved for the seed. - fn set_namespace(&mut self, namespace: SeedNamespace); -} - -/// What the source loop attached for one source: its expanded metadata and usages. -pub type AttachedSource = project_models::AttachedSource; - -/// One attached source held by the compiled project. -pub type SourceRecord = project_models::SourceRecord; - -/// What a source record projects from its Python values, each built once. -pub trait SourceViews { - /// The source's declared column facts and audit key rows, read once from its expanded specs. - fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; - - /// The attached `SourceEntry`: the discovered entry with its expanded metadata. - fn attached_entry(&self, py: Python<'_>) -> PyResult>; - - /// The assembled `SourceEntry`: the attached entry in its resolved namespace. - fn assembled_entry(&self, py: Python<'_>) -> PyResult>; -} diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs index a6d408cadc..a5d31f1819 100644 --- a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs @@ -19,8 +19,8 @@ use sqlbuild_core::constants::STAGED_PROMOTION_MODE; use sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::main::model_reads::enum_columns::enum_columns; -use crate::compiled_views::main::records::compiled_project_facts::compiled_project_facts; +use crate::compiled_views::main::compiled_models::enum_columns::enum_columns; +use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; use crate::contracts::_helpers::model_contracts::ConflictRow; use crate::contracts::_helpers::model_contracts::DiagnosticRow; use crate::contracts::_helpers::model_contracts::evaluate_project_model_contracts; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs index 11fd00d1b8..5421ba0f18 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs @@ -1,9 +1,3 @@ -pub(crate) mod algorithms; -pub(crate) mod execution_gates; -pub(crate) mod getters; -pub(crate) mod keys; -pub(crate) mod project_graph; -pub(crate) mod python_lifecycle; -pub(crate) mod python_node_graph; -pub(crate) mod python_selection; -pub(crate) mod views; +pub(crate) mod nodes; +pub(crate) mod project; +pub(crate) mod python_nodes; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/keys.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/keys.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs new file mode 100644 index 0000000000..34146c3c53 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod algorithms; +pub(crate) mod execution_gates; +pub(crate) mod keys; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/getters.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/getters.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/getters.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/project/getters.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs new file mode 100644 index 0000000000..e779cfe25c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod getters; +pub(crate) mod project_graph; +pub(crate) mod views; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/project_graph.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/views.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs similarity index 99% rename from crates/sqlbuild-python-project/src/graph/_helpers/views.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs index 441551c7d0..0eb817a63d 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/views.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::{GraphIndexes, ProjectGraph}; -use crate::graph::_helpers::keys::native_key; +use crate::graph::_helpers::nodes::keys::native_key; use crate::graph::models::KeyCache; use crate::graph::models::NativeProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs new file mode 100644 index 0000000000..4388c92d11 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod python_lifecycle; +pub(crate) mod python_node_graph; +pub(crate) mod python_selection; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs similarity index 100% rename from crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs similarity index 98% rename from crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index 9906d242ad..7eb1c11e83 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -8,8 +8,8 @@ use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNod use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use crate::graph::_helpers::python_selection::resolve_python_sql_run_selection; -use crate::graph::_helpers::views::compiled_graph_keys; +use crate::graph::_helpers::project::views::compiled_graph_keys; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs similarity index 99% rename from crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index c9e90a972f..4d4c79fced 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -21,7 +21,7 @@ use sqlbuild_analysis::python_selection::models::{ SqlSelectionGraph, }; -use crate::compiled_project::main::job_catalog::needed_jobs::needed_jobs; +use crate::compiled_project::main::needed_jobs::needed_jobs; use crate::graph::models::CompiledProjectGraph; use crate::graph::models::NativeProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; diff --git a/crates/sqlbuild-python-project/src/graph/constants.rs b/crates/sqlbuild-python-project/src/graph/constants.rs index 3d1c7c916d..574ce12638 100644 --- a/crates/sqlbuild-python-project/src/graph/constants.rs +++ b/crates/sqlbuild-python-project/src/graph/constants.rs @@ -1,6 +1,6 @@ //! Public constants of graph. -use crate::graph::_helpers::getters::read_compiled_project_graph; +use crate::graph::_helpers::project::getters::read_compiled_project_graph; use crate::graph::models::CompiledProjectGraph; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs b/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs deleted file mode 100644 index af1ff0b160..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/algorithms/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod closure; -pub mod inverted; -pub mod register_algorithms; -pub mod transitive_closure_many; diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs b/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs deleted file mode 100644 index d3bd9b5e81..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/execution_gates/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod model_python_hook_names; -pub mod register_execution_gates; -pub mod sql_ref_key; diff --git a/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs b/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs deleted file mode 100644 index b92019db42..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/keys/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod native_key; diff --git a/crates/sqlbuild-python-project/src/graph/main/mod.rs b/crates/sqlbuild-python-project/src/graph/main/mod.rs index 139c906297..5db9403881 100644 --- a/crates/sqlbuild-python-project/src/graph/main/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/main/mod.rs @@ -1,8 +1,3 @@ -pub mod algorithms; -pub mod execution_gates; -pub mod keys; -pub mod project_graph; -pub mod python_lifecycle; -pub mod python_node_graph; -pub mod python_selection; -pub mod views; +pub mod nodes; +pub mod project; +pub mod python_nodes; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs similarity index 91% rename from crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs index 659cac1686..7247706e2a 100644 --- a/crates/sqlbuild-python-project/src/graph/main/algorithms/closure.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs @@ -1,6 +1,6 @@ //! Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -use crate::graph::_helpers::algorithms; +use crate::graph::_helpers::nodes::algorithms; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs index 7dceb79c46..a735594aeb 100644 --- a/crates/sqlbuild-python-project/src/graph/main/algorithms/inverted.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs @@ -1,6 +1,6 @@ //! Reversed edges: every source key first, then each new target in first-seen order. -use crate::graph::_helpers::algorithms; +use crate::graph::_helpers::nodes::algorithms; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs new file mode 100644 index 0000000000..c79e1b9d5e --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs @@ -0,0 +1,8 @@ +pub mod closure; +pub mod inverted; +pub mod model_python_hook_names; +pub mod native_key; +pub mod register_algorithms; +pub mod register_execution_gates; +pub mod sql_ref_key; +pub mod transitive_closure_many; diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs similarity index 90% rename from crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs index d58fedf433..fd5028aa03 100644 --- a/crates/sqlbuild-python-project/src/graph/main/execution_gates/model_python_hook_names.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs @@ -1,6 +1,6 @@ //! The Python hooks attached to one model, in lifecycle order without repeats. -use crate::graph::_helpers::execution_gates; +use crate::graph::_helpers::nodes::execution_gates; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs index 32bd82849b..9e3e75a810 100644 --- a/crates/sqlbuild-python-project/src/graph/main/keys/native_key.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs @@ -1,6 +1,6 @@ //! The native `(resource type, name)` pair of a `CompiledObjectKey`. -use crate::graph::_helpers::keys; +use crate::graph::_helpers::nodes::keys; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs similarity index 86% rename from crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs index c8f4b55a2a..c6dd33da76 100644 --- a/crates/sqlbuild-python-project/src/graph/main/algorithms/register_algorithms.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs @@ -1,6 +1,6 @@ //! Register the algorithms functions and classes. -use crate::graph::_helpers::algorithms; +use crate::graph::_helpers::nodes::algorithms; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs similarity index 86% rename from crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs index e4e4d60ad3..560abdebc1 100644 --- a/crates/sqlbuild-python-project/src/graph/main/execution_gates/register_execution_gates.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs @@ -1,6 +1,6 @@ //! Register the execution gates functions and classes. -use crate::graph::_helpers::execution_gates; +use crate::graph::_helpers::nodes::execution_gates; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs index d8fcec2bfb..63b9a30e04 100644 --- a/crates/sqlbuild-python-project/src/graph/main/execution_gates/sql_ref_key.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs @@ -1,6 +1,6 @@ //! The compiled graph key a typed model, source or seed reference names. -use crate::graph::_helpers::execution_gates; +use crate::graph::_helpers::nodes::execution_gates; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs similarity index 92% rename from crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs rename to crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs index a122938835..cb447c3b7b 100644 --- a/crates/sqlbuild-python-project/src/graph/main/algorithms/transitive_closure_many.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs @@ -1,6 +1,6 @@ //! Keys within `max_depth` edges of any start, plus the starts when requested. -use crate::graph::_helpers::algorithms; +use crate::graph::_helpers::nodes::algorithms; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs similarity index 91% rename from crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs rename to crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs index 83322b6b1c..a60832e5ac 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/compiled_graph_keys.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs @@ -1,6 +1,6 @@ //! `CompiledObjectKey`s for native graph key pairs. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use pyo3::prelude::Bound; use pyo3::prelude::PyResult; use pyo3::prelude::Python; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs rename to crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs index c63899f901..e686143404 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/edge_dict.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs @@ -1,6 +1,6 @@ //! Edge dict. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use crate::graph::models::KeyCache; use pyo3::prelude::Bound; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs similarity index 90% rename from crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs rename to crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs index b835290885..58e1f1ba79 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/graph_from_view_mappings.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs @@ -1,6 +1,6 @@ //! A native graph over Python's index mappings, keeping their order. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use crate::graph::models::GraphViewMappings; use crate::graph::models::NativeProjectGraph; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/mod.rs b/crates/sqlbuild-python-project/src/graph/main/project/mod.rs similarity index 75% rename from crates/sqlbuild-python-project/src/graph/main/views/mod.rs rename to crates/sqlbuild-python-project/src/graph/main/project/mod.rs index 22ae97ac95..d1aee43338 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/mod.rs @@ -2,5 +2,7 @@ pub mod compiled_graph_keys; pub mod edge_dict; pub mod graph_from_view_mappings; pub mod native_graph_from_edges; +pub mod project_graph; pub mod project_lineage_views; +pub mod register_project_graph; pub mod register_views; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs similarity index 91% rename from crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs rename to crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs index ae31746421..ca951027b7 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/native_graph_from_edges.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs @@ -1,6 +1,6 @@ //! A native graph holding these edges in their order, with no selector indexes. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use crate::graph::models::NativeProjectGraph; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs similarity index 88% rename from crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs rename to crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs index c9c83f9d74..1afb981d97 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project_graph/project_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs @@ -1,6 +1,6 @@ //! The project graph indexes of these resources, in project order. -use crate::graph::_helpers::project_graph; +use crate::graph::_helpers::project::project_graph; use crate::graph::models::NativeProjectGraph; use crate::graph::models::ResourceRow; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs similarity index 91% rename from crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs rename to crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs index 628079c7c3..f61255e529 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/project_lineage_views.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs @@ -1,6 +1,6 @@ //! The project's lineage edges and selector indexes, from its native graph. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs similarity index 86% rename from crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs rename to crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs index bb053d40c1..73da7dd6e7 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project_graph/register_project_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs @@ -1,6 +1,6 @@ //! Register the project graph functions and classes. -use crate::graph::_helpers::project_graph; +use crate::graph::_helpers::project::project_graph; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/views/register_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs similarity index 86% rename from crates/sqlbuild-python-project/src/graph/main/views/register_views.rs rename to crates/sqlbuild-python-project/src/graph/main/project/register_views.rs index fabae49000..b004bc34e4 100644 --- a/crates/sqlbuild-python-project/src/graph/main/views/register_views.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs @@ -1,6 +1,6 @@ //! Register the views functions and classes. -use crate::graph::_helpers::views; +use crate::graph::_helpers::project::views; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs b/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs deleted file mode 100644 index 523a463166..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/project_graph/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod project_graph; -pub mod register_project_graph; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs deleted file mode 100644 index 83049aebc7..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod build_python_sql_run_lifecycle; -pub mod register_python_lifecycle; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs deleted file mode 100644 index 34b988f757..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod python_node_graph; -pub mod register_python_node_graph; -pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs index a4d3f8863b..6b9b06f140 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/build_python_sql_run_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs @@ -1,6 +1,6 @@ //! `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -use crate::graph::_helpers::python_lifecycle; +use crate::graph::_helpers::python_nodes::python_lifecycle; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs new file mode 100644 index 0000000000..07eb988543 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs @@ -0,0 +1,8 @@ +pub mod build_python_sql_run_lifecycle; +pub mod python_node_graph; +pub mod python_node_names; +pub mod register_python_lifecycle; +pub mod register_python_node_graph; +pub mod register_python_selection; +pub mod resolve_python_check_excludes; +pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs similarity index 87% rename from crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs index 2d9e69b330..a24ec554af 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs @@ -1,6 +1,6 @@ //! The internal executable Python-node graph of discovered project inputs. -use crate::graph::_helpers::python_node_graph; +use crate::graph::_helpers::python_nodes::python_node_graph; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs index 60c8fe0ea4..6ebd5b8578 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_selection/python_node_names.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs @@ -1,6 +1,6 @@ //! The Python node names `select` minus `exclude` resolve to; every node when unselected. -use crate::graph::_helpers::python_selection; +use crate::graph::_helpers::python_nodes::python_selection; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs similarity index 84% rename from crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs index ece0f909a4..b634ee17cb 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_lifecycle/register_python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs @@ -1,6 +1,6 @@ //! Register the python lifecycle functions and classes. -use crate::graph::_helpers::python_lifecycle; +use crate::graph::_helpers::python_nodes::python_lifecycle; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs similarity index 84% rename from crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs index fe05b78fa1..15e439c377 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/register_python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs @@ -1,6 +1,6 @@ //! Register the python node graph functions and classes. -use crate::graph::_helpers::python_node_graph; +use crate::graph::_helpers::python_nodes::python_node_graph; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs similarity index 84% rename from crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs index fcdcebefff..94d80dec33 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_selection/register_python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs @@ -1,6 +1,6 @@ //! Register the python selection functions and classes. -use crate::graph::_helpers::python_selection; +use crate::graph::_helpers::python_nodes::python_selection; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs similarity index 89% rename from crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs index 7ef0c30a2e..5956df1a1e 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_selection/resolve_python_check_excludes.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs @@ -1,6 +1,6 @@ //! The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. -use crate::graph::_helpers::python_selection; +use crate::graph::_helpers::python_nodes::python_selection; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs index 7a3218c23c..dfb821cfd5 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_node_graph/run_selection_from_inputs.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs @@ -1,6 +1,6 @@ //! `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. -use crate::graph::_helpers::python_node_graph; +use crate::graph::_helpers::python_nodes::python_node_graph; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs deleted file mode 100644 index e98ca2796b..0000000000 --- a/crates/sqlbuild-python-project/src/graph/main/python_selection/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod python_node_names; -pub mod register_python_selection; -pub mod resolve_python_check_excludes; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index e23602141e..de033e9274 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -1,15 +1,15 @@ //! Public records of graph. -use crate::graph::_helpers::project_graph::IndexesRow; -use crate::graph::_helpers::project_graph::KeysOutcome; -use crate::graph::_helpers::project_graph::UniqueIdMap; -use crate::graph::_helpers::project_graph::dependency_edges; -use crate::graph::_helpers::project_graph::outcome; -use crate::graph::_helpers::project_graph::project_graph; -use crate::graph::_helpers::python_node_graph::python_node_graph; -use crate::graph::_helpers::python_selection::project_sources; -use crate::graph::_helpers::python_selection::selection_graph; -use crate::graph::_helpers::views::lineage_graph_views; +use crate::graph::_helpers::project::project_graph::IndexesRow; +use crate::graph::_helpers::project::project_graph::KeysOutcome; +use crate::graph::_helpers::project::project_graph::UniqueIdMap; +use crate::graph::_helpers::project::project_graph::dependency_edges; +use crate::graph::_helpers::project::project_graph::outcome; +use crate::graph::_helpers::project::project_graph::project_graph; +use crate::graph::_helpers::project::views::lineage_graph_views; +use crate::graph::_helpers::python_nodes::python_node_graph::python_node_graph; +use crate::graph::_helpers::python_nodes::python_selection::project_sources; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/lib.rs b/crates/sqlbuild-python-project/src/lib.rs index 79a2063373..7af797d10d 100644 --- a/crates/sqlbuild-python-project/src/lib.rs +++ b/crates/sqlbuild-python-project/src/lib.rs @@ -18,8 +18,10 @@ pub mod planner_phases; pub mod planner_scope; pub mod planner_selection; pub mod planner_strategy; +pub mod project_snapshot; pub mod references; pub mod relation_naming; +pub mod resource_views; pub mod semantic_shapes; pub mod sql_test_glue; pub mod type_system; diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index bcd17e0ee7..ae4fff6db3 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -21,22 +21,22 @@ use sqlbuild_analysis::planner_identity::models::{ AvailableUpstreams, ExecutionTest, GraphKey, MissingUpstream, }; -use crate::compiled_views::main::project_reads::functions::functions; -use crate::compiled_views::main::project_reads::models::models; -use crate::compiled_views::main::project_reads::seeds::seeds; -use crate::compiled_views::main::project_reads::sources::sources; -use crate::compiled_views::main::project_reads::sql_tests::sql_tests; -use crate::compiled_views::main::resource_reads::graph_row::graph_row; -use crate::graph::main::algorithms::closure::closure; -use crate::graph::main::algorithms::inverted::inverted; -use crate::graph::main::keys::native_key::native_key; -use crate::graph::main::views::edge_dict::edge_dict; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::closure::closure; +use crate::graph::main::nodes::inverted::inverted; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; use crate::graph::models::NativeProjectGraph; use crate::planner_identity::_helpers::hashing::graph_key; use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; use crate::references::models::GateRow; use crate::references::models::HookRow; +use crate::resource_views::main::resource_reads::graph_row::graph_row; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -103,7 +103,7 @@ pub(crate) fn execution_deps( ) -> PyResult)>> { let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); let collections: [Bound<'_, PyAny>; 4] = [ - models(project)?, + compiled_models(project)?, sources(project)?, seeds(project)?, functions(project)?, diff --git a/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs b/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs index 54dc4f9392..dc83f41ecb 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs @@ -2,7 +2,7 @@ use sqlbuild_host::entries::models::HostId; -use crate::compiled_views::main::project_reads::collection::collection; +use crate::compiled_views::main::project::collection::collection; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; use pyo3::{BoundObject, IntoPyObject}; @@ -147,7 +147,7 @@ pub(crate) fn by_name<'py>( let node = node?; pyo3::types::PyDictMethods::set_item( &nodes, - crate::compiled_views::main::resource_reads::name::name(&node)?, + crate::resource_views::main::resource_reads::name::name(&node)?, node, )?; } diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs index 8d8b0206b7..2c6bb4bacc 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs @@ -1,6 +1,6 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::graph::main::views::edge_dict::edge_dict; +use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; use crate::planner_scope::_helpers::values::dependency_names; use crate::planner_scope::_helpers::values::edges; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs index 40db90ad5a..51d8b6a255 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -38,7 +38,7 @@ pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { let item = item?; Ok(( item.get_item(0)?.extract()?, - crate::graph::main::keys::native_key::native_key(&item.get_item(1)?)?, + crate::graph::main::nodes::native_key::native_key(&item.get_item(1)?)?, )) }) .collect::>()?, diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs index 9892da4ce2..53f73cb188 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs @@ -1,8 +1,8 @@ //! Planner scope assembly using native graph and selector operations. -use crate::compiled_views::main::model_reads::name_object::name_object; -use crate::compiled_views::main::project_reads::sources::sources; -use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::sources::sources; use crate::graph::models::NativeProjectGraph; use crate::native_host::models::ModelHandle; use crate::planner_identity::main::execution_deps::execution_deps; @@ -14,6 +14,7 @@ use crate::planner_scope::models::ScopeData; use crate::planner_selection::main::py_strings::py_strings; use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; use crate::planner_selection::models::Carrier; +use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; @@ -84,7 +85,7 @@ pub(crate) fn build_planner_scope<'py>( let source = source?; if !native_field(&source_entry(&source)?, "loader")?.is_none() { managed.insert( - crate::compiled_views::main::source_reads::name::name(&source)?.extract()?, + crate::resource_views::main::source_reads::name::name(&source)?.extract()?, ); } } @@ -114,7 +115,7 @@ pub(crate) fn build_planner_scope<'py>( let mut models: std::collections::HashMap = std::collections::HashMap::new(); let mut model_order: Vec = Vec::new(); - for model in crate::compiled_views::main::project_reads::models::models(&project)?.try_iter()? { + for model in compiled_models(&project)?.try_iter()? { let model = model?; let name: String = name_object(&model)?.extract()?; if !models.contains_key(&name) { diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs index aa0740f72d..9348f3b9c7 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs @@ -2,12 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::compiled_views::main::project_reads::sources::sources; -use crate::compiled_views::main::source_reads::source_entry::source_entry; -use crate::graph::main::keys::native_key::native_key; +use crate::compiled_views::main::project::sources::sources; +use crate::graph::main::nodes::native_key::native_key; use crate::planner_scope::_helpers::loaders::intermediate_map; use crate::planner_scope::_helpers::values::edges; use crate::planner_scope::_helpers::values::keys; +use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyDictMethods; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -45,7 +45,7 @@ pub(crate) fn source_load_map<'py>( let source = source?; if let Some(keys) = &relevant && !keys.contains(&native_key( - &crate::compiled_views::main::source_reads::key::key(&source)?, + &crate::resource_views::main::source_reads::key::key(&source)?, )?) { continue; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs index 3d6bb63a4a..34cdb9d8bd 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs @@ -1,8 +1,8 @@ //! Python function identity and compiled graph conversion. -use crate::compiled_views::main::project_reads::sources::sources; -use crate::compiled_views::main::source_reads::source_entry::source_entry; -use crate::graph::main::keys::native_key::native_key; +use crate::compiled_views::main::project::sources::sources; +use crate::graph::main::nodes::native_key::native_key; +use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; diff --git a/crates/sqlbuild-python-project/src/planner_scope/models.rs b/crates/sqlbuild-python-project/src/planner_scope/models.rs index a58d239e0a..91a26242c2 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/models.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/models.rs @@ -1,6 +1,6 @@ //! Public records of planner scope. -use crate::graph::main::views::edge_dict::edge_dict; +use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; use crate::native_host::models::ModelHandle; use pyo3::IntoPyObjectExt; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs index e09b274fbd..cbcf3939a9 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::planner_selection::models::{ ModelSelectorNames, SelectableModel, SelectorExpansion, }; +use crate::compiled_views::main::project::compiled_models::compiled_models; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -87,11 +88,10 @@ pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( term: &Bound<'py, PyString>, ) -> PyResult> { let py: Python<'py> = project.py(); - let models: Vec = - crate::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - .map(|model| selectable_model(&model?)) - .collect::>()?; + let models: Vec = compiled_models(project)? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { Ok(models .iter() diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs index 4bf3f48784..cc380deaaf 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs @@ -10,9 +10,9 @@ use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_to use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; -use crate::compiled_project::main::job_catalog::graph_with_jobs::graph_with_jobs; -use crate::graph::main::algorithms::closure::closure; -use crate::graph::main::views::graph_from_view_mappings::graph_from_view_mappings; +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::graph::main::nodes::closure::closure; +use crate::graph::main::project::graph_from_view_mappings::graph_from_view_mappings; use crate::graph::models::CompiledProjectGraph; use crate::graph::models::GraphViewMappings; use crate::graph::models::KeyCache; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 117aa1d658..0eaea0c386 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -12,7 +12,7 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; -use crate::compiled_project::main::job_catalog::job_names::job_names; +use crate::compiled_project::main::job_names::job_names; use crate::planner_selection::_helpers::carrier::py_strings; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs similarity index 86% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs index 91d994e8e3..cf62a44560 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs @@ -1,5 +1,6 @@ //! Relation fixture completion over the Rust fixture planning context. +use crate::planner_strategy::models::FixtureCompletion; use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; @@ -8,32 +9,24 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; -use crate::planner_strategy::_helpers::fixture_context::planning_context as project_planning_context; -use crate::planner_strategy::_helpers::fixture_preparation::PreparedFixtures; -use crate::planner_strategy::_helpers::fixture_preparation::prepare; -use crate::planner_strategy::_helpers::fixture_reads::required_fixture_columns; -use crate::planner_strategy::_helpers::fixture_selection::fixture_keys_requiring_analysis; -use crate::planner_strategy::_helpers::fixture_selection::unknown_fixture_column_diagnostics; -use crate::planner_strategy::_helpers::fixture_sql::CompletionSql; -use crate::planner_strategy::_helpers::fixture_sql::render_completed; -use crate::planner_strategy::_helpers::fixture_types::expected_fixture_types; -use crate::planner_strategy::_helpers::fixture_values::columns; -use crate::planner_strategy::_helpers::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixture_values::label; -use crate::planner_strategy::_helpers::fixture_values::record; +use crate::planner_strategy::_helpers::fixtures::fixture_context::planning_context as project_planning_context; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::prepare; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::required_fixture_columns; +use crate::planner_strategy::_helpers::fixtures::fixture_selection::fixture_keys_requiring_analysis; +use crate::planner_strategy::_helpers::fixtures::fixture_selection::unknown_fixture_column_diagnostics; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::CompletionSql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::render_completed; +use crate::planner_strategy::_helpers::fixtures::fixture_types::expected_fixture_types; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixtures::fixture_values::label; +use crate::planner_strategy::_helpers::fixtures::fixture_values::record; use crate::planner_strategy::models::ExpectedTypes; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::records::types::RecordId; -/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. -pub struct FixtureCompletion<'py, 'a> { - pub sql_by_key: Bound<'py, PyDict>, - pub(crate) inferred_by_fixture: Bound<'py, PyDict>, - pub(crate) expected_types: ExpectedTypes<'a>, - pub diagnostics: Vec>, -} - impl ExpectedTypes<'_> { /// The relation's expected column types: the chain's propagated types, else the context's. pub(crate) fn get(&self, key: &FixtureKey) -> Option<&[(String, String)]> { diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs similarity index 90% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs index 6191648c71..6da49ca819 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs @@ -11,19 +11,16 @@ use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use crate::analysis_session::models::FixtureColumnInferences; -use crate::compiled_views::main::model_reads::name_object::name_object; -use crate::compiled_views::main::project_reads::seeds::seeds; -use crate::compiled_views::main::project_reads::sources::sources; -use crate::compiled_views::main::resource_reads::name::name; -use crate::compiled_views::main::seed_reads::schema_entry::schema_entry; -use crate::compiled_views::main::source_reads::source_entry::source_entry; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; use crate::native_host::models::ModelHandle; -use crate::planner_strategy::_helpers::fixture_metadata::model_fixture_relation; -use crate::planner_strategy::_helpers::fixture_reads::lineage_reads; -use crate::planner_strategy::_helpers::fixture_types::projections; -use crate::planner_strategy::_helpers::fixture_values::columns; -use crate::planner_strategy::_helpers::fixture_values::folded_name; -use crate::planner_strategy::_helpers::types::FixtureRelationInput; +use crate::planner_strategy::_helpers::fixtures::fixture_metadata::model_fixture_relation; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::lineage_reads; +use crate::planner_strategy::_helpers::fixtures::fixture_types::projections; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::planner_strategy::constants::SEED_REFERENCE_KIND; use crate::planner_strategy::constants::SOURCE_REFERENCE_KIND; @@ -32,6 +29,10 @@ use crate::planner_strategy::models::FixtureRelation; use crate::planner_strategy::models::ModelFixtureFacts; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; +use crate::planner_strategy::types::FixtureRelationInput; +use crate::resource_views::main::resource_reads::name::name; +use crate::resource_views::main::seed_reads::schema_entry::schema_entry; +use crate::resource_views::main::source_reads::source_entry::source_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; static FIXTURE_READS: PyOnceLock> = PyOnceLock::new(); @@ -64,9 +65,7 @@ pub(crate) fn planning_context( .collect::>()?, None => { let mut models: HashMap = HashMap::new(); - for model in - crate::compiled_views::main::project_reads::models::models(project)?.try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; models.insert(name_object(&model)?.extract()?, ModelHandle::read(&model)); } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs similarity index 82% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs index 86584191bb..e874a437e8 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_metadata.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs @@ -4,11 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use crate::compiled_views::main::model_reads::schema_entry::schema_entry; -use crate::compiled_views::main::model_reads::star_flags::star_flags; -use crate::planner_strategy::_helpers::fixture_values::columns; -use crate::planner_strategy::_helpers::fixture_values::folded_name; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::schema_entry::schema_entry; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; use crate::planner_strategy::constants::ENFORCED_CONTRACT; use crate::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs similarity index 93% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs index 5f8374c2dc..48a75e89cd 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs @@ -4,9 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; -use crate::planner_strategy::_helpers::fixture_sql::complete_empty_fixture_sql; -use crate::planner_strategy::_helpers::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixture_values::label; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::complete_empty_fixture_sql; +use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixtures::fixture_values::label; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs similarity index 92% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs index 256bffbe43..89871c2c6c 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; -use crate::compiled_views::main::model_reads::dep_rows::dep_rows; -use crate::compiled_views::main::model_reads::lineage_facts::lineage_facts; -use crate::compiled_views::main::model_reads::proven_inputs::proven_inputs; -use crate::compiled_views::main::model_reads::query_sql::query_sql; -use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::compiled_views::main::compiled_models::dep_rows::dep_rows; +use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; +use crate::compiled_views::main::compiled_models::proven_inputs::proven_inputs; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; use crate::compiled_views::models::LineageFactRow; -use crate::planner_strategy::_helpers::fixture_context::work_counts; +use crate::planner_strategy::_helpers::fixtures::fixture_context::work_counts; use crate::planner_strategy::constants::HIGH_CONFIDENCE; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; @@ -65,7 +65,8 @@ pub(crate) fn required_fixture_columns( }; let mut references: Vec = Vec::new(); for reference in - crate::compiled_views::main::model_reads::references::references(&model)?.try_iter()? + crate::compiled_views::main::compiled_models::references::references(&model)? + .try_iter()? { let reference = reference?; references.push(FixtureReadReference { diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs similarity index 87% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs index df94f2d543..c62bee7701 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs @@ -9,10 +9,10 @@ use sqlbuild_analysis::planner_fixtures::main::{ unknown_columns::unknown_columns, }; -use crate::planner_strategy::_helpers::fixture_preparation::PreparedFixtures; -use crate::planner_strategy::_helpers::fixture_values::columns; -use crate::planner_strategy::_helpers::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixture_values::label; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixtures::fixture_values::label; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs similarity index 100% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs similarity index 98% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs index beb985c529..7a4ffe29ce 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs @@ -9,7 +9,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::compiled_views::main::model_reads::lineage_facts::lineage_facts; +use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; use crate::compiled_views::models::LineageFactRow; use crate::planner_strategy::constants::DIRECT_TRANSFORM; use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_values.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs similarity index 100% rename from crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_values.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs new file mode 100644 index 0000000000..1f9df7eb9f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod fixture_completion; +pub(crate) mod fixture_context; +pub(crate) mod fixture_metadata; +pub(crate) mod fixture_preparation; +pub(crate) mod fixture_reads; +pub(crate) mod fixture_selection; +pub(crate) mod fixture_sql; +pub(crate) mod fixture_types; +pub(crate) mod fixture_values; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs index c655c1598f..1d7e992f42 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs @@ -1,10 +1 @@ -pub(crate) mod fixture_completion; -pub(crate) mod fixture_context; -pub(crate) mod fixture_metadata; -pub(crate) mod fixture_preparation; -pub(crate) mod fixture_reads; -pub(crate) mod fixture_selection; -pub(crate) mod fixture_sql; -pub(crate) mod fixture_types; -pub(crate) mod fixture_values; -pub(crate) mod types; +pub(crate) mod fixtures; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs deleted file mode 100644 index e797065aaa..0000000000 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/types.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Fixture planning key and input shapes. - -/// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. -pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs index 490721a438..406816e11c 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs @@ -1,6 +1,6 @@ //! An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. -use crate::planner_strategy::_helpers::fixture_sql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql; use crate::planner_strategy::models::FixtureRelation; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs index 3afa4aa192..fd96d34ab4 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs @@ -1,7 +1,7 @@ //! Complete the fixture groups of one chain against the planning context. -use crate::planner_strategy::_helpers::fixture_completion; -use crate::planner_strategy::_helpers::fixture_completion::FixtureCompletion; +use crate::planner_strategy::_helpers::fixtures::fixture_completion; +use crate::planner_strategy::models::FixtureCompletion; use crate::planner_strategy::models::RelationFixturePlanningContext; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs index bfb98df764..6d24d3d769 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs @@ -1,6 +1,6 @@ //! Fixture SQL with its untyped null columns cast to the expected types. -use crate::planner_strategy::_helpers::fixture_sql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs index bdac8debc7..cf884a46fd 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs @@ -1,6 +1,6 @@ //! Count one batch resolution of SQL-test model chains. -use crate::planner_strategy::_helpers::fixture_context; +use crate::planner_strategy::_helpers::fixtures::fixture_context; use pyo3::prelude::PyResult; use pyo3::prelude::Python; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs index f3227f4772..75b022ec9a 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs @@ -1,6 +1,6 @@ //! The planning context of a project, over its models or the given name-to-model map. -use crate::planner_strategy::_helpers::fixture_context; +use crate::planner_strategy::_helpers::fixtures::fixture_context; use crate::planner_strategy::models::RelationFixturePlanningContext; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs index 5b6fbb8c7d..f60e68fdaf 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs @@ -1,6 +1,6 @@ //! Register the fixture completion functions and classes. -use crate::planner_strategy::_helpers::fixture_completion; +use crate::planner_strategy::_helpers::fixtures::fixture_completion; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs index 1142f6ae24..cb61136494 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs @@ -1,6 +1,6 @@ //! Register the fixture context functions and classes. -use crate::planner_strategy::_helpers::fixture_context; +use crate::planner_strategy::_helpers::fixtures::fixture_context; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs index 01fac20a6d..3328d77af0 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs @@ -1,6 +1,6 @@ //! Register the fixture reads functions and classes. -use crate::planner_strategy::_helpers::fixture_reads; +use crate::planner_strategy::_helpers::fixtures::fixture_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/models.rs b/crates/sqlbuild-python-project/src/planner_strategy/models.rs index ac462fd145..8bd1580a41 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/models.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/models.rs @@ -7,7 +7,7 @@ use crate::planner_strategy::types::FixtureKey; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::pyclass; -use pyo3::types::PyAny; +use pyo3::types::PyDict; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; use sqlbuild_analysis::planner_fixtures::models::FixtureTypeProjection; @@ -17,6 +17,15 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::Mutex; +/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. +#[derive(Debug)] +pub struct FixtureCompletion<'py, 'a> { + pub sql_by_key: Bound<'py, PyDict>, + pub inferred_by_fixture: Bound<'py, PyDict>, + pub(crate) expected_types: ExpectedTypes<'a>, + pub diagnostics: Vec>, +} + /// Per-extension cache replacing the Python fixture resolver's LRU. #[pyclass] #[derive(Debug, Default)] @@ -47,10 +56,10 @@ pub(crate) struct ModelFixtureFacts { #[derive(Debug)] pub struct RelationFixturePlanningContext { pub models_by_name: HashMap, - pub(crate) relations: HashMap, + pub relations: HashMap, pub authoritative_columns: HashMap>, pub expected_types: HashMap>, - pub fixture_inferences: Py, + pub fixture_inferences: Py, pub(crate) model_facts: Mutex>>, } @@ -64,48 +73,57 @@ pub(crate) struct ExpectedTypes<'a> { /// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. #[derive(Debug)] pub struct ModelEntryDecisions<'py> { - pub model: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub snapshot: Bound<'py, PyAny>, - pub change_result: Bound<'py, PyAny>, - pub action: Bound<'py, PyAny>, - pub reason: Bound<'py, PyAny>, - pub materialization_type: Bound<'py, PyAny>, - pub resolved_sql: Bound<'py, PyAny>, + pub model: Bound<'py, pyo3::types::PyAny>, + pub adapter: Bound<'py, pyo3::types::PyAny>, + pub snapshot: Bound<'py, pyo3::types::PyAny>, + pub change_result: Bound<'py, pyo3::types::PyAny>, + pub action: Bound<'py, pyo3::types::PyAny>, + pub reason: Bound<'py, pyo3::types::PyAny>, + pub materialization_type: Bound<'py, pyo3::types::PyAny>, + pub resolved_sql: Bound<'py, pyo3::types::PyAny>, pub sql_analysis_enabled: bool, - pub hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub cursor_column: Bound<'py, PyAny>, - pub cursor_input_relations: Bound<'py, PyAny>, + pub hooks: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub cursor_column: Bound<'py, pyo3::types::PyAny>, + pub cursor_input_relations: Bound<'py, pyo3::types::PyAny>, pub runtime_owned_cursor_bounds: bool, - pub cursor_bounds: Bound<'py, PyAny>, - pub microbatch_range: Bound<'py, PyAny>, - pub empty_cursor_inputs: Bound<'py, PyAny>, + pub cursor_bounds: Bound<'py, pyo3::types::PyAny>, + pub microbatch_range: Bound<'py, pyo3::types::PyAny>, + pub empty_cursor_inputs: Bound<'py, pyo3::types::PyAny>, pub waiting_on_empty_inputs: bool, - pub backfill: Bound<'py, PyAny>, - pub cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub invocation_time: Bound<'py, PyAny>, + pub backfill: Bound<'py, pyo3::types::PyAny>, + pub cursor_overrides: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub cursor_configs: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub invocation_time: Bound<'py, pyo3::types::PyAny>, } /// The project lookups that resolve one model's cursor inputs. #[derive(Debug)] pub struct CursorInputLookups<'py> { pub models_by_name: ModelLookup, - pub functions_by_name: Bound<'py, PyAny>, - pub source_map: Bound<'py, PyAny>, - pub source_warehouse_columns: Bound<'py, PyAny>, - pub model_locations: Bound<'py, PyAny>, - pub seed_locations: Bound<'py, PyAny>, - pub runtime_cursor_producer_names: Bound<'py, PyAny>, + pub functions_by_name: Bound<'py, pyo3::types::PyAny>, + pub source_map: Bound<'py, pyo3::types::PyAny>, + pub source_warehouse_columns: Bound<'py, pyo3::types::PyAny>, + pub model_locations: Bound<'py, pyo3::types::PyAny>, + pub seed_locations: Bound<'py, pyo3::types::PyAny>, + pub runtime_cursor_producer_names: Bound<'py, pyo3::types::PyAny>, } /// The relations and lineage that render and schedule selected audits. #[derive(Debug)] pub struct AuditPlanning<'py> { - pub model_locations: Bound<'py, PyAny>, - pub seed_locations: Bound<'py, PyAny>, - pub source_map: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub model_materializations: Bound<'py, PyAny>, + pub model_locations: Bound<'py, pyo3::types::PyAny>, + pub seed_locations: Bound<'py, pyo3::types::PyAny>, + pub source_map: Bound<'py, pyo3::types::PyAny>, + pub adapter: Bound<'py, pyo3::types::PyAny>, + pub model_materializations: Bound<'py, pyo3::types::PyAny>, pub lineage: Py, } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/types.rs b/crates/sqlbuild-python-project/src/planner_strategy/types.rs index f662e8ce6d..b631369d0b 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/types.rs @@ -2,3 +2,6 @@ /// `(resource type value, name)` of one fixture relation. pub type FixtureKey = (String, String); + +/// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. +pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs new file mode 100644 index 0000000000..a397cc9af2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs new file mode 100644 index 0000000000..d930a14b58 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_roles; +pub(crate) mod snapshot_worker; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs similarity index 92% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs index 6d78d47b1a..050e2e0a07 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs @@ -1,6 +1,6 @@ //! Project metadata stays native until its immutable snapshot is sent over HTTP. -use crate::compiled_project::models::NativeOrchestrationSnapshot; +use crate::project_snapshot::models::NativeOrchestrationSnapshot; use std::collections::BTreeMap; use pyo3::exceptions::PyValueError; @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use serde_json::Value; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::compiled_project::models::NativeOrchestrationWorker; +use crate::project_snapshot::models::NativeOrchestrationWorker; /// Duplicate resource identities must fail, rather than silently replacing an earlier node. pub(crate) fn insert_node( diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs similarity index 95% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs index f627d5815a..28838025f2 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs @@ -12,13 +12,13 @@ use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::compiled_project::_helpers::snapshot::destination; -use crate::compiled_project::_helpers::snapshot::insert_node; -use crate::compiled_project::_helpers::values::plain_value; +use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::models::NativeCompiledSeed; use crate::compiled_views::models::NativeProjectView; -use crate::compiled_views::types::SeedViews; +use crate::project_snapshot::_helpers::snapshot::snapshot::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::path_posix::path_posix; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_declarations.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs similarity index 96% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs index b27f86c1c6..8fdb23e755 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::compiled_project::_helpers::snapshot::dependency_keys; -use crate::compiled_project::_helpers::snapshot::destination; -use crate::compiled_project::_helpers::snapshot::insert_node; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeProjectView; use crate::compiled_views::types::ModelInputRecord; +use crate::project_snapshot::_helpers::snapshot::snapshot::dependency_keys; +use crate::project_snapshot::_helpers::snapshot::snapshot::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs index 91917d39d9..7272d0416f 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs @@ -7,17 +7,17 @@ use serde_json::{Value, json}; use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::compiled_project::_helpers::snapshot::dependency_keys; -use crate::compiled_project::_helpers::snapshot::destination; -use crate::compiled_project::_helpers::snapshot::insert_node; -use crate::compiled_project::_helpers::snapshot_boundary::boundary_nodes; -use crate::compiled_project::_helpers::snapshot_boundary::columns; -use crate::compiled_project::_helpers::snapshot_boundary::metadata; -use crate::compiled_project::_helpers::snapshot_boundary::relative_path; -use crate::compiled_project::_helpers::values::plain_value; +use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::models::NativeCompiledFunction; use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot::dependency_keys; +use crate::project_snapshot::_helpers::snapshot::snapshot::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::boundary_nodes; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::columns; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::metadata; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::relative_path; +use crate::resource_views::models::NativeCompiledFunction; pub(crate) fn resource_nodes( py: Python<'_>, diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_roles.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_worker.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_worker.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_worker.rs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs similarity index 92% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs index 3fef29043d..3b8a69038c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/compile_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs @@ -1,7 +1,7 @@ //! Compile declarations. -use crate::compiled_project::_helpers::snapshot_declarations; use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs index f165383417..753d6dc24e 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_roles/compile_project_roles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs @@ -1,7 +1,7 @@ //! Compile project roles. -use crate::compiled_project::_helpers::snapshot_roles; use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_roles; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use sqlbuild_analysis::graph::models::ProjectGraph; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs new file mode 100644 index 0000000000..ad14c41e07 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs @@ -0,0 +1,7 @@ +pub mod compile_declarations; +pub mod compile_project_roles; +pub mod model_nodes; +pub mod python_declarations; +pub mod register_snapshot; +pub mod relative_path; +pub mod resource_nodes; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs similarity index 92% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs index 522e48c1d9..b911aeae8b 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_models/model_nodes.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs @@ -1,8 +1,8 @@ //! Each model's snapshot node, and the `columns` document of ranges kept out of node identity. -use crate::compiled_project::_helpers::snapshot_models; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_models; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use serde_json::Map; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs index 2253891986..bde5be109c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_declarations/python_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs @@ -1,6 +1,6 @@ //! The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. -use crate::compiled_project::_helpers::snapshot_declarations; +use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs index 6889acfb6f..2517ab3690 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot/register_snapshot.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs @@ -1,6 +1,6 @@ //! Register the snapshot functions and classes. -use crate::compiled_project::_helpers::snapshot; +use crate::project_snapshot::_helpers::snapshot::snapshot; use pyo3::prelude::Bound; use pyo3::prelude::PyModule; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs similarity index 78% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs index 48158c1483..a0671cf6aa 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_boundary/relative_path.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs @@ -1,6 +1,6 @@ //! Relative path. -use crate::compiled_project::_helpers::snapshot_boundary; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs index 165cdc17f6..84701e6305 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/snapshot_resources/resource_nodes.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs @@ -1,8 +1,8 @@ //! Resource nodes. -use crate::compiled_project::_helpers::snapshot_resources; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_resources; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use sqlbuild_orchestration::snapshots::models::SnapshotNode; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/mod.rs new file mode 100644 index 0000000000..a111c35a51 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/mod.rs @@ -0,0 +1,6 @@ +//! The compiled-project snapshot written and read across processes: models, resources, declarations, roles and its worker. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/models.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs new file mode 100644 index 0000000000..d4dc746edd --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -0,0 +1,991 @@ +//! Public records of project snapshot. + +use crate::compiled_project::constants::NO_CHANGE_REASON; +use crate::compiled_project::constants::REPORT_ATTEMPTS; +use crate::compiled_project::constants::SKIP_ACTION; +use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use crate::compiled_project::constants::STORE_BUSY_STATUS; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::Completion; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::is_function; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::poisoned; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::severity; +use crate::project_snapshot::_helpers::snapshot::snapshot_worker::WorkerOwner; +use crate::project_snapshot::_helpers::snapshot::snapshot_worker::unavailable; +use crate::project_snapshot::types::NodeKeys; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyStringMethods; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use sqlbuild_host::models::SqlTestOutcome; +use sqlbuild_host::types::ResultEnum; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::runs::models::NodeOutcome; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::runs::models::RunCommand; +use sqlbuild_orchestration::runs::models::RunExecution; +use sqlbuild_orchestration::runs::models::RunOutcome; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_python_base::records::main::fields::field::field; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use uuid::Uuid; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationSnapshot { + pub input: SnapshotInput, +} + +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + pub(crate) fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + pub(crate) fn publish( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + py.detach(|| { + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + pub(crate) fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } +} + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub struct WorkerBuild { + pub worker: Py, + pub(crate) execution: RunExecution, + pub(crate) completed: Mutex>, + pub(crate) checks: Mutex>>, + /// The snapshot's audits and SQL tests, each with the node keys it checks. + pub(crate) attached: BTreeMap>, + /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. + pub(crate) blocked: BTreeMap>, + /// How many of each function's planned tests have still to report; a function is built before its tests run. + pub(crate) untested: Mutex>, + /// A built function's result, held until its tests report so a failing one can fail it instead. + pub(crate) held: Mutex>, + /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. + pub(crate) upstream: BTreeMap>, + /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. + pub(crate) causes: Mutex>, + /// The worst attached outcome of each node, for commands that only run tests or audits. + pub(crate) checked: Mutex>, + /// Whether the command returned, having run every check it planned, rather than being refused or raising. + pub(crate) returned: AtomicBool, + /// Why the worker refused the run before executing anything, reported as the run's status reason. + pub(crate) refusal: Mutex>, +} + +impl WorkerBuild { + pub fn new( + worker: &Bound<'_, NativeOrchestrationWorker>, + (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), + ) -> PyResult> { + let state = Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + checks: Mutex::default(), + attached, + blocked, + untested: Mutex::default(), + held: Mutex::default(), + upstream, + causes: Mutex::default(), + checked: Mutex::default(), + returned: AtomicBool::new(false), + refusal: Mutex::default(), + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; + } + Ok(state) + } + + pub fn command(&self) -> RunCommand { + self.execution.claim.request.command + } + + /// The server run this build executes. + pub fn run_id(&self) -> uuid::Uuid { + self.execution.claim.request.run_id + } + + /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. + pub(crate) fn checks_only(&self) -> bool { + matches!(self.command(), RunCommand::Test | RunCommand::Audit) + } + + /// Record that the command returned instead of being refused or raising. + pub fn command_returned(&self) { + self.returned.store(true, Ordering::SeqCst); + } + + pub fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) + } + + /// Record why the run is refused; the terminal report carries it as the run's status reason. + pub fn refuse(&self, reason: &str) -> PyResult<()> { + *self.refusal.lock().map_err(|_| poisoned())? = Some(reason.to_owned()); + Ok(()) + } + + pub fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject writers outside the frozen write set before any executor connection. + pub fn enforce_writers(&self, py: Python<'_>, keys: &[String]) -> Result<(), pyo3::PyErr> { + self.check_cancelled(py)?; + match keys + .iter() + .find(|key| !self.execution.claim.request.write_set.contains(*key)) + { + Some(key) => Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))), + None => Ok(()), + } + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub fn enforce_plan_writers(&self, pipeline: &Bound<'_, PyAny>) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + self.count_planned_tests(&plan)?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, (None, None)))?; + } + } + } + Ok(()) + } + + /// Count each function's tests this build plans, so a built function waits only for tests that will report. + pub(crate) fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + for entry in field(plan, "test_entries")?.try_iter()? { + let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); + for node in self.blocked.get(&test).into_iter().flatten() { + if is_function(node) { + *untested.entry(node.clone()).or_default() += 1; + } + } + } + Ok(()) + } + + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. + pub fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { + for key in keys { + self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; + } + Ok(()) + } + + /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. + pub fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { + for (key, why) in nodes { + self.complete( + py, + key, + ( + NodeOutcome::Skipped, + None, + (None, Some(format!("held back: {why}"))), + ), + )?; + } + Ok(()) + } + + /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. + pub(crate) fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, (error_code, error_message)): ( + NodeOutcome, + Option, + (Option, Option), + ), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + let skipped_because_of = match outcome { + NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { + Some(cause) => Some(cause.clone()), + None if !self.checks_only() => self.failed_upstream(key)?, + None => None, + }, + _ => None, + }; + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: skipped_because_of.clone(), + error_message: error_message.as_deref().map(redact_error_text), + }, + )?; + let cause = match outcome { + NodeOutcome::Failed => Some(key.to_owned()), + _ => skipped_because_of, + }; + if let Some(cause) = cause { + self.causes + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), cause); + } + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + self.start_key(py, &format!("{kind}:{name}")) + } + + /// Report a selected node as started unless its outcome is final, such as a planner skip. + pub fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + if self.execution.claim.request.selection.contains(key) + && !self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { + node_key: key.into(), + }, + )?; + } + Ok(()) + } + + /// Report a selected node's outcome when the command reports no executor result for it. + pub fn finish_key(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + self.complete(py, key, (outcome, None, (None, None))) + } + + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. + pub fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } + if self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.into(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), None); + if self.checks_only() { + for node in nodes { + self.start_key(py, node)?; + } + } + Ok(()) + } + + /// A test or audit finished; a check the snapshot does not record is not reported. + pub fn finish_attached(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if outcome != NodeOutcome::Skipped { + self.start_attached(py, key)?; + } + let outcome = if outcome == NodeOutcome::Failed + && self.worker.bind(py).get().refresh_cancellation(py)? + { + NodeOutcome::Cancelled + } else { + outcome + }; + if self.execution.claim.request.selection.contains(key) { + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; + } + if outcome == NodeOutcome::Skipped { + return Ok(()); + } + let mut checked = self.checked.lock().map_err(|_| poisoned())?; + for node in nodes { + let worst = checked.get(node).map_or(outcome, |previous| { + if severity(outcome) > severity(*previous) { + outcome + } else { + *previous + } + }); + checked.insert(node.clone(), worst); + } + Ok(()) + } + + /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. + pub(crate) fn failed_upstream(&self, key: &str) -> PyResult> { + let causes = self.causes.lock().map_err(|_| poisoned())?; + let mut seen: BTreeSet<&str> = BTreeSet::from([key]); + let mut pending: VecDeque<&str> = VecDeque::from([key]); + while let Some(node) = pending.pop_front() { + for upstream in self.upstream.get(node).into_iter().flatten() { + if let Some(cause) = causes.get(upstream) { + return Ok(Some(cause.clone())); + } + if seen.insert(upstream) { + pending.push_back(upstream); + } + } + } + Ok(None) + } + + /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. + pub(crate) fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py = result.py(); + let outcome: String = field(result, "outcome")?.str()?.extract()?; + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; + let name: String = field(result, "test_name")?.extract()?; + let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if !passed && !cancelled { + for model in self.blocked.get(&key).into_iter().flatten() { + self.complete( + py, + model, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + } + self.release_tested(py, &key) + } + + /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. + pub(crate) fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + let mut released: Vec<(String, Completion)> = Vec::new(); + { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + let mut held = self.held.lock().map_err(|_| poisoned())?; + for node in self.blocked.get(test).into_iter().flatten() { + let Some(remaining) = untested.get_mut(node) else { + continue; + }; + *remaining = remaining.saturating_sub(1); + if *remaining == 0 + && let Some(completion) = held.remove(node) + { + released.push((node.clone(), completion)); + } + } + } + for (node, completion) in released { + self.complete(py, &node, completion)?; + } + Ok(()) + } + + /// Publish a node's result, or hold a built function's while its tests have still to report. + pub(crate) fn complete_or_hold( + &self, + py: Python<'_>, + key: &str, + completion: Completion, + ) -> PyResult<()> { + let waiting = !self.checks_only() + && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) + && self + .untested + .lock() + .map_err(|_| poisoned())? + .get(key) + .is_some_and(|remaining| *remaining > 0); + if waiting { + self.held + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), completion); + return Ok(()); + } + self.complete(py, key, completion) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if key.starts_with(SQL_TEST_KEY_PREFIX) { + return self.finished_test(result); + } + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; + } + Ok(()) + } + + pub fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let key = format!("check:{name}"); + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.clone(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key, None); + Ok(()) + } + + /// A completion without our actual-execution start callback is a blocked check, not a failure. + pub fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = field(result, "node_name")?.extract()?; + let key = format!("check:{name}"); + let started = self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(&key); + let outcome = if !started { + NodeOutcome::Skipped + } else if field(result, "passed")?.is_truthy()? { + NodeOutcome::Success + } else if field(result, "warned")?.is_truthy()? { + NodeOutcome::Warn + } else if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + }; + self.finish_check(result.py(), &key, outcome) + } + + pub(crate) fn finish_check( + &self, + py: Python<'_>, + key: &str, + outcome: NodeOutcome, + ) -> PyResult<()> { + self.worker.bind(py).get().change( + py, + &format!("check-finished/{key}"), + RunChange::CheckFinished { + node_key: key.into(), + outcome, + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), Some(outcome)); + Ok(()) + } + + /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. + pub fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + let checks_only = self.checks_only(); + if checks_only && !cancelled { + let checked = self.checked.lock().map_err(|_| poisoned())?.clone(); + let returned = self.returned.load(Ordering::SeqCst); + for key in &self.execution.claim.request.selection { + let seen = checked.get(key).copied(); + let outcome = if returned { + Some(seen.unwrap_or(NodeOutcome::Success)) + } else { + seen.filter(|outcome| { + matches!(outcome, NodeOutcome::Failed | NodeOutcome::Cancelled) + }) + }; + if let Some(outcome) = outcome { + self.complete(py, key, (outcome, None, (None, None)))?; + } + } + } + let held: Vec<(String, Completion)> = + std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) + .into_iter() + .collect(); + for (key, completion) in held { + self.complete(py, &key, completion)?; + } + let returned = self.returned.load(Ordering::SeqCst); + let build = self.command().is_build(); + let source_needs_no_run = |key: &str| build && key.starts_with("source:"); + for key in &self.execution.claim.request.selection { + let outcome = if cancelled { + NodeOutcome::Cancelled + } else if returned && source_needs_no_run(key) { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(py, key, (outcome, None, (None, None)))?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let outcomes: Vec = completed.values().copied().collect(); + let good = completed.iter().any(|(key, value)| { + matches!(value, NodeOutcome::Success | NodeOutcome::Warn) + && !source_needs_no_run(key) + }); + let bad = outcomes.iter().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if outcomes.contains(&NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + let unfinished: Vec = self + .checks + .lock() + .map_err(|_| poisoned())? + .iter() + .filter(|(_, outcome)| outcome.is_none()) + .map(|(key, _)| key.clone()) + .collect(); + for key in unfinished { + self.finish_check( + py, + &key, + if outcome == RunOutcome::Cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + )?; + } + let reason: String = self + .refusal + .lock() + .map_err(|_| poisoned())? + .clone() + .unwrap_or_else(|| "worker executor completed and closed its connections".into()); + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason, + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationWorker { + pub(crate) owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. + pub(crate) fn with_owner( + &self, + py: Python<'_>, + work: impl FnOnce(&WorkerOwner) -> PyResult + Send, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + work(owner.as_ref().ok_or_else(unavailable)?) + }) + } + + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + self.with_owner(py, |owner| Ok(owner.session.execution.clone())) + } + + /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. + pub fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + self.with_owner(py, |owner| { + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(error) + if attempts < REPORT_ATTEMPTS + && match &error { + WorkerError::Transport(_) => true, + WorkerError::Rejected(status) => { + status.as_u16() == STORE_BUSY_STATUS + } + _ => false, + } => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + pub(crate) fn execution_json(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + pub fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + pub(crate) fn report( + &self, + py: Python<'_>, + event_id: &str, + change_json: &str, + ) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + self.with_owner(py, |owner| { + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + pub(crate) fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/types.rs b/crates/sqlbuild-python-project/src/project_snapshot/types.rs new file mode 100644 index 0000000000..daee71c2f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/types.rs @@ -0,0 +1,13 @@ +//! Public traits and type aliases of project snapshot. + +use crate::project_snapshot::models::WorkerBuild; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use std::collections::BTreeMap; +use std::sync::Arc; + +/// Node keys by snapshot node key, such as the models each test checks. +pub type NodeKeys = BTreeMap>; + +/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. +pub type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index 7cbc31e81a..3ee0f3a367 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -26,7 +26,7 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; -use crate::graph::main::execution_gates::model_python_hook_names::model_python_hook_names; +use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; use crate::references::_helpers::reference_facts::ProjectFacts; use crate::references::_helpers::reference_facts::project_facts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs similarity index 99% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs index 774e19300e..454c3aa050 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::compiled_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledAudit; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs similarity index 94% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs index 5937a9cb69..fc7996ef74 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs @@ -1,9 +1,10 @@ //! Audits attached by the native loop, held by the compiled project, and their views. -use crate::compiled_views::models::NativeAuditInput; -use crate::compiled_views::models::NativeCompiledAudit; -use crate::compiled_views::types::AttachedAudit; -use crate::compiled_views::types::AuditRecord; +use crate::resource_views::models::AuditProjections; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AuditRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -13,10 +14,10 @@ use pyo3::{FromPyObject, pymethods}; use sqlbuild_project::types::ObjectKeyRow; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::compiled_model_view::object_key; -use crate::compiled_views::_helpers::model_input_view::InputViewClasses; -use crate::compiled_views::_helpers::model_input_view::reference_tuple; -use crate::compiled_views::_helpers::model_input_view::view_classes; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; @@ -39,12 +40,6 @@ pub(crate) type AuditAssemblyRow = ( Option<(String, String)>, ); -/// An audit record's Python projections, each built once on first access. -pub struct AuditProjections { - pub(crate) references: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - /// The audit record of an attached audit, before assembly resolves its scope dependencies. pub(crate) fn audit_record(attached: AttachedAudit) -> AuditRecord { AuditRecord { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs similarity index 99% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs index 65e385eee0..03a64e433e 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::compiled_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledFunction; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs similarity index 94% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs index 7d27b19075..cf6aa59013 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs @@ -1,9 +1,10 @@ //! Functions attached by the native loop, held by the compiled project, and their views. -use crate::compiled_views::models::NativeCompiledFunction; -use crate::compiled_views::models::NativeFunctionInput; -use crate::compiled_views::types::AttachedFunction; -use crate::compiled_views::types::FunctionRecord; +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::FunctionRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -14,12 +15,12 @@ use sqlbuild_python_base::positional_record; use sqlbuild_project::types::{NamedTypeRow, ObjectKeyRow}; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::compiled_model_view::defaulted_destination; -use crate::compiled_views::_helpers::compiled_model_view::object_key; -use crate::compiled_views::_helpers::compiled_model_view::relation_location; -use crate::compiled_views::_helpers::model_input_view::InputViewClasses; -use crate::compiled_views::_helpers::model_input_view::reference_tuple; -use crate::compiled_views::_helpers::model_input_view::view_classes; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -51,14 +52,6 @@ positional_record! { } } -/// A function record's Python projections, each built once on first access. -pub struct FunctionProjections { - arguments: PyOnceLock>, - return_columns: PyOnceLock>, - pub(crate) references: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - /// The function record of an attached function, before assembly resolves its dependencies. pub(crate) fn function_record(attached: AttachedFunction) -> FunctionRecord { FunctionRecord { diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs new file mode 100644 index 0000000000..c93a560bd0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod audit_reads; +pub(crate) mod audit_views; +pub(crate) mod function_reads; +pub(crate) mod function_views; +pub(crate) mod resource_reads; +pub(crate) mod seed_reads; +pub(crate) mod seed_views; +pub(crate) mod source_reads; +pub(crate) mod source_views; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs similarity index 95% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs index f463c3b8de..6d70b71eba 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/resource_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs @@ -3,13 +3,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::compiled_views::_helpers::compiled_model_view::model_graph_row; -use crate::compiled_views::models::NativeCompiledFunction; +use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeCompiledSeed; -use crate::compiled_views::models::NativeCompiledSource; -use crate::graph::main::keys::native_key::native_key; +use crate::graph::main::nodes::native_key::native_key; use crate::graph::models::ResourceRow; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs similarity index 98% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs index e0aae31e96..f8b594f936 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::compiled_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSeed; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs index c0ee62033b..17af7c6a95 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs @@ -1,9 +1,10 @@ //! Seeds paired with their declarations, held by the compiled project, and their views. -use crate::compiled_views::models::SeedNamespace; -use crate::compiled_views::models::SeedParts; -use crate::compiled_views::types::SeedRecord; -use crate::compiled_views::types::SeedViews; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::models::SeedParts; +use crate::resource_views::types::SeedRecord; +use crate::resource_views::types::SeedViews; use pyo3::exceptions::PyIndexError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -20,17 +21,6 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; -/// A seed's discovered objects and the key its views build. -pub struct SeedHandles { - /// The discovered CSV file. - pub(crate) seed_file: Py, - /// The discovered declaration, whose spec carries user-authored column metadata. - pub(crate) schema_entry: Py, - /// The schema file declaring the seed. - pub(crate) schema_file: Py, - pub(crate) key: PyOnceLock>, -} - /// Pair each declaration with the CSV file of its name, raising for the first without one. pub(crate) fn paired_seeds( declarations: Vec<(Py, Py)>, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs similarity index 97% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs index ef85363ea9..306022c624 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::compiled_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeCompiledSource; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs index 3891fe4ffd..8e2c145071 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs @@ -1,20 +1,21 @@ //! Sources attached by the native loop, held by the compiled project, and their views. -use crate::compiled_views::models::NativeCompiledSource; -use crate::compiled_views::models::NativeSourceInput; -use crate::compiled_views::types::AttachedSource; -use crate::compiled_views::types::SourceRecord; -use crate::compiled_views::types::SourceViews; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::models::SourceKeyFacts; +use crate::resource_views::models::SourceProjections; +use crate::resource_views::types::AttachedSource; +use crate::resource_views::types::SourceRecord; +use crate::resource_views::types::SourceViews; use pyo3::exceptions::PyIndexError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_python_base::positional_record; -use crate::compiled_project::main::schema_facts::python_column_facts::python_column_facts; +use crate::compiled_project::main::python_column_facts::python_column_facts; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::relation_keys::audit_key_rows; +use crate::compiled_views::main::compiled_models::audit_key_rows::audit_key_rows; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; pub(crate) const SOURCE_RESOURCE: &str = "source"; @@ -30,20 +31,6 @@ positional_record! { } } -/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. -pub struct SourceKeyFacts { - pub columns: Vec, - pub(crate) audits: Vec<(String, Option)>, -} - -/// A source record's key facts and entries, each built once from its Python values. -pub struct SourceProjections { - key_facts: PyOnceLock, - attached_entry: PyOnceLock>, - assembled_entry: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - /// The source record of an attached source, before assembly resolves its namespace. pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { SourceRecord { diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs index b258240250..2a3678b8d1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/always_run.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs @@ -1,6 +1,6 @@ //! The compiled audit's `always_run`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs index 0c6556b10a..de5f871f1c 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_column_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs @@ -1,6 +1,6 @@ //! The compiled audit's `attached_column_name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs index bf80d2713c..7c27b4d747 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_kind.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs @@ -1,6 +1,6 @@ //! The compiled audit's `attached_target_kind`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs index 2dc16e7c44..9c03088229 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/attached_target_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs @@ -1,6 +1,6 @@ //! The compiled audit's `attached_target_name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs index 35503fc542..29bdf90c00 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/definition_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs @@ -1,6 +1,6 @@ //! The compiled audit's `definition_name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs index 545c290bd7..a7c6f46d88 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/description.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs @@ -1,6 +1,6 @@ //! The compiled audit's `description`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs index edc2036c70..65a0b29d98 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evaluation_mode.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs @@ -1,6 +1,6 @@ //! The compiled audit's `evaluation_mode`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs index a21294a9ef..10db612acc 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_limit.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs @@ -1,6 +1,6 @@ //! The compiled audit's `evidence_limit`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs index 9f5441be22..96f2fe8ea8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/evidence_sql.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs @@ -1,6 +1,6 @@ //! The compiled audit's `evidence_sql`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs index 9dd0b1121b..a8c7ae0dd6 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs @@ -1,6 +1,6 @@ //! The compiled audit's `key`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs index 33dafbc8cb..318c99d9b4 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/measurement_contract.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs @@ -1,6 +1,6 @@ //! The compiled audit's `measurement_contract`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs index 1c5abbc2b2..c27710e29f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/meta.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs @@ -1,6 +1,6 @@ //! The compiled audit's `meta`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs index 33e8be110a..c2a6305db9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/minimum_samples.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs @@ -1,6 +1,6 @@ //! The compiled audit's `minimum_samples`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs index 8fdc26131f..a42fcd4383 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs @@ -1,6 +1,6 @@ //! The compiled audit's `name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs index 2a3f18f444..2014ed52d0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/references.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs @@ -1,6 +1,6 @@ //! The compiled audit's `references`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs index f0eb9d41a8..1118606722 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/run_scope.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs @@ -1,6 +1,6 @@ //! The compiled audit's `run_scope`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs index 7afcfd5752..f06237c75c 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/scope_deps.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs @@ -1,6 +1,6 @@ //! The compiled audit's `scope_deps`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs index 1a5fc0609f..8bbfc23c77 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/severity.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs @@ -1,6 +1,6 @@ //! The compiled audit's `severity`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs index 4b1bed68cb..e873228d83 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/sql_body.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs @@ -1,6 +1,6 @@ //! The compiled audit's `sql_body`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs index e043d67cf1..20a1fabb63 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_reads/thresholds.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs @@ -1,6 +1,6 @@ //! The compiled audit's `thresholds`, as its Python attribute reads it. -use crate::compiled_views::_helpers::audit_reads; +use crate::resource_views::_helpers::audit_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs similarity index 67% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs index 36c30e7160..050b4d852f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_record.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs @@ -1,8 +1,8 @@ //! The audit record of an attached audit, before assembly resolves its scope dependencies. -use crate::compiled_views::_helpers::audit_views; -use crate::compiled_views::types::AttachedAudit; -use crate::compiled_views::types::AuditRecord; +use crate::resource_views::_helpers::audit_views; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AuditRecord; /// The audit record of an attached audit, before assembly resolves its scope dependencies. #[inline] diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs index ef7a0e56da..b1013abedb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/audit_reference_rows.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs @@ -1,6 +1,6 @@ //! A native audit view's reference rows, or `None` for a hand-built audit record. -use crate::compiled_views::_helpers::audit_views; +use crate::resource_views::_helpers::audit_views; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/audit_views/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/audit_views/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs index f1ead33bf9..2d3fff3dd8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/arguments.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs @@ -1,6 +1,6 @@ //! The compiled function's `arguments`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs index 0871c4ab5f..e2c27bfe80 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/body_sql.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs @@ -1,6 +1,6 @@ //! The compiled function's `body_sql`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs index 6cce87ce83..5fb585af71 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/deps.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs @@ -1,6 +1,6 @@ //! The compiled function's `deps`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs index cf57c8a8ea..740a4f3c86 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs @@ -1,6 +1,6 @@ //! The compiled function's `destination`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs index ec8b8520b2..bb6e970aaf 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/entry_point.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs @@ -1,6 +1,6 @@ //! The compiled function's `entry_point`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs index 80f5d1f42b..b97b6f2555 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/fingerprint_destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs @@ -1,6 +1,6 @@ //! The compiled function's `fingerprint_destination`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs index e4a166a6fa..ccd9ffd5da 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs @@ -1,6 +1,6 @@ //! The compiled function's `key`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs index f460494416..76865efa26 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/language.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs @@ -1,6 +1,6 @@ //! The compiled function's `language`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs index 08624d0217..95376c0b8a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs @@ -1,6 +1,6 @@ //! The compiled function's `name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs index 2feefc0e07..d3ed3c6ab9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/packages.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs @@ -1,6 +1,6 @@ //! The compiled function's `packages`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs index 5a1d4e69ed..f53adeed38 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/relative_path.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs @@ -1,6 +1,6 @@ //! The compiled function's `relative_path`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs index 1bc51ce82c..ee0a326ec2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/return_columns.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs @@ -1,6 +1,6 @@ //! The compiled function's `return_columns`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs index 96ead2f322..758d378812 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/returns.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs @@ -1,6 +1,6 @@ //! The compiled function's `returns`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs index 560c226b93..888613af23 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/runtime_version.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs @@ -1,6 +1,6 @@ //! The compiled function's `runtime_version`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs index b0493f3389..4831656309 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_reads/source_file_path.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs @@ -1,6 +1,6 @@ //! The compiled function's `source_file_path`, as its Python attribute reads it. -use crate::compiled_views::_helpers::function_reads; +use crate::resource_views::_helpers::function_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs similarity index 67% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs index faecf3109d..a5388f4ead 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/function_views/function_record.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs @@ -1,8 +1,8 @@ //! The function record of an attached function, before assembly resolves its dependencies. -use crate::compiled_views::_helpers::function_views; -use crate::compiled_views::types::AttachedFunction; -use crate::compiled_views::types::FunctionRecord; +use crate::resource_views::_helpers::function_views; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::FunctionRecord; /// The function record of an attached function, before assembly resolves its dependencies. #[inline] diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_views/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/function_views/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/function_views/mod.rs diff --git a/crates/sqlbuild-python-project/src/resource_views/main/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/mod.rs new file mode 100644 index 0000000000..97fbabd537 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/mod.rs @@ -0,0 +1,9 @@ +pub mod audit_reads; +pub mod audit_views; +pub mod function_reads; +pub mod function_views; +pub mod resource_reads; +pub mod seed_reads; +pub mod seed_views; +pub mod source_reads; +pub mod source_views; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs index 50785fde2e..45331cee28 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs @@ -1,6 +1,6 @@ //! A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. -use crate::compiled_views::_helpers::resource_reads; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs index f49f015bce..de3003f408 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/graph_row.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs @@ -1,7 +1,7 @@ //! A resource's graph facts without constructing its public keys or dependency tuple. -use crate::compiled_views::_helpers::resource_reads; use crate::graph::models::ResourceRow; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs index 1ec3c3cf2f..2191f77479 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs @@ -1,6 +1,6 @@ //! A resource's `key` from whichever native view it is; any other object is read by attribute, counted. -use crate::compiled_views::_helpers::resource_reads; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs index c5914046aa..eb4a2d5815 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs @@ -1,6 +1,6 @@ //! A resource's `name` from whichever native view it is; any other object is read by attribute, counted. -use crate::compiled_views::_helpers::resource_reads; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs index 35cce1730e..5187c2cdd8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/references.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs @@ -1,6 +1,6 @@ //! A resource's `references` from whichever native view it is; any other object is read by attribute, counted. -use crate::compiled_views::_helpers::resource_reads; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs rename to crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs index c2815e2fcf..902b8fba1a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/resource_reads/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs @@ -1,6 +1,6 @@ //! A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. -use crate::compiled_views::_helpers::resource_reads; +use crate::resource_views::_helpers::resource_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs index ff40d3e05a..34ce77c8bf 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs @@ -1,6 +1,6 @@ //! The compiled seed's `destination`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs index 9bd052d3e3..337c75c948 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/external.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs @@ -1,6 +1,6 @@ //! The compiled seed's `external`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs index dd87458694..dbf7df7677 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs @@ -1,6 +1,6 @@ //! The compiled seed's `key`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs index 287b4b749b..62318d1ce4 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs @@ -1,6 +1,6 @@ //! The compiled seed's `name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs similarity index 88% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs index e13a77f862..f2f1546ba1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs @@ -1,6 +1,6 @@ //! The compiled seed's `schema_entry`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs index 48efeea871..487f2ce6ca 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_reads/seed_file.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs @@ -1,6 +1,6 @@ //! The compiled seed's `seed_file`, as its Python attribute reads it. -use crate::compiled_views::_helpers::seed_reads; +use crate::resource_views::_helpers::seed_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_views/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_views/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs rename to crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs index 3b5ea5ef51..872c61cfa3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/seed_views/paired_seeds.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs @@ -1,7 +1,7 @@ //! Pair each declaration with the CSV file of its name, raising for the first without one. -use crate::compiled_views::_helpers::seed_views; -use crate::compiled_views::types::SeedRecord; +use crate::resource_views::_helpers::seed_views; +use crate::resource_views::types::SeedRecord; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs index 75c51ec5f8..896a8333ac 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs @@ -1,6 +1,6 @@ //! The compiled source's `key`, as its Python attribute reads it. -use crate::compiled_views::_helpers::source_reads; +use crate::resource_views::_helpers::source_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_reads/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_reads/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs similarity index 86% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs index 43b6240a06..7523cde498 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs @@ -1,6 +1,6 @@ //! The compiled source's `name`, as its Python attribute reads it. -use crate::compiled_views::_helpers::source_reads; +use crate::resource_views::_helpers::source_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs index 069d75749e..22529c585a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs @@ -1,6 +1,6 @@ //! The compiled source's `source_entry`, as its Python attribute reads it. -use crate::compiled_views::_helpers::source_reads; +use crate::resource_views::_helpers::source_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs similarity index 87% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs index 6efdf8210f..7b8893850a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_reads/source_file.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs @@ -1,6 +1,6 @@ //! The compiled source's `source_file`, as its Python attribute reads it. -use crate::compiled_views::_helpers::source_reads; +use crate::resource_views::_helpers::source_reads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/mod.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_views/mod.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_views/mod.rs diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs index 011b217b41..de07103deb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/read_source_view.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs @@ -1,7 +1,7 @@ //! Read the source record behind a native source view, or `None` for any other object. -use crate::compiled_views::_helpers::source_views; -use crate::compiled_views::types::SourceRecord; +use crate::resource_views::_helpers::source_views; +use crate::resource_views::types::SourceRecord; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs index c6e02aa97c..56ed3255f9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/replaced.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs @@ -1,6 +1,6 @@ //! `dataclasses.replace(value, **changes)`. -use crate::compiled_views::_helpers::source_views; +use crate::resource_views::_helpers::source_views; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs similarity index 72% rename from crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs rename to crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs index 9837fa8c19..8a026e5bfe 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/source_views/retain_sources.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs @@ -1,9 +1,9 @@ //! Keep `attached` in `owner`; return each source's view. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::source_views; -use crate::compiled_views::models::NativeSourceInput; -use crate::compiled_views::types::AttachedSource; +use crate::resource_views::_helpers::source_views; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::types::AttachedSource; use pyo3::prelude::Bound; /// Keep `attached` in `owner`; return each source's view. diff --git a/crates/sqlbuild-python-project/src/resource_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/mod.rs new file mode 100644 index 0000000000..d68d2e3db3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/mod.rs @@ -0,0 +1,6 @@ +//! Python views of compiled audits, functions, seeds and sources, and their reads. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/resource_views/models.rs b/crates/sqlbuild-python-project/src/resource_views/models.rs new file mode 100644 index 0000000000..a0afc09f48 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/models.rs @@ -0,0 +1,653 @@ +//! Public records of resource views. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; +use crate::graph::models::ResourceRow; +use crate::resource_views::_helpers::function_views::NOT_ASSEMBLED; +use crate::resource_views::_helpers::function_views::read_function; +use crate::resource_views::_helpers::seed_views::SEED_RESOURCE; +use crate::resource_views::_helpers::seed_views::TAGS_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::read_seed; +use crate::resource_views::_helpers::source_views::SOURCE_RESOURCE; +use crate::resource_views::_helpers::source_views::SourceAssemblyRow; +use crate::resource_views::_helpers::source_views::read_source; +use crate::resource_views::types::AttachedSource; +use crate::resource_views::types::SourceViews; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// An attached audit's user values. +#[derive(Debug)] +pub struct AuditHandles { + /// The discovered (or built-in) audit file and block the SQL comes from. + pub audit_file: Py, + pub audit_block: Py, + /// The authored measurement policy specs, carrying user values. + pub thresholds: Py, + pub minimum_samples: Py, + pub evidence_limit: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// An audit record's Python projections, each built once on first access. +pub struct AuditProjections { + pub references: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeAuditInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeAuditInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledAudit { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledAudit { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +impl NativeCompiledAudit { + /// The `CompiledAudit` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.scope_deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.definition_name(py)?.into_bound_py_any(py)?, + self.audit_file(py)?.into_bound(py), + self.audit_block(py)?.into_bound(py), + self.sql_body(py)?.into_bound_py_any(py)?, + self.evaluation_mode(py)?.into_bound(py), + self.measurement_contract(py)?.into_bound(py), + self.thresholds(py)?.into_bound(py), + self.minimum_samples(py)?.into_bound(py), + self.measure_sql(py)?.into_bound_py_any(py)?, + self.evidence_sql(py)?.into_bound_py_any(py)?, + self.evidence_limit(py)?.into_bound(py), + self.references(py)?.into_bound(py).into_any(), + self.attached_target_kind(py)?, + self.attached_target_name(py)?.into_bound_py_any(py)?, + self.attached_column_name(py)?.into_bound_py_any(py)?, + self.severity(py)?, + self.run_scope(py)?.into_bound_py_any(py)?, + pyo3::types::PyBool::new(py, self.always_run(py)?) + .to_owned() + .into_any(), + self.description(py)?.into_bound_py_any(py)?, + self.meta(py)?.into_bound(py), + ]) + } +} + +/// An attached function's user values. +#[derive(Debug)] +pub struct FunctionHandles { + /// The discovered function file. + pub function_file: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// A function record's Python projections, each built once on first access. +pub struct FunctionProjections { + pub(crate) arguments: PyOnceLock>, + pub(crate) return_columns: PyOnceLock>, + pub references: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeFunctionInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeFunctionInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledFunction { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination and fingerprint destination with target defaults applied. + pub(crate) defaulted: Option<(DestinationFact, DestinationFact)>, + pub(crate) destination: PyOnceLock>, + pub(crate) fingerprint_destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledFunction { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledFunction") + .finish_non_exhaustive() + } +} + +impl NativeCompiledFunction { + pub fn new(owner: Py, index: usize) -> Self { + Self::defaulted(owner, index, None) + } + + /// The function's project graph row: its key, the resources it reads and its tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + Ok(ResourceRow { + key: ( + function.attached.node_type().to_owned(), + function.attached.name.clone(), + ), + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) + }) + } + + pub(crate) fn defaulted( + owner: Py, + index: usize, + defaulted: Option<(DestinationFact, DestinationFact)>, + ) -> Self { + Self { + owner, + index, + defaulted, + destination: PyOnceLock::new(), + fingerprint_destination: PyOnceLock::new(), + } + } + + /// The destination and fingerprint destination, with target defaults once applied. + pub fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_function(&self.owner, self.index, py, |function, _| { + Ok(( + function.attached.destination.clone(), + function.attached.fingerprint_destination.clone(), + )) + }) + } +} + +impl NativeCompiledFunction { + /// The `CompiledFunction` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.relative_path(py)?.into_bound(py), + self.arguments(py)?.into_bound(py).into_any(), + self.returns(py)?.into_bound_py_any(py)?, + self.body_sql(py)?.into_bound_py_any(py)?, + self.destination(py)?.into_bound(py), + self.fingerprint_destination(py)?.into_bound(py), + self.return_columns(py)?.into_bound(py).into_any(), + self.references(py)?.into_bound(py).into_any(), + self.language(py)?, + self.source_file_path(py)?.into_bound(py), + self.runtime_version(py)?.into_bound_py_any(py)?, + self.entry_point(py)?.into_bound_py_any(py)?, + self.packages(py)?.into_any(), + self.tags(py)?.into_any(), + self.description(py)?.into_bound_py_any(py)?, + ]) + } +} + +/// `(database, schema, logical database, logical schema)` of a seed's destination. +#[derive(Debug)] +pub struct SeedNamespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for SeedNamespace { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (database, schema, logical_database, logical_schema): ( + Option, + Option, + Option, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + database, + schema, + logical_database, + logical_schema, + }) + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +#[derive(Debug)] +pub struct SeedParts<'a> { + pub name: &'a str, + pub seed_file: &'a Py, + pub schema_entry: &'a Py, + pub schema_file: &'a Py, +} + +/// A seed's discovered objects and the key its views build. +pub struct SeedHandles { + /// The discovered CSV file. + pub(crate) seed_file: Py, + /// The discovered declaration, whose spec carries user-authored column metadata. + pub(crate) schema_entry: Py, + /// The schema file declaring the seed. + pub(crate) schema_file: Py, + pub(crate) key: PyOnceLock>, +} + +/// One attached seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSeedInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSeedInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSeedInput { + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } +} + +/// One assembled seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledSeed { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination with target defaults applied, replacing the assembled one. + pub(crate) defaulted: Option, + pub(crate) destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledSeed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledSeed") + .finish_non_exhaustive() + } +} + +impl NativeCompiledSeed { + pub fn new(owner: Py, index: usize) -> Self { + Self { + owner, + index, + defaulted: None, + destination: PyOnceLock::new(), + } + } + + /// The seed's project graph row: its key and its declared tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, py| { + let mut tags: Vec = Vec::new(); + for tag in native_field(seed.host.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), + tags, + folder: None, + }) + }) + } + + pub fn current_destination(&self, py: Python<'_>) -> PyResult { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_seed(&self.owner, self.index, py, |seed, _| { + seed.destination + .clone() + .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) + }) + } +} + +#[pymethods] +impl NativeCompiledSeed { + /// The seed's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_seed(&self.owner, self.index, py, |seed, py| { + seed.host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SEED_RESOURCE, &seed.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) + } + + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } + + /// The seed's planner-facing `CompiledRelationLocation`. + #[getter] + pub fn destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destination(py)?) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// This seed with the adapter's default schema and database filled into its destination. + pub fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let defaulted: DestinationFact = + defaulted_destination(&self.current_destination(py)?, defaults, &render)?; + Ok(Self { + owner: self.owner.clone_ref(py), + index: self.index, + defaulted: Some(defaulted), + destination: PyOnceLock::new(), + }) + } + + #[getter(external)] + pub fn is_external(&self) -> bool { + false + } +} + +impl NativeCompiledSeed { + /// The `CompiledSeed` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.seed_file(py)?.into_bound(py), + self.schema_entry(py)?.into_bound(py), + self.schema_file(py)?.into_bound(py), + self.destination(py)?.into_bound(py), + pyo3::types::PyBool::new(py, self.is_external()) + .to_owned() + .into_any(), + ]) + } +} + +/// An attached source's user values. +#[derive(Debug)] +pub struct SourceHandles { + /// The discovered source file. + pub source_file: Py, + /// The discovered entry; its spec carries user-authored metadata. + pub entry: Py, + /// Template-expanded user metadata. + pub meta: Py, + /// Template-expanded column specs, which carry user metadata and audit arguments. + pub columns: Py, + /// Template-expanded table audit specs, which carry user audit arguments. + pub audits: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. +#[derive(Debug)] +pub struct SourceKeyFacts { + pub columns: Vec, + pub audits: Vec<(String, Option)>, +} + +/// A source record's key facts and entries, each built once from its Python values. +pub struct SourceProjections { + pub(crate) key_facts: PyOnceLock, + pub(crate) attached_entry: PyOnceLock>, + pub(crate) assembled_entry: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSourceInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSourceInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSourceInput { + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.attached_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } + + #[getter] + pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.usages.clone_ref(py)) + }) + } + + /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. + #[getter] + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, py| { + let attached: &AttachedSource = &source.attached; + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) + }) + } +} + +/// One assembled source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledSource { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledSource { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + /// The source's project graph row: its key, with no dependencies, tags or folder. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) + }) + } +} + +#[pymethods] +impl NativeCompiledSource { + /// The source's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_source(&self.owner, self.index, py, |source, py| { + source + .host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(source.attached.name.clone()) + }) + } + + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.assembled_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } +} + +impl NativeCompiledSource { + /// The `CompiledSource` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.source_entry(py)?.into_bound(py), + self.source_file(py)?.into_bound(py), + ]) + } +} diff --git a/crates/sqlbuild-python-project/src/resource_views/types.rs b/crates/sqlbuild-python-project/src/resource_views/types.rs new file mode 100644 index 0000000000..2a583e5a18 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/types.rs @@ -0,0 +1,53 @@ +//! Public traits and type aliases of resource views. + +use crate::compiled_views::models::PyProjectHost; +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::models::SeedParts; +use crate::resource_views::models::SourceKeyFacts; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_project::models as project_models; + +/// What the audit loop attached for one singular audit block or attached generic audit. +pub type AttachedAudit = project_models::AttachedAudit; + +/// One attached audit held by the compiled project. +pub type AuditRecord = project_models::AuditRecord; + +/// What the function loop attached for one SQL or Python function. +pub type AttachedFunction = project_models::AttachedFunction; + +/// One attached function held by the compiled project. +pub type FunctionRecord = project_models::FunctionRecord; + +/// One seed CSV paired with its declaration; the discovered objects are discovery's. +pub type SeedRecord = project_models::SeedRecord; + +/// What a seed record reads and keeps through its Python values. +pub trait SeedViews { + /// The seed's name, CSV file, declaration and declaring schema file. + fn parts(&self) -> SeedParts<'_>; + + /// Keep the destination native project assembly resolved for the seed. + fn set_namespace(&mut self, namespace: SeedNamespace); +} + +/// What the source loop attached for one source: its expanded metadata and usages. +pub type AttachedSource = project_models::AttachedSource; + +/// One attached source held by the compiled project. +pub type SourceRecord = project_models::SourceRecord; + +/// What a source record projects from its Python values, each built once. +pub trait SourceViews { + /// The source's declared column facts and audit key rows, read once from its expanded specs. + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; + + /// The attached `SourceEntry`: the discovered entry with its expanded metadata. + fn attached_entry(&self, py: Python<'_>) -> PyResult>; + + /// The assembled `SourceEntry`: the attached entry in its resolved namespace. + fn assembled_entry(&self, py: Python<'_>) -> PyResult>; +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs index e06d5b178a..719a832680 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs @@ -6,8 +6,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use crate::compiled_views::main::model_reads::star_flags::star_flags; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; use crate::semantic_shapes::_helpers::delta_shapes::semantic_reasons; use crate::semantic_shapes::_helpers::shapes::project_shapes; use sqlbuild_project::types::InferredRow; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs index dbe6672176..b92343e453 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs @@ -13,11 +13,11 @@ use crate::analysis::models::BindingCatalog; use crate::analysis::models::CatalogKey; use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; use crate::compiled_project::models::ReferenceTuple; -use crate::compiled_views::main::model_reads::declared_rows::declared_rows; -use crate::compiled_views::main::model_reads::dynamic_columns_declared::dynamic_columns_declared; -use crate::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use crate::compiled_views::main::model_reads::star_flags::star_flags; -use crate::compiled_views::main::model_reads::unchecked_columns::unchecked_columns; +use crate::compiled_views::main::compiled_models::declared_rows::declared_rows; +use crate::compiled_views::main::compiled_models::dynamic_columns_declared::dynamic_columns_declared; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::compiled_views::main::compiled_models::unchecked_columns::unchecked_columns; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::types::DeclaredRow; use sqlbuild_project::types::InferredRow; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs similarity index 98% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs index 4604f2f372..60d0a5ae42 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs @@ -8,8 +8,8 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; -use crate::compiled_views::main::model_reads::name_object::name_object; -use crate::compiled_views::main::model_reads::query_sql::query_sql; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; use crate::sql_test_glue::constants::CURSOR_ALGEBRA_EXCEPTIONS; use crate::sql_test_glue::constants::CURSOR_END_KEY; use crate::sql_test_glue::constants::CURSOR_START_KEY; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs similarity index 100% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs new file mode 100644 index 0000000000..5dd33aa2e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod cursor_window; +pub(crate) mod fixture_validation; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs index 593b57eae6..ea1ae2f6d1 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs @@ -1,12 +1,2 @@ -pub(crate) mod cursor_window; -pub(crate) mod fixture_validation; -pub(crate) mod plan_assembly; -pub(crate) mod plan_driver; -pub(crate) mod plan_entries; -pub(crate) mod plan_errors; -pub(crate) mod plan_fixtures; -pub(crate) mod plan_rendering; -pub(crate) mod plan_rows; -pub(crate) mod plan_values; -pub(crate) mod planning; -pub(crate) mod types; +pub(crate) mod inputs; +pub(crate) mod plan; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs new file mode 100644 index 0000000000..62587e8392 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod plan_assembly; +pub(crate) mod plan_driver; +pub(crate) mod plan_entries; +pub(crate) mod plan_errors; +pub(crate) mod plan_fixtures; +pub(crate) mod plan_rendering; +pub(crate) mod plan_rows; +pub(crate) mod plan_values; +pub(crate) mod planning; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs similarity index 94% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs index f9808ee903..b68f080f10 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_assembly.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::compiled_views::main::model_reads::name_object::name_object; +use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::planner_strategy::main::planning_context::planning_context; use crate::planner_strategy::models::RelationFixturePlanningContext; -use crate::sql_test_glue::_helpers::plan_driver::planned as planned_tests; +use crate::sql_test_glue::_helpers::plan::plan_driver::planned as planned_tests; use crate::sql_test_glue::models::SqlTestAssembly; use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -61,7 +61,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( } else { given.cast_into::()? }; - let validated = crate::sql_test_glue::_helpers::plan_fixtures::validate( + let validated = crate::sql_test_glue::_helpers::plan::plan_fixtures::validate( tests, (project, adapter), shared.get(), @@ -118,7 +118,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( }; let mut errors: Vec = native_field(&plan, "error_messages")?.extract()?; errors.extend( - crate::sql_test_glue::_helpers::plan_entries::expected_diagnostics( + crate::sql_test_glue::_helpers::plan::plan_entries::expected_diagnostics( test, &plan, context.as_ref().map(Bound::get), @@ -141,7 +141,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( &[ ( "entry", - crate::sql_test_glue::_helpers::plan_entries::entry( + crate::sql_test_glue::_helpers::plan::plan_entries::entry( test, &plan, (project, &models), diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs similarity index 90% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs index 82ab4f3613..d89c3cce89 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs @@ -6,11 +6,11 @@ use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing use crate::planner_phases::main::object::object; use crate::planner_strategy::main::count_chain_resolution::count_chain_resolution; -use crate::sql_test_glue::_helpers::plan_values::tuple; -use crate::sql_test_glue::_helpers::planning::ChainRequest; -use crate::sql_test_glue::_helpers::planning::PlanningRequest; -use crate::sql_test_glue::_helpers::planning::plan_compiled_sql_tests; -use crate::sql_test_glue::_helpers::planning::resolve_compiled_sql_test_chains; +use crate::sql_test_glue::_helpers::plan::plan_values::tuple; +use crate::sql_test_glue::_helpers::plan::planning::ChainRequest; +use crate::sql_test_glue::_helpers::plan::planning::PlanningRequest; +use crate::sql_test_glue::_helpers::plan::planning::plan_compiled_sql_tests; +use crate::sql_test_glue::_helpers::plan::planning::resolve_compiled_sql_test_chains; use crate::sql_test_glue::models::PlanRow; use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::main::imported::imported; @@ -56,9 +56,11 @@ pub(crate) fn chains<'py>( let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { - return Err(crate::sql_test_glue::_helpers::plan_errors::planning_error( - py, error, tests, - )?); + return Err( + crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( + py, error, tests, + )?, + ); } Err(error) => return Err(error), }; @@ -92,10 +94,11 @@ pub(crate) fn planned_rows<'py>( let start: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; - let model_sql = crate::sql_test_glue::_helpers::plan_rendering::models(project, adapter)?; - let functions = crate::sql_test_glue::_helpers::plan_rendering::functions(project, adapter)?; + let model_sql = crate::sql_test_glue::_helpers::plan::plan_rendering::models(project, adapter)?; + let functions = + crate::sql_test_glue::_helpers::plan::plan_rendering::functions(project, adapter)?; let overrides = - crate::sql_test_glue::_helpers::plan_rendering::overrides(project, tests, adapter)?; + crate::sql_test_glue::_helpers::plan::plan_rendering::overrides(project, tests, adapter)?; let request = PlanningRequest { models: native_field(project, "models")?.extract()?, rendered_model_sql: model_sql.extract()?, @@ -126,9 +129,11 @@ pub(crate) fn planned_rows<'py>( let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { - return Err(crate::sql_test_glue::_helpers::plan_errors::planning_error( - py, error, tests, - )?); + return Err( + crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( + py, error, tests, + )?, + ); } Err(error) => return Err(error), }; @@ -166,7 +171,7 @@ pub(crate) fn planned<'py>( let rows = planned_rows((project, adapter), tests, options)?; let mut plans: Vec> = Vec::new(); for row in rows { - let plan = crate::sql_test_glue::_helpers::plan_rows::plan(py, row)?; + let plan = crate::sql_test_glue::_helpers::plan::plan_rows::plan(py, row)?; if options.artifact { let sql = native_field(&plan, "sql")?; plans.push(new_record_with( diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs similarity index 95% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs index d4235c81c8..f22b0d047b 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_entries.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs @@ -8,7 +8,7 @@ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::planner_strategy::models::RelationFixturePlanningContext; -use crate::sql_test_glue::_helpers::plan_values::model_payload; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; use crate::sql_test_glue::constants::TABLE_FN; use crate::sql_test_glue::constants::UDF; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; @@ -187,11 +187,12 @@ pub(crate) fn expected_diagnostics( else { continue; }; - let location = crate::sql_test_glue::_helpers::fixture_validation::fixture_location( - test, - &native_field(&resource_types, "SQL_TEST")?, - &name.extract::()?, - )?; + let location = + crate::sql_test_glue::_helpers::inputs::fixture_validation::fixture_location( + test, + &native_field(&resource_types, "SQL_TEST")?, + &name.extract::()?, + )?; result.push(format!( "SQL test '{}': {}: {}", native_field(test, "name")?.extract::()?, diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs similarity index 98% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs index 8ab51cb46b..1d5a35b824 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs @@ -1,6 +1,6 @@ //! Project native planning failures into existing located diagnostic exceptions. -use crate::sql_test_glue::_helpers::plan_values::string; +use crate::sql_test_glue::_helpers::plan::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs similarity index 93% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs index 42a6a51a27..52631017fb 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_fixtures.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs @@ -4,9 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::planner_strategy::models::RelationFixturePlanningContext; -use crate::sql_test_glue::_helpers::fixture_validation::validated_fixtures; -use crate::sql_test_glue::_helpers::plan_driver::resolve_sql_test_model_chains; -use crate::sql_test_glue::_helpers::plan_values::model_payload; +use crate::sql_test_glue::_helpers::inputs::fixture_validation::validated_fixtures; +use crate::sql_test_glue::_helpers::plan::plan_driver::resolve_sql_test_model_chains; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; @@ -80,7 +80,7 @@ pub(crate) fn validate<'py>( tests: &[Bound<'py, PyAny>], inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), context: &RelationFixturePlanningContext, -) -> PyResult> { +) -> PyResult> { let py = inputs.0.py(); let mut model_tests: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs similarity index 90% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs index 2b89f128b4..e02fee468d 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs @@ -1,16 +1,16 @@ //! Adapter rendering and declared cursor-window boundaries for native test planning. -use crate::compiled_views::main::model_reads::name_object::name_object; -use crate::compiled_views::main::model_reads::query_sql::query_sql; -use crate::sql_test_glue::_helpers::plan_values::model_payload; -use crate::sql_test_glue::_helpers::plan_values::string; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; +use crate::sql_test_glue::_helpers::plan::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::sql_test_glue::_helpers::cursor_window::declared_window_cursor_models; -use crate::sql_test_glue::_helpers::cursor_window::declares_cursor_window; -use crate::sql_test_glue::_helpers::cursor_window::render_test_cursor_intrinsics; +use crate::sql_test_glue::_helpers::inputs::cursor_window::declared_window_cursor_models; +use crate::sql_test_glue::_helpers::inputs::cursor_window::declares_cursor_window; +use crate::sql_test_glue::_helpers::inputs::cursor_window::render_test_cursor_intrinsics; use crate::sql_test_glue::constants::SENTINEL; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -64,7 +64,7 @@ pub(crate) fn overrides<'py>( let chains = if indexes.is_empty() { Vec::new() } else { - crate::sql_test_glue::_helpers::plan_driver::chains( + crate::sql_test_glue::_helpers::plan::plan_driver::chains( project, &PyTuple::new(py, windowed)?.into_any(), )? diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rows.rs similarity index 100% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rows.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rows.rs diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs similarity index 100% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs similarity index 98% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs index 4f9fa1f2af..f8c71ce1ff 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs @@ -13,8 +13,8 @@ use sqlbuild_analysis::compiler::models::{ SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; -use crate::compiled_views::main::model_reads::name_object::name_object; -use crate::compiled_views::main::model_reads::query_sql::query_sql; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs index ecd602f9e0..42d02b9353 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs @@ -1,6 +1,6 @@ //! Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. -use crate::sql_test_glue::_helpers::plan_assembly; +use crate::sql_test_glue::_helpers::plan::plan_assembly; use crate::sql_test_glue::models::SqlTestAssembly; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs index 06dfdd9335..d75b8a5177 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs @@ -1,6 +1,6 @@ //! `plan_compiled_sql_test_artifacts` as a function of `module`. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; /// `plan_compiled_sql_test_artifacts` as a function of `module`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs index 31d15fcd73..939389f748 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs @@ -1,6 +1,6 @@ //! `plan_sql_tests` as a function of `module`. -use crate::sql_test_glue::_helpers::plan_assembly; +use crate::sql_test_glue::_helpers::plan::plan_assembly; /// `plan_sql_tests` as a function of `module`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs index 6e6e3cf4f5..43ad71a6a3 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs @@ -1,6 +1,6 @@ //! `plan_sql_tests_natively` as a function of `module`. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; /// `plan_sql_tests_natively` as a function of `module`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs index 8a52300595..d65eb82839 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs @@ -1,6 +1,6 @@ //! Native plans of one SQL-test batch, with adapter rendering and timing accounted for. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; use crate::sql_test_glue::models::PlanRow; use crate::sql_test_glue::models::SqlTestPlanOptions; use pyo3::prelude::Bound; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs index 4226cd70ea..8114d1917f 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs @@ -1,6 +1,6 @@ //! Return model closures resolved natively in one project batch, keyed by SQL test key. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs index c905c4b594..a0a3391ef5 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs @@ -1,6 +1,6 @@ //! `sql_test_model_chain_names_by_key` as a function of `module`. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; /// `sql_test_model_chain_names_by_key` as a function of `module`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs index 44e9dccf4e..bfb26a146c 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs @@ -1,6 +1,6 @@ //! `sql_test_plan_error_messages` as a function of `module`. -use crate::sql_test_glue::_helpers::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver; /// `sql_test_plan_error_messages` as a function of `module`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs index 4cff23cef2..7938daf0af 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs similarity index 56% rename from crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/types.rs index e230e76bf0..b8d22b2788 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/types.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs @@ -1,6 +1,7 @@ -//! SQL-test fixture validation boundary values. +//! Public traits and type aliases of sql test glue. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; use std::collections::BTreeMap; pub(crate) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index 853a047f26..322bfa51a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -5,6 +5,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog; +use sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references; #[pyfunction] fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { @@ -27,11 +29,9 @@ fn parse_warehouse_type(py: Python<'_>, spelling: &str, dialect: &str) -> PyResu } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog( - module, - )?; + register_binding_catalog(module)?; crate::bindings::_helpers::analysis::required_columns::register(module)?; - sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references(module)?; + register_explicit_references(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 2b9554924d..40645126b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -35,11 +35,11 @@ use sqlbuild_python_project::analysis_session::models::FamilyRow; use sqlbuild_python_project::analysis_session::models::ModelRow; use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; use sqlbuild_python_project::analysis_session::models::RequestRow; -use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_analysis_sql::input_analysis_sql; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_model_name::input_model_name; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_sql_macro_checks::input_sql_macro_checks; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_model_name::input_model_name; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_sql_macro_checks::input_sql_macro_checks; use sqlbuild_python_project::compiled_views::models::AttachedSchema; use sqlbuild_python_project::compiled_views::models::NativeModelInput; use sqlbuild_python_project::semantic_shapes::constants::VALID_FROM_DEFAULT; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 61b81b210c..8eb7797022 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -42,8 +42,8 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; -use sqlbuild_python_project::compiled_project::types::VerifiedRun; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs index 8a06cf2b74..42c46345ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs @@ -1,8 +1,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::auditing::main::register_render::register_render; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::auditing::identity::register(module)?; crate::bindings::_helpers::auditing::measurement::register(module)?; - sqlbuild_python_project::auditing::main::register_render::register_render(module)?; + register_render(module)?; crate::bindings::_helpers::auditing::schema_audits::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 423da7418d..5faa65a20e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -34,8 +34,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index a50b17398d..22c526cf64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -59,7 +59,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; const EXECUTION_COMMAND: &str = "sqb build"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 5de64046c0..0596037169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -31,7 +31,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::models::SecretPolicy; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index b5d40e9b4a..c073f3eaac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -29,8 +29,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; const SUCCESS_STATUS: &str = "success"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index 8f14f874f9..2ca936d586 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -47,7 +47,7 @@ use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; const PASS_OUTCOME: &str = "pass"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 3245d10cc4..7cf4d4902f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -23,14 +23,14 @@ use crate::bindings::_helpers::change_selection::fingerprints::{ }; use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::function_reads::source_file_path::source_file_path; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; -use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; /// Header values that rename rather than redefine a model. const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; @@ -242,10 +242,10 @@ fn model_node( ) -> PyResult<(String, NodeDefinition)> { let name: String = model_name(model)?; let config: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?; + sqlbuild_python_project::compiled_views::main::compiled_models::config::config(model)?; let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; let schema: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; let contract: String = match config_values.cast::() { @@ -304,7 +304,7 @@ fn source_node( fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { let name: String = - sqlbuild_python_project::compiled_views::main::source_reads::name::name(source)? + sqlbuild_python_project::resource_views::main::source_reads::name::name(source)? .extract()?; let mut encoded: Value = fingerprints.canonical(&source_entry(source)?, &[])?; let mut shape: Map = Map::new(); @@ -343,10 +343,10 @@ fn seed_node( fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { let name: String = - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?.extract()?; + sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?.extract()?; let (version_hash, _) = seed_identity(seed)?; let schema: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry( + sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry( seed, )?; let columns: String = if schema.is_none() { @@ -464,9 +464,7 @@ fn function_nodes( ) -> PyResult> { let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); let functions: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - project, - )?; + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?; let hashes: Bound<'_, PyDict> = function_local_hashes( &functions, None, @@ -475,13 +473,13 @@ fn function_nodes( for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; let key: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?; + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?; let kind: String = text_of(&native_field( &native_field(&key, "resource_type")?, "value", )?)?; let name: String = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? .extract()?; let Some(hash) = hashes.get_item(&name)? else { continue; @@ -609,7 +607,7 @@ pub(crate) fn definition_snapshot( ); let (connection_key, connection) = connection_node(project, discovered, &fingerprints)?; nodes.insert(connection_key, connection); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let (key, definition) = model_node(&model?, &fingerprints, &uses)?; nodes.insert(key, definition); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index af780e7b0b..293c886880 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -29,9 +29,10 @@ use crate::bindings::_helpers::change_selection::environment_nodes::with_current use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; const MACRO_TEST_MODE: &str = "macro"; /// The JSON document key of the changes and selection of a `--since` run. @@ -333,13 +334,10 @@ pub(crate) fn tests_and_models<'py>( tests.insert(native_field(&test, "name")?.extract()?, test); } let mut models: ByName<'py> = HashMap::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'py, PyAny> = model?; models.insert( - sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( &model, )?, model, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 37b62b99c9..14347b476b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -30,9 +30,9 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; -use sqlbuild_python_project::graph::main::algorithms::transitive_closure_many::transitive_closure_many; -use sqlbuild_python_project::graph::main::execution_gates::sql_ref_key::sql_ref_key; -use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::nodes::sql_ref_key::sql_ref_key; +use sqlbuild_python_project::graph::main::nodes::transitive_closure_many::transitive_closure_many; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; @@ -53,8 +53,8 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; -use sqlbuild_python_project::graph::main::python_selection::python_node_names::python_node_names; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 1301432fc5..1dce014f02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -10,7 +10,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_project::graph::main::python_selection::resolve_python_check_excludes::resolve_python_check_excludes; +use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_core::build_command::errors::BuildCommandError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 5ae1faf784..f29dfb6eaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -56,7 +56,7 @@ use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_rec use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::models::MissingSecretsScope; -use sqlbuild_python_project::compiled_project::main::job_catalog::with_job_scope::with_job_scope; +use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 18f39627af..78d31418c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -11,8 +11,8 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; -use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; -use sqlbuild_python_project::compiled_views::main::records::record_model_analyses::record_model_analyses; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; use sqlbuild_python_project::references::main::explicit_reference_report::explicit_reference_report; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 2cff136e89..7decd6713c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -45,8 +45,8 @@ use sqlbuild_python_project::analysis_session::constants::ADAPTER_RULE_ID; use sqlbuild_python_project::analysis_session::main::expression_source_shapes::expression_source_shapes; use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; use sqlbuild_python_project::analysis_session::main::text_pairs::text_pairs; -use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; -use sqlbuild_python_project::compiled_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::resource_views::main::source_views::read_source_view::read_source_view; use sqlbuild_python_project::semantic_shapes::main::declared_types::declared_types; use sqlbuild_python_project::semantic_shapes::main::function_names::function_names; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 1a632256b5..4b396dea53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -26,7 +26,7 @@ use sqlbuild_python_front::render::main::calls::top_level_macro_call_names::top_ use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; use sqlbuild_python_project::analysis_session::main::query_column_rows::query_column_rows; use sqlbuild_python_project::analysis_session::types::ColumnsRow; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_analysis_sql::input_analysis_sql; use sqlbuild_python_project::compiled_views::models::NativeModelInput; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index 2675a9007a..9ae7596cda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -21,9 +21,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::graph::main::views::project_lineage_views::project_lineage_views; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::graph::main::project::project_lineage_views::project_lineage_views; /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. struct KeyIndexes<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 624b1c1be8..16dbc2b370 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -12,8 +12,8 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_views::main::records::compile_inputs_view::compile_inputs_view; -use sqlbuild_python_project::compiled_views::main::records::inputs_with::inputs_with; +use sqlbuild_python_project::compiled_views::main::project::compile_inputs_view::compile_inputs_view; +use sqlbuild_python_project::compiled_views::main::project::inputs_with::inputs_with; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::input_loops::attachment::{attach_checks, attach_resources}; use sqlbuild_python_front::model_config::main::config::run_id::run_id; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 3b2138ba47..a43af45a37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -22,7 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index bc6f15c382..7cdd47a44b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -13,10 +13,10 @@ use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_ use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::HashSet; use pyo3::exceptions::PyException; @@ -30,7 +30,7 @@ use sqlbuild_python_base::native_host::main::connection_failure::connection_fail use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; const SINGLE_CONNECTION: i64 = 1; @@ -68,16 +68,15 @@ fn without_loaders<'py>( } } let mut sources: Vec<(String, String)> = Vec::new(); - for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? - .try_iter()? + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? { let source = source?; let loader: Option = native_field(&source_entry(&source)?, "loader")?.extract()?; if loader.is_some_and(|loader| dropped.contains(&loader)) { sources.push(( SOURCE.to_owned(), - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? .extract()?, )); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index a2adff7b02..421fab6a16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -41,7 +41,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; -use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; use sqlbuild_python_project::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Optional keywords of the project graph builders. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index f31b5b77ae..cdd6b92def 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -6,7 +6,7 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::replaced::replaced; -use sqlbuild_python_project::graph::main::python_lifecycle::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -23,7 +23,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; const SKIPPED_UPSTREAM_CODE: &str = "P501"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index fa23b192b9..12cc3c0e8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -18,7 +18,7 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; /// Locations of the deferred target's relations, when the options defer to one. fn deferred_locations<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 1a1455efae..149855f344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -9,10 +9,10 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; -use sqlbuild_python_project::compiled_views::models::NativeCompiledFunction; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; -use sqlbuild_python_project::compiled_views::models::NativeCompiledSeed; +use sqlbuild_python_project::resource_views::models::NativeCompiledFunction; +use sqlbuild_python_project::resource_views::models::NativeCompiledSeed; /// The adapter defaults every compiled location falls back to. pub(crate) struct TargetDefaults<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index fd0b8bc90c..2c8f452329 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -1,16 +1,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{compiled_views, input_loops}; +use sqlbuild_python_project::compiled_project::main::register_project::register_project; +use sqlbuild_python_project::project_snapshot::main::register_snapshot::register_snapshot; /// Register the compiled project, then its input loops, then its views and records. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - sqlbuild_python_project::compiled_project::main::project::register_project::register_project( - module, - )?; + register_project(module)?; input_loops::registration::register(module)?; compiled_views::registration::register(module)?; crate::bindings::_helpers::location_command::command::register(module)?; - sqlbuild_python_project::compiled_project::main::snapshot::register_snapshot::register_snapshot( - module, - ) + register_snapshot(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs index fb27f03cf3..05c8d30010 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs @@ -1,7 +1,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::compiled_views::main::project::register_records::register_records; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - sqlbuild_python_project::compiled_views::main::records::register_records::register_records( - module, - ) + register_records(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs index a99b3356d2..ebd65842a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -16,16 +16,16 @@ use crate::bindings::_helpers::planner_identity::metadata::{ }; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; -use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::function_reads::language::language; -use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; pub(crate) const MODEL: &str = "model"; pub(crate) const SEED: &str = "seed"; @@ -54,16 +54,8 @@ enum Compiled<'py> { impl<'py> Compiled<'py> { fn destination(&self) -> PyResult> { match self { - Self::Model(model) => { - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - model, - ) - } - Self::Seed(seed) => { - sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( - seed, - ) - } + Self::Model(model) => sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model), + Self::Seed(seed) => sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(seed), } } } @@ -235,7 +227,7 @@ pub(crate) fn build_facts( ) -> PyResult { let py: Python<'_> = plan.py(); let mut compiled: HashMap> = HashMap::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; compiled.insert( node_key(MODEL, &model_name(&model)?), @@ -247,7 +239,7 @@ pub(crate) fn build_facts( compiled.insert( node_key( SEED, - &sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + &sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? .extract::()?, ), Compiled::Seed(seed), @@ -256,13 +248,11 @@ pub(crate) fn build_facts( let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); let mut function_bodies: HashMap<(String, String), Option> = HashMap::new(); let functions: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - project, - )?; + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; let key = graph_key( - &sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + &sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, )?; function_deps.insert( (key.kind.clone(), key.name.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs index ac9ae34ca7..299520c88f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs @@ -11,7 +11,7 @@ use sqlbuild_state::consumption::models::{ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::main::job_catalog::triggered_jobs::triggered_jobs; +use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; use sqlbuild_python_project::graph::models::NativeProjectGraph; const DEFINITION: &str = "definition"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 5b59678efc..91312c9289 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -27,10 +27,10 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; -use sqlbuild_python_project::compiled_project::types::VerifiedRun; use sqlbuild_python_project::planner_selection::main::expand_project_upstream_keys::expand_project_upstream_keys; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index af6b8d4689..dbc2621dd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -1,13 +1,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::graph::main::nodes::register_algorithms::register_algorithms; +use sqlbuild_python_project::graph::main::nodes::register_execution_gates::register_execution_gates; +use sqlbuild_python_project::graph::main::project::register_project_graph::register_project_graph; +use sqlbuild_python_project::graph::main::project::register_views::register_views; +use sqlbuild_python_project::graph::main::python_nodes::register_python_lifecycle::register_python_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::register_python_node_graph::register_python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::register_python_selection::register_python_selection; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - sqlbuild_python_project::graph::main::project_graph::register_project_graph::register_project_graph(module)?; - sqlbuild_python_project::graph::main::algorithms::register_algorithms::register_algorithms( - module, - )?; - sqlbuild_python_project::graph::main::execution_gates::register_execution_gates::register_execution_gates(module)?; - sqlbuild_python_project::graph::main::python_selection::register_python_selection::register_python_selection(module)?; - sqlbuild_python_project::graph::main::python_lifecycle::register_python_lifecycle::register_python_lifecycle(module)?; - sqlbuild_python_project::graph::main::python_node_graph::register_python_node_graph::register_python_node_graph(module)?; - sqlbuild_python_project::graph::main::views::register_views::register_views(module) + register_project_graph(module)?; + register_algorithms(module)?; + register_execution_gates(module)?; + register_python_selection(module)?; + register_python_lifecycle(module)?; + register_python_node_graph(module)?; + register_views(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index f52c5310b4..b32e6d6f06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -46,8 +46,8 @@ use sqlbuild_python_front::model_config::models::PyNode; use sqlbuild_python_project::compiled_project::models::AuditBlockRow; use sqlbuild_python_project::compiled_project::models::AuditFileRow; use sqlbuild_python_project::compiled_project::models::AuditSettings; -use sqlbuild_python_project::compiled_views::models::AuditHandles; -use sqlbuild_python_project::compiled_views::types::AttachedAudit; +use sqlbuild_python_project::resource_views::models::AuditHandles; +use sqlbuild_python_project::resource_views::types::AttachedAudit; const THRESHOLDS_KEY: &str = "thresholds"; const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 11da72c9af..7fd436c750 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -46,8 +46,8 @@ use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; use sqlbuild_python_project::compiled_project::models::FunctionRow; use sqlbuild_python_project::compiled_project::models::FunctionSettings; -use sqlbuild_python_project::compiled_views::models::FunctionHandles; -use sqlbuild_python_project::compiled_views::types::AttachedFunction; +use sqlbuild_python_project::resource_views::models::FunctionHandles; +use sqlbuild_python_project::resource_views::types::AttachedFunction; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs index 589559d103..25ff2af941 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs @@ -15,6 +15,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::declaration_rows_function::declaration_rows_function; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; @@ -378,8 +379,7 @@ pub(crate) fn model_loop_classes<'py>( py, vec![( "declaration_rows", - sqlbuild_python_project::compiled_views::main::records::declaration_rows_function::declaration_rows_function(py)? - .into_any(), + declaration_rows_function(py)?.into_any(), )], )?; dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 7e1e2b5e99..92f6014339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -110,12 +110,12 @@ use sqlbuild_python_project::compiled_project::models::TestFileRow; use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; use sqlbuild_python_project::compiled_project::types::SourceRow; use sqlbuild_python_project::compiled_project::types::SourceSettings; -use sqlbuild_python_project::compiled_views::main::source_views::retain_sources::retain_sources; use sqlbuild_python_project::compiled_views::models::NativeModelInput; -use sqlbuild_python_project::compiled_views::models::NativeSourceInput; -use sqlbuild_python_project::compiled_views::types::AttachedAudit; -use sqlbuild_python_project::compiled_views::types::AttachedFunction; -use sqlbuild_python_project::compiled_views::types::AttachedSource; +use sqlbuild_python_project::resource_views::main::source_views::retain_sources::retain_sources; +use sqlbuild_python_project::resource_views::models::NativeSourceInput; +use sqlbuild_python_project::resource_views::types::AttachedAudit; +use sqlbuild_python_project::resource_views::types::AttachedFunction; +use sqlbuild_python_project::resource_views::types::AttachedSource; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index 87ff131735..b4213a78c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -45,7 +45,7 @@ use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::HeaderFamilies; use sqlbuild_python_front::model_config::models::NativeColumnSpec; use sqlbuild_python_front::model_config::types::HeaderValue; -use sqlbuild_python_project::compiled_project::main::schema_facts::family_fact::family_fact; +use sqlbuild_python_project::compiled_project::main::family_fact::family_fact; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index be3d594607..5a10f397dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -34,10 +34,10 @@ use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::models::ScannedReference; use sqlbuild_python_project::compiled_project::constants::CONTRACT_KEY; use sqlbuild_python_project::compiled_project::constants::INCREMENTAL_MODE_KEY; -use sqlbuild_python_project::compiled_project::main::project::config_values::config_values; -use sqlbuild_python_project::compiled_project::main::project::declarations::declarations; -use sqlbuild_python_project::compiled_project::main::project::materialization::materialization; -use sqlbuild_python_project::compiled_project::main::project::text_value::text_value; +use sqlbuild_python_project::compiled_project::main::config_values::config_values; +use sqlbuild_python_project::compiled_project::main::declarations::declarations; +use sqlbuild_python_project::compiled_project::main::materialization::materialization; +use sqlbuild_python_project::compiled_project::main::text_value::text_value; use sqlbuild_python_project::compiled_project::models::DeclarationPy; use sqlbuild_python_project::compiled_views::models::InputProjections; use sqlbuild_python_project::compiled_views::models::ModelInputHandles; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs index 94f123ba8f..51a18e67a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_er use crate::bindings::_helpers::input_loops::model_loop_inputs::present; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::models::HeaderColumn; -use sqlbuild_python_project::compiled_project::main::schema_facts::python_column_fact::python_column_fact; +use sqlbuild_python_project::compiled_project::main::python_column_fact::python_column_fact; const MODEL_SCHEMA_KEY: &str = "model_schema"; const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs index 98aceeb651..65d017392a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs @@ -1,7 +1,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::input_loops::resource_attachment::RESOURCE_ATTACHMENT; -use sqlbuild_python_project::compiled_project::main::attachment_inputs::install::install; +use sqlbuild_python_project::compiled_project::main::install::install; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { install(RESOURCE_ATTACHMENT); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index eeeffff647..e6c890408a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -23,13 +23,13 @@ use sqlbuild_python_project::compiled_project::models::TestFileRow; use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; use sqlbuild_python_project::compiled_project::types::SourceRow; use sqlbuild_python_project::compiled_project::types::SourceSettings; -use sqlbuild_python_project::compiled_views::models::NativeAuditInput; -use sqlbuild_python_project::compiled_views::models::NativeFunctionInput; -use sqlbuild_python_project::compiled_views::models::NativeSourceInput; -use sqlbuild_python_project::compiled_views::models::SeedParts; -use sqlbuild_python_project::compiled_views::types::AttachedFunction; -use sqlbuild_python_project::compiled_views::types::AttachedSource; -use sqlbuild_python_project::compiled_views::types::SeedViews; +use sqlbuild_python_project::resource_views::models::NativeAuditInput; +use sqlbuild_python_project::resource_views::models::NativeFunctionInput; +use sqlbuild_python_project::resource_views::models::NativeSourceInput; +use sqlbuild_python_project::resource_views::models::SeedParts; +use sqlbuild_python_project::resource_views::types::AttachedFunction; +use sqlbuild_python_project::resource_views::types::AttachedSource; +use sqlbuild_python_project::resource_views::types::SeedViews; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; const SOURCE_OWNER: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 80d65ebb2a..fe58536011 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -36,8 +36,8 @@ use sqlbuild_python_project::compiled_project::models::PythonResourceRow; use sqlbuild_python_project::compiled_project::models::ScenarioDescriptionRow; use sqlbuild_python_project::compiled_project::types::LoaderRow; use sqlbuild_python_project::compiled_project::types::ResourceDiagnostics; -use sqlbuild_python_project::compiled_views::models::SeedParts; -use sqlbuild_python_project::compiled_views::types::SeedViews; +use sqlbuild_python_project::resource_views::models::SeedParts; +use sqlbuild_python_project::resource_views::types::SeedViews; const MISSING_DESCRIPTION_CODE: &str = "P010"; const DESCRIPTION_LENGTH_CODE: &str = "P027"; @@ -405,7 +405,7 @@ fn docstring_text(description: &Bound<'_, PyString>) -> String { /// A source's P010, falling back to its loader's description and naming the loader in help. fn source_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - source: &sqlbuild_python_project::compiled_views::types::SourceRecord, + source: &sqlbuild_python_project::resource_views::types::SourceRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = source.attached(); @@ -452,7 +452,7 @@ fn source_description<'py>( /// A SQL or Python function's P010, located at its header or `@udf` decorator. fn function_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - function: &sqlbuild_python_project::compiled_views::types::FunctionRecord, + function: &sqlbuild_python_project::resource_views::types::FunctionRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = &function.attached; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index b2f49ad343..ad171a1351 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -37,9 +37,9 @@ use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; use sqlbuild_python_project::compiled_project::types::SourceRow; use sqlbuild_python_project::compiled_project::types::SourceSettings; -use sqlbuild_python_project::compiled_views::main::source_views::replaced::replaced; -use sqlbuild_python_project::compiled_views::models::SourceHandles; -use sqlbuild_python_project::compiled_views::types::AttachedSource; +use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; +use sqlbuild_python_project::resource_views::models::SourceHandles; +use sqlbuild_python_project::resource_views::types::AttachedSource; const MACRO_TOKEN: char = '@'; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 7ae05f852a..6cb701b356 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -52,7 +52,7 @@ use sqlbuild_python_project::compiled_project::models::SqlTestSettings; use sqlbuild_python_project::compiled_project::models::TestBlockRow; use sqlbuild_python_project::compiled_project::models::TestFileRow; use sqlbuild_python_project::compiled_project::types::TestCaseRow; -use sqlbuild_python_project::compiled_views::main::source_views::replaced::replaced; +use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; const MODEL_MODE: &str = "model"; const MACRO_MODE: &str = "macro"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index b5010bee7a..82be27aa65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -20,7 +20,7 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_front::state_rows::main::migrations::read_histories::read_histories as read_view_histories; use sqlbuild_python_front::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; @@ -378,7 +378,7 @@ pub(crate) fn pending_migration_origins( continue; } let model_name: String = - sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( &model, )?; for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 9cf0abb825..da32d00298 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -34,7 +34,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::models::ProjectCatalog; -use sqlbuild_python_project::compiled_views::main::model_reads::star_flags::star_flags; +use sqlbuild_python_project::compiled_views::main::compiled_models::star_flags::star_flags; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const FAST_MODE: &str = "fast"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs index 1092f50850..587a6747cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs @@ -6,8 +6,8 @@ use sqlbuild_analysis::lineage::models::{ }; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::declared_rows::declared_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::declared_rows::declared_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; type TypedColumns = Vec<(String, Option)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 5f0ff9bd7a..aa218631f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -36,9 +36,9 @@ use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; -use sqlbuild_python_project::compiled_project::types::NodeKeys; use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::project_snapshot::types::NodeKeys; fn status(py: Python<'_>, message: &str) -> PyResult<()> { write_stderr(py, &format!("{message}\n")) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 86260de532..879f3acf19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -24,17 +24,17 @@ use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; -use sqlbuild_python_project::compiled_project::main::snapshot_declarations::compile_declarations::compile_declarations; -use sqlbuild_python_project::compiled_project::main::snapshot_declarations::python_declarations::python_declarations; -use sqlbuild_python_project::compiled_project::main::snapshot_models::model_nodes::model_nodes; -use sqlbuild_python_project::compiled_project::main::snapshot_resources::resource_nodes::resource_nodes; -use sqlbuild_python_project::compiled_project::main::snapshot_roles::compile_project_roles::compile_project_roles; -use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::graph::models::ProjectSelection; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; use sqlbuild_python_project::planner_scope::main::project_graph::project_graph; +use sqlbuild_python_project::project_snapshot::main::compile_declarations::compile_declarations; +use sqlbuild_python_project::project_snapshot::main::compile_project_roles::compile_project_roles; +use sqlbuild_python_project::project_snapshot::main::model_nodes::model_nodes; +use sqlbuild_python_project::project_snapshot::main::python_declarations::python_declarations; +use sqlbuild_python_project::project_snapshot::main::resource_nodes::resource_nodes; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index edb35a9292..b95a1a7b94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -13,7 +13,7 @@ use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_se use sqlbuild_python_base::orchestration_reads::main::provider_class_settings::provider_class_settings; use sqlbuild_python_base::orchestration_reads::main::setting_rows::setting_rows; use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_project::compiled_project::main::snapshot_boundary::relative_path::relative_path; +use sqlbuild_python_project::project_snapshot::main::relative_path::relative_path; /// How a consumer's code runs inside other snapshot nodes, besides its own node. #[derive(Clone, Copy)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4fc281df93..a7203be03f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -48,11 +48,11 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; -use sqlbuild_python_project::compiled_project::models::NativeOrchestrationSnapshot; -use sqlbuild_python_project::compiled_project::models::NativeOrchestrationWorker; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; -use sqlbuild_python_project::compiled_project::types::NodeKeys; -use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationWorker; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::NodeKeys; /// Each audit and SQL test the snapshot publishes, with the node keys it checks. fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index e1f2510e89..ec89f1474c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -22,7 +22,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_project::graph::main::views::native_graph_from_edges::native_graph_from_edges; +use sqlbuild_python_project::graph::main::project::native_graph_from_edges::native_graph_from_edges; use sqlbuild_python_project::graph::models::NativeProjectGraph; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index af841ab91b..f05239f1ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -23,7 +23,7 @@ use sqlbuild_python_front::discovery::main::python_files::provider_by_name::prov use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; use sqlbuild_python_front::discovery::models::SecretPolicy; -use sqlbuild_python_project::compiled_project::main::snapshot_declarations::python_declarations::python_declarations; +use sqlbuild_python_project::project_snapshot::main::python_declarations::python_declarations; pub(crate) fn evaluate_sensor( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index aa60faa737..61e4e26117 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -49,7 +49,7 @@ use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; use sqlbuild_python_front::authored_values::main::sql_value::sql_value; use sqlbuild_python_front::authored_values::models::SqlValueClasses; -use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; /// One read-only adapter session serves every page of one synchronization. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index 3cde5a20f5..d553a72682 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -19,9 +19,9 @@ use sqlbuild_python_front::native_host::main::append_observations::append_observ use sqlbuild_python_front::native_host::models::ConsumptionConnection; use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; -use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::config_str::config_str; /// Dependency kind of a source in a model's `dep_rows`. @@ -45,7 +45,7 @@ fn cursor_reads( ) -> PyResult> { let succeeded: HashSet<&str> = successful_models.iter().map(String::as_str).collect(); let mut reads: Vec = Vec::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; let name: String = name_object(&model)?.extract()?; if !succeeded.contains(name.as_str()) @@ -56,10 +56,7 @@ fn cursor_reads( let Some(cursor_column) = config_str(&model, "cursor")? else { continue; }; - let destination: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?; + let destination: Bound<'_, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; let qualified: Bound<'_, PyAny> = native_field(&destination, "qualified_name")?; if qualified.is_none() || native_field(&destination, "schema")?.is_none() { continue; @@ -95,12 +92,9 @@ fn source_column_rows( if described.is_empty() { return Ok(BTreeMap::new()); } - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let destination: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?; + let destination: Bound<'_, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; if native_field(&destination, "schema")?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index cfcb2bfb8c..f5a3ad7932 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -11,9 +11,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_front::model_config::main::config::config_text::config_text; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::function_reads::language::language; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; use sqlbuild_analysis::planner_changes::constants::{ CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, @@ -49,11 +49,12 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal as native_types_equal; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::star_flags::star_flags; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::star_flags::star_flags; use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::function_reads::key::key; use sqlbuild_python_project::semantic_shapes::main::snapshot_validity_columns::snapshot_validity_columns; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; @@ -115,23 +116,19 @@ pub(crate) fn detect_changes<'py>( native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? .try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?; + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = vec![( "fingerprint_sql", PyString::new(py, &fingerprint_sql).into_any(), )]; if selected.selected.contains( - &sqlbuild_python_project::graph::main::keys::native_key::native_key( - &sqlbuild_python_project::compiled_views::main::function_reads::key::key( - &function, - )?, - )?, + &sqlbuild_python_project::graph::main::nodes::native_key::native_key(&key(&function)?)?, ) { let reason: &str = if full_refresh { "full_refresh" @@ -230,9 +227,7 @@ pub(crate) fn scope_model_changes<'py>( } }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - project, - )?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, native_field(project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -514,7 +509,7 @@ fn model_schema_findings<'py>( warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); } let schema_entry: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; let mut yml: Vec<(String, String)> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 704dad962f..37a0ab4c90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -21,10 +21,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::model_reads::references::references; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 91738ddaaa..d958342e66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -29,12 +29,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; -use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; -use sqlbuild_python_project::graph::main::views::edge_dict::edge_dict; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::graph::models::NativeProjectGraph; use sqlbuild_python_project::graph::models::PyNodeSet; @@ -42,6 +40,9 @@ use sqlbuild_python_project::planner_identity::main::execution_deps::execution_d use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_selection::main::resolve_graph_selection::resolve_graph_selection; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; const ENABLED_KEY: &str = "enabled"; @@ -141,22 +142,19 @@ fn resolved_source_entries<'py>( let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let read = PySet::empty(py)?; let mut readers: Vec> = Vec::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; if recreated_view(&model, &selected)? { readers.push(model); } } for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? .try_iter()? { let function = function?; if selected.contains( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, )? { readers.push(function); } @@ -173,7 +171,7 @@ fn resolved_source_entries<'py>( for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; if read.contains( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, )? { entries.push(source_entry(&source)?); } @@ -187,31 +185,24 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = object( - py, - build_model_locations( - &sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)?, - )?, - )?; + let model_locations = object(py, build_model_locations(&compiled_models(project)?)?)?; let seed_locations = object( py, build_seed_locations( - &sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, + &sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, )?, )?; let function_locations = object( py, build_function_locations( - &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - project, - )?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, )?, )?; let source_map = PyDict::new(py); for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; source_map.set_item( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, source_entry(&source)?, )?; } @@ -257,7 +248,7 @@ fn model_entry<'py>( "reason", planner_type(py, RecordId::PlanReason, "NO_CHANGE")?, ), - ("destination", sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(model)?), + ("destination", sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?), ("fingerprint_query_sql", string(py, "")), ("resolved_sql", resolved_sql), ("logical_ddl", string(py, "")), @@ -273,10 +264,7 @@ fn clone_model_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; if !selected.contains(model_key(&model)?)? || disabled_model(&model)? { continue; @@ -285,14 +273,14 @@ fn clone_model_entries<'py>( let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + sqlbuild_python_project::compiled_views::main::project::functions::functions( project, )? .try_iter()? { let function = function?; functions.set_item( - sqlbuild_python_project::compiled_views::main::function_reads::name::name( + sqlbuild_python_project::resource_views::main::function_reads::name::name( &function, )?, function, @@ -355,14 +343,13 @@ fn clone_source_entries<'py>( ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); - for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? - .try_iter()? + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? { let source = source?; let entry = source_entry(&source)?; if !selected.contains( - sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, )? || !native_field(&entry, "managed")?.is_truthy()? || native_field(&entry, "loader")?.is_none() || !native_field(&entry, "expression")?.is_none() @@ -394,20 +381,20 @@ fn clone_source_entries<'py>( }), )?; entries.set_item( - sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, planner_model( py, RecordId::CloneSourcePlanEntry, &[ ( "key", - sqlbuild_python_project::compiled_views::main::source_reads::key::key( + sqlbuild_python_project::resource_views::main::source_reads::key::key( &source, )?, ), ( "name", - sqlbuild_python_project::compiled_views::main::source_reads::name::name( + sqlbuild_python_project::resource_views::main::source_reads::name::name( &source, )?, ), @@ -428,12 +415,12 @@ fn clone_function_entries<'py>( let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? .try_iter()? { let function = function?; if !selected.contains( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, )? { continue; } @@ -485,7 +472,7 @@ fn clone_function_entries<'py>( ], )?; entries.set_item( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, entry, )?; } @@ -498,9 +485,9 @@ fn seed_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { py, RecordId::SeedPlanEntry, &[ - ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(seed)?), - ("name", sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?), - ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(seed)?), + ("name", sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(seed)?), ( "file_path", native_field(&seed_file(seed)?, "file_path")?, @@ -521,8 +508,8 @@ fn seed_entries<'py>( wanted: &Bound<'py, PyAny>, ) -> PyResult> { let mut entries: Vec> = Vec::new(); - for seed in sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)? - .try_iter()? + for seed in + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?.try_iter()? { let seed = seed?; if wanted.contains(native_field(&seed, field)?)? { @@ -538,10 +525,7 @@ fn origin_model_entries<'py>( ) -> PyResult> { let py = project.py(); let mut entries: Vec> = Vec::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; if !selected_names.contains(name_object(&model)?)? || disabled_model(&model)? { continue; @@ -568,19 +552,18 @@ fn origin_source_entries<'py>( let py = project.py(); let mut keys: Vec> = Vec::new(); let mut order: Vec> = Vec::new(); - for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? - .try_iter()? + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? { let source = source?; if selected_names.contains( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, )? { keys.push( - sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, ); } - order.push(sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?); + order.push(sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?); } clone_source_entries( project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 9cd6b18a13..8e402988e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -27,15 +27,15 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::function_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; -use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; -use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use std::collections::{BTreeMap, BTreeSet}; /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. @@ -165,7 +165,7 @@ pub(crate) fn build_audit_command_plan<'py>( ) -> PyResult> { let py = project.py(); let materializations = PyDict::new(py); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; materializations.set_item( name_object(&model)?, @@ -207,12 +207,12 @@ fn called_function_keys<'py>( let py: Python<'py> = project.py(); let functions: Bound<'py, PyDict> = PyDict::new(py); for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? .try_iter()? { let function = function?; functions.set_item( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, function, )?; } @@ -284,14 +284,12 @@ pub(crate) fn build_test_command_plan<'py>( let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - &project, - )? - .try_iter()? + sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)? + .try_iter()? { let function = function?; if !function_keys.contains( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, )? { continue; } @@ -353,7 +351,7 @@ pub(crate) fn build_seed_command_plan<'py>( for seed in seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys - .contains(sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?)? + .contains(sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?)? { continue; } @@ -361,7 +359,7 @@ pub(crate) fn build_seed_command_plan<'py>( let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); let previous = mapping_get( &existing, - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?, + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?, )?; let reason = if previous.is_none() { "FIRST_RUN" @@ -374,9 +372,9 @@ pub(crate) fn build_seed_command_plan<'py>( py, RecordId::SeedPlanEntry, &[ - ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?), - ("name", sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?), - ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?), + ("name", sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", native_field(&seed_file(&seed)?, "file_path")?, @@ -431,7 +429,7 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( for seed in seeds.try_iter()? { let seed = seed?; let destination = - sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( + sqlbuild_python_project::resource_views::main::seed_reads::destination::destination( &seed, )?; let schema = native_field(&destination, "schema")?; @@ -445,7 +443,7 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( .or_insert_with(|| ((database, schema), BTreeSet::new())) .1 .insert( - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? .extract()?, ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index 28ced3eb0b..0a182d7c85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -13,11 +13,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::graph::main::views::edge_dict::edge_dict; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_phases::main::object::object; @@ -63,7 +63,9 @@ fn display_entry<'py>( )? .is_truthy()?; let query_sql = - sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql(model)?; + sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql( + model, + )?; let custom_name = if materialization_type.eq(CUSTOM)? { config(model, "materialized")? } else { @@ -144,7 +146,7 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .collect::>>()?, )?; let mut entries: Vec> = Vec::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; if selected.contains(name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 5551fef3cd..403ee43e17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -43,8 +43,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::microbatch_guidance; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; @@ -426,7 +426,7 @@ pub(crate) fn plan_entries<'py>( for function in functions(&project)?.try_iter()? { let function = function?; functions_by_name.set_item( - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?, function, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index dddf2165ee..ee41bcdd48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -18,24 +18,25 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::function_reads::arguments::arguments; -use sqlbuild_python_project::compiled_views::main::function_reads::entry_point::entry_point; -use sqlbuild_python_project::compiled_views::main::function_reads::fingerprint_destination::fingerprint_destination; -use sqlbuild_python_project::compiled_views::main::function_reads::language::language; -use sqlbuild_python_project::compiled_views::main::function_reads::packages::packages; -use sqlbuild_python_project::compiled_views::main::function_reads::relative_path::relative_path; -use sqlbuild_python_project::compiled_views::main::function_reads::return_columns::return_columns; -use sqlbuild_python_project::compiled_views::main::function_reads::returns::returns; -use sqlbuild_python_project::compiled_views::main::function_reads::runtime_version::runtime_version; -use sqlbuild_python_project::compiled_views::main::function_reads::source_file_path::source_file_path; -use sqlbuild_python_project::compiled_views::main::project_reads::sql_tests::sql_tests; -use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; -use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::build_source_load_entries::build_source_load_entries; use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::fingerprint_destination::fingerprint_destination; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::relative_path::relative_path; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::resource_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; const LOADER_CONSUMER: &str = "loader"; @@ -117,23 +118,23 @@ fn seed_entries<'py>( let metadata_jsons = native_field(extras, "seed_metadata_jsons")?; let plan_reasons = native_field(extras, "seed_plan_reasons")?; let mut entries: Vec> = Vec::new(); - for seed in sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)? - .try_iter()? + for seed in + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys - .contains(sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?)? + .contains(sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?)? { continue; } - let name = sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?; + let name = sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?; entries.push(planner_model( py, RecordId::SeedPlanEntry, &[ - ("key", sqlbuild_python_project::compiled_views::main::seed_reads::key::key(&seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?), ("name", name.clone()), - ("destination", sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", native_field(&seed_file(&seed)?, "file_path")?, @@ -389,10 +390,10 @@ pub(super) fn function_entry<'py>( py, RecordId::FunctionPlanEntry, &[ - ("key", sqlbuild_python_project::compiled_views::main::function_reads::key::key(function)?), - ("name", sqlbuild_python_project::compiled_views::main::function_reads::name::name(function)?), + ("key", sqlbuild_python_project::resource_views::main::function_reads::key::key(function)?), + ("name", sqlbuild_python_project::resource_views::main::function_reads::name::name(function)?), ("relative_path", relative_path(function)?), - ("destination", sqlbuild_python_project::compiled_views::main::function_reads::destination::destination(function)?), + ("destination", sqlbuild_python_project::resource_views::main::function_reads::destination::destination(function)?), ("arguments", arguments(function)?), ("returns", returns(function)?), ("body_sql", body_sql), @@ -430,17 +431,17 @@ fn function_entries<'py>( let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let mut entries: Vec> = Vec::new(); for function in - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)? + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? .try_iter()? { let function = function?; if !selected_keys.contains( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, )? { continue; } let name = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)?; + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?; let change = functions.get_item(&name)?; let previous_query_sql = if previous.contains(&name)? { native_field(&previous.get_item(&name)?, "definition")? @@ -797,6 +798,6 @@ fn build_selected_test_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function(module)?)?; + module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index af455f52a1..efaf15b9fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -22,16 +22,16 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; -use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; -use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::keys::keys; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::main::source_load_map::source_load_map; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; use std::collections::HashSet; /// Relation keys needed to render selected audits without selecting those relations. @@ -106,7 +106,7 @@ pub(crate) fn planner_relations_context<'py>( deferral }; let selected_keys = native_field(&scope, "selected_keys")?; - let mut model_locations = object(py, build_model_locations(&models(&project)?)?)?; + let mut model_locations = object(py, build_model_locations(&compiled_models(&project)?)?)?; let mut seed_locations = object(py, build_seed_locations(&seeds(&project)?)?)?; let function_locations = object(py, build_function_locations(&functions(&project)?)?)?; let deferred_locations = native_field(&deferral, "deferred_locations")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index f0fa1f9685..13e30cf38e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -26,9 +26,9 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; @@ -123,20 +123,20 @@ pub(crate) fn direct_model_version_identities<'py>( for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: String = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? .extract()?; let Some(function_hash) = function_hashes.get_item(&name)? else { continue; }; graph.nodes.insert( graph_key( - &sqlbuild_python_project::compiled_views::main::function_reads::key::key( + &sqlbuild_python_project::resource_views::main::function_reads::key::key( &function, )?, )?, IdentityNode { upstream_keys: graph_keys( - &sqlbuild_python_project::compiled_views::main::function_reads::deps::deps( + &sqlbuild_python_project::resource_views::main::function_reads::deps::deps( &function, )?, )?, @@ -151,7 +151,7 @@ pub(crate) fn direct_model_version_identities<'py>( let seed: Bound<'py, PyAny> = seed?; let (seed_hash, metadata_json): (String, String) = seed_identity(&seed)?; let name: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?; + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?; seed_hashes.set_item(&name, &seed_hash)?; seed_metadata.set_item(&name, metadata_json)?; } @@ -208,7 +208,7 @@ pub(crate) fn direct_model_version_identities<'py>( key_of(MODEL_RESOURCE, &name), IdentityNode { upstream_keys: graph_keys( - &sqlbuild_python_project::compiled_views::main::model_reads::deps::deps( + &sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps( &model, )?, )?, @@ -314,7 +314,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { upstream_keys: graph_keys( - &sqlbuild_python_project::compiled_views::main::model_reads::deps::deps( + &sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps( &model, )?, )?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index bf77c9e628..f9a3ca7624 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -29,20 +29,20 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::function_reads::arguments::arguments; -use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; -use sqlbuild_python_project::compiled_views::main::function_reads::entry_point::entry_point; -use sqlbuild_python_project::compiled_views::main::function_reads::packages::packages; -use sqlbuild_python_project::compiled_views::main::function_reads::return_columns::return_columns; -use sqlbuild_python_project::compiled_views::main::function_reads::returns::returns; -use sqlbuild_python_project::compiled_views::main::function_reads::runtime_version::runtime_version; -use sqlbuild_python_project::compiled_views::main::model_reads::config::config; -use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::seed_reads::external::external; -use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::resource_views::main::seed_reads::external::external; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -191,7 +191,7 @@ fn contract_output_signature<'py>( ) -> PyResult>> { let py: Python<'py> = model.py(); let schema_entry: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; if schema_entry.is_none() { @@ -208,7 +208,7 @@ fn contract_output_signature<'py>( return Ok(None); } let enum_columns: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::enum_columns::enum_columns( + sqlbuild_python_project::compiled_views::main::compiled_models::enum_columns::enum_columns( model, )?; let rendered: Bound<'py, PyList> = PyList::empty(py); @@ -274,7 +274,7 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { let py: Python<'_> = model.py(); let schema_entry: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; let payload: Bound<'_, PyDict> = PyDict::new(py); @@ -317,7 +317,7 @@ fn recorded_function_unchanged( } let py: Python<'_> = function.py(); let language: String = - sqlbuild_python_project::compiled_views::main::function_reads::language::language( + sqlbuild_python_project::resource_views::main::function_reads::language::language( function, )? .str()? @@ -375,10 +375,10 @@ pub(crate) fn function_local_hashes<'py>( for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: String = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? .extract()?; let language: Bound<'py, PyAny> = native_field( - &sqlbuild_python_project::compiled_views::main::function_reads::language::language( + &sqlbuild_python_project::resource_views::main::function_reads::language::language( &function, )?, "value", @@ -464,7 +464,7 @@ fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { Ok(function_fingerprint_sql(&FunctionFingerprintFields { language: - sqlbuild_python_project::compiled_views::main::function_reads::language::language( + sqlbuild_python_project::resource_views::main::function_reads::language::language( function, )? .str()? @@ -486,7 +486,7 @@ pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { let py: Python<'_> = seed.py(); let schema_entry: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry( + sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry( seed, )?; let csv_settings: Bound<'_, PyAny> = native_field(&schema_entry, "csv_settings")?; @@ -505,7 +505,7 @@ pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String let payload: Bound<'_, PyDict> = PyDict::new(py); payload.set_item( "name", - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(seed)?, + sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?, )?; payload.set_item( "csv_settings", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index f4ea999b7a..6f8df1dd2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -11,9 +11,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(super) fn candidates<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index b950a5095c..1ea1e3457b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -20,8 +20,8 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_column::newest_column; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn decide<'py>( @@ -70,7 +70,7 @@ pub(super) fn decide<'py>( } } let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)?.call1(( - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &candidate.model, )?, ))?; @@ -111,7 +111,7 @@ pub(super) fn decide<'py>( RecordId::ColumnMigrationPlanEntry, &[ ("model_name", name_object(&candidate.model)?), - ("destination", sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&candidate.model)?), + ("destination", sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&candidate.model)?), ( "origin_column", if let Some(origin) = origin { @@ -166,7 +166,7 @@ pub(super) fn checked<'py>( }); } let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &candidate.model, )?; let qualified = native_field(&destination, "qualified_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 8939aadbb1..4c39e4bb25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -8,8 +8,8 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::planner_identity::main::model_definition_hash::model_definition_hash; use sqlbuild_python_project::planner_phases::main::object::object; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index c069479ec7..f8302572dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -15,9 +15,9 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 80fed3706d..77ea8f59e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -13,8 +13,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; /// Decide column renames and return the existing planning result model. #[pyfunction(name = "plan_column_migrations")] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 46c66a6a9a..fd9d453326 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -18,8 +18,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { config_value(model.py(), &model_values(model)?, key)?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index fd05522d0e..c9164a2d4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -13,7 +13,7 @@ use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use std::collections::BTreeSet; /// Manual and automatically discovered migration requests with their destination fingerprints. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 4db9bddcab..dc9590f417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -10,7 +10,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; @@ -115,10 +115,7 @@ pub(super) fn resumed<'py>( if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { continue; } - let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - item, - )?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(item)?; let event = newest(&events, &destination)?; if event.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 6f8938dff5..ad16642e95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -12,10 +12,11 @@ use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_orde use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_project::compiled_views::main::model_reads::deps::deps; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; @@ -23,11 +24,7 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in sqlbuild_python_project::compiled_views::main::project_reads::models::models( - &runtime_field(runtime, |runtime| &runtime.project)?, - )? - .try_iter()? - { + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; let name: String = name_object(&item)?.extract()?; let mut upstream: Vec = Vec::new(); @@ -77,7 +74,7 @@ pub(super) fn metadata<'py>( let functions = object( py, build_function_local_hashes( - &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( + &sqlbuild_python_project::compiled_views::main::project::functions::functions( &project, )?, native_field(&project, "sql_analysis_dialect")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index dfbe2bf286..9af31f02f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -19,9 +19,9 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 9dc771ef71..53d4a0e078 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migrati use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::native_host::models::ModelLookup; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index 45c2568b3f..821272dfc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -11,9 +11,9 @@ use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use std::collections::{BTreeMap, BTreeSet}; fn variants<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 23cbf5d42a..9e7f22bb28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -22,7 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -77,7 +77,7 @@ pub(super) fn decide<'py>( let request = decision_request.request; let item = request.model.clone(); let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &item, )?; let origin = decision_request.origin.clone(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 7ad3c7c271..cd6e60dc9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -7,8 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::event_rows::presumed_utc::presumed_utc; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use sqlbuild_python_project::planner_phases::main::object::object; @@ -54,7 +54,7 @@ pub(super) fn handover<'py>( metadata = native_field(&json, "dumps")?.call((payload,), Some(&kwargs))?; } let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( model, )?; replace( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 73adf1da2c..4f9a734749 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -25,8 +25,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; @@ -102,7 +102,9 @@ fn discover<'py>( let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? { + for item in + compiled_models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? + { if let Some(request) = declared(&item?)? { declared_requests.push(request); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 3dbd2b5546..eb374c54d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -19,8 +19,8 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; impl<'py> OldNamePlanner<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index a08ea9ba27..4f7811e34b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -11,7 +11,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -45,7 +45,7 @@ pub(super) fn retention<'py>( ) -> PyResult> { old_name_retention( &native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::config::config(item)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config(item)?, "values", )?, project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index de43d54488..4dbe2e9c79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -14,9 +14,10 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::HashMap; @@ -81,10 +82,9 @@ pub(crate) fn plan_old_name_views<'py>( let runtime = phase.runtime.clone(); let scope = scope_data(&phase.scope)?; let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let models: Vec> = - sqlbuild_python_project::compiled_views::main::project_reads::models::models(&project)? - .try_iter()? - .collect::>()?; + let models: Vec> = compiled_models(&project)? + .try_iter()? + .collect::>()?; let mut by_name: HashMap> = HashMap::new(); for item in &models { by_name.insert(model_name(item)?, item.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index d2f3f40209..1f5087138f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -16,8 +16,8 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, @@ -102,11 +102,7 @@ pub(crate) fn old_name_claim_errors<'py>( if !kind.eq("view")? { continue; } - let destination = relation( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?, - )?; + let destination = relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)?; for view in arguments.value("views")?.try_iter()? { let view = view?; if native_field(&view, "destination_model")?.eq(&name)? @@ -160,13 +156,9 @@ pub(crate) fn old_name_view_skip_reason<'py>( let mut reuser: Option = None; for model in arguments.value("models")?.try_iter()? { let model = model?; - if relation( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?, - )? - .call_method1("matches", (&old,))? - .is_truthy()? + if relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? + .call_method1("matches", (&old,))? + .is_truthy()? { reuser = Some(name_object(&model)?.extract()?); break; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 0960fe2804..53fb78acba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -12,8 +12,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, @@ -34,7 +34,7 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; if name_object(&item)?.eq(name)? { return Ok((native_field(&item, "destination")?, name.clone())); @@ -71,13 +71,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; - for item in models(&project)?.try_iter()? { - let database = native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &item?, - )?, - "database", - )?; + for item in compiled_models(&project)?.try_iter()? { + let database = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "database")?; if !database.is_none() { return Ok(database); } @@ -88,13 +83,8 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { - let schema = native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &item?, - )?, - "schema", - )?; + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { + let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -149,7 +139,7 @@ pub(super) fn resolve_origin<'py>( }; let item = request.model.clone(); let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &item, )?; let mut parts: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 3ff4f9fe2c..9dbc69077b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -11,13 +11,13 @@ use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::compiled_views::main::seed_reads::external::external; -use sqlbuild_python_project::compiled_views::main::seed_reads::key::key; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::seed_reads::external::external; +use sqlbuild_python_project::resource_views::main::seed_reads::key::key; pub(crate) fn check_selected_scope_buildability<'py>( runtime: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index fbdb9de575..ef44a32502 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -9,9 +9,9 @@ use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_phases::main::strings::strings; @@ -40,7 +40,7 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( cli_full_refresh: bool, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { names.push(name_object(&model)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 37c74cb8b6..19ae42d660 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -17,8 +17,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeMap, HashSet}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index e7b19fb1b7..0db0b3022e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -24,8 +24,9 @@ use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::construct::construct; use sqlbuild_python_project::planner_phases::main::member::member; use sqlbuild_python_project::planner_phases::main::native::native; @@ -69,10 +70,8 @@ fn selected_models<'py>( ) -> PyResult>> { let scope = scope_data(scope)?; let mut models: Vec<_> = Vec::new(); - for model in sqlbuild_python_project::compiled_views::main::project_reads::models::models( - &runtime_field(runtime, |runtime| &runtime.project)?, - )? - .try_iter()? + for model in + compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let model = model?; if scope @@ -104,7 +103,9 @@ pub(crate) fn plan_table_types<'py>( for model in &models { if native_field( &native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + model, + )?, "table_type", )?, "declared", @@ -164,10 +165,7 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; let limit: i64 = adapter .call_method0("maximum_identifier_length")? .extract()?; @@ -355,7 +353,7 @@ fn relation_retention<'py>( } let desired_days: i64 = native_field(&retention, "desired_days")?.extract()?; let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( model, )?; let schema = native_field(&destination, "schema")?; @@ -495,10 +493,7 @@ fn bigquery_retention<'py>( { let model = model?; let retention = native_field(&native_field(&model, "config")?, "time_travel_retention")?; - let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; if native_field(&retention, "unmanaged")?.is_truthy()? || native_field(&retention, "desired_days")?.is_none() || native_field(&destination, "schema")?.is_none() @@ -515,10 +510,7 @@ fn bigquery_retention<'py>( } let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); for model in models { - let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - model, - )?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?; let database: Option = native_field(&destination, "database")?.extract()?; let schema: String = native_field(&destination, "schema")?.str()?.extract()?; scopes.insert((database.clone().unwrap_or_default(), schema, database)); @@ -624,7 +616,9 @@ pub(crate) fn plan_retention<'py>( let mut models: Vec<_> = Vec::new(); for model in selected_models(runtime, scope)? { let retention = native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::config::config(&model)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + &model, + )?, "time_travel_retention", )?; if !native_field(&retention, "unmanaged")?.is_truthy()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index ab52663c8c..a682495b34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -19,7 +19,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::main::views::compiled_graph_keys::compiled_graph_keys; +use sqlbuild_python_project::graph::main::project::compiled_graph_keys::compiled_graph_keys; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::time::Instant; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 0a9324e570..5cbd26a115 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -32,7 +32,7 @@ use sqlbuild_python_front::model_config::main::config::config_cursor_bound::conf use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -66,7 +66,7 @@ fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> return Ok(false); } let destination = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( model, )?; let key = call_with( @@ -84,9 +84,7 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> let target = native_field(snapshot, "target_relation")?; Ok(!target.is_none() && target.ne(native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - model, - )?, + &sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?, "qualified_name", )?)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 46dc2c47a5..b5d65ccb4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -17,7 +17,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index a1372a0ce0..605b995324 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -44,7 +44,7 @@ use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs index ca79e40fb3..01961f6424 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination; -use sqlbuild_python_project::compiled_views::main::resource_reads::name::name; +use sqlbuild_python_project::resource_views::main::resource_reads::destination::destination; +use sqlbuild_python_project::resource_views::main::resource_reads::name::name; fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let result = PyDict::new(items.py()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index bda4cad401..453f203e19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -33,9 +33,9 @@ use sqlbuild_python_front::model_config::main::config::config_text_object::confi use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; const APPEND_STRATEGY: &str = "append"; @@ -330,7 +330,7 @@ pub(crate) fn resolve_function_sql<'py>( }; let function = args.value("function")?; let name: String = - sqlbuild_python_project::compiled_views::main::function_reads::name::name(&function)? + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? .extract()?; let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; Ok(PyString::new(py, &sql).into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index f8779a0d4e..ab2e5b1c87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -19,7 +19,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::object::object; #[pyfunction] @@ -141,17 +142,14 @@ fn namespace<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let models = PyDict::new(py); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; models.set_item(name_object(&model)?, model)?; } for field in ["target_model_names", "model_names"] { for name in native_field(graph, field)?.try_iter()? { if let Some(model) = models.get_item(name?)? { - let destination = sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; return Ok(( native_field(&destination, "database")?, native_field(&destination, "schema")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 2a8feb2211..b26466437c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -15,9 +15,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::seed_reads::key::key; -use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; -use sqlbuild_python_project::compiled_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::resource_views::main::seed_reads::key::key; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; pub(super) fn seed_entries<'py>( project: &Bound<'py, PyAny>, @@ -26,7 +26,7 @@ pub(super) fn seed_entries<'py>( ) -> PyResult> { let py = project.py(); let seeds = by_attribute( - &sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, + &sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, "name", )?; let fixtures = native_field(graph, "seed_fixture_names")?; @@ -59,7 +59,7 @@ pub(super) fn seed_entries<'py>( ("key", key(&seed)?), ( "name", - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)?, + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?, ), ("destination", destination), ("file_path", native_field(&seed_file(&seed)?, "file_path")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 20b15a47c1..d8b51c4d17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -34,10 +34,11 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::records::compiled_model_record::compiled_model_record; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::compiled_model_record::compiled_model_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; @@ -102,20 +103,13 @@ pub(crate) fn build_scenario_execution_plan<'py>( } else { analysis.is_truthy()? }; - let models = by_attribute( - &sqlbuild_python_project::compiled_views::main::project_reads::models::models(&project)?, - "name", - )?; + let models = by_attribute(&compiled_models(&project)?, "name")?; let functions = by_attribute( - &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - &project, - )?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)?, "key", )?; let locations = build_function_locations( - &sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - &project, - )?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)?, )? .into_any(); let upstream = execution_deps(&project)?; @@ -288,9 +282,14 @@ fn model_context<'py>( ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), models_by_name: models.clone().into_any().into(), - functions_by_name: by_attribute(&sqlbuild_python_project::compiled_views::main::project_reads::functions::functions(project)?, "name")? - .into_any() - .unbind(), + functions_by_name: by_attribute( + &sqlbuild_python_project::compiled_views::main::project::functions::functions( + project, + )?, + "name", + )? + .into_any() + .unbind(), seed_locations: native_field(&context.relations, "seed_locations")?.unbind(), function_locations: locations.clone().unbind(), source_map: native_field(&context.relations, "source_map")?.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index da872f0498..e369c1a279 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -10,11 +10,11 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; #[pyfunction] @@ -38,7 +38,7 @@ pub(crate) fn plan_scenario_graph<'py>( dbt_refs: native_field(scenario, "dbt_ref_fixture_names")?.extract()?, ..ScenarioGraphInput::default() }; - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; let deps = native_field(&model, "deps")? .try_iter()? @@ -48,13 +48,13 @@ pub(crate) fn plan_scenario_graph<'py>( } for source in sources(project)?.try_iter()? { input.project_sources.insert( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source?)? + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source?)? .extract()?, ); } for seed in seeds(project)?.try_iter()? { input.project_seeds.insert( - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed?)? + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed?)? .extract()?, ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index a62aa5995d..02f6055a68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -14,7 +14,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; #[pyfunction(name = "build_scenario_relation_plan")] #[pyo3(signature = (**kwargs))] @@ -65,14 +65,13 @@ pub(crate) fn build_scenario_relation_plan<'py>( )?; } let sources = PyDict::new(py); - for source in sqlbuild_python_project::compiled_views::main::project_reads::sources::sources( - &args.project, - )? - .try_iter()? + for source in + sqlbuild_python_project::compiled_views::main::project::sources::sources(&args.project)? + .try_iter()? { let source = source?; sources.set_item( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, source_entry(&source)?, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 10ff313842..b3d3ebd8e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -16,28 +16,28 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_project::auditing::main::render_audit_sql::render_audit_sql; -use sqlbuild_python_project::compiled_views::main::audit_reads::always_run::always_run; -use sqlbuild_python_project::compiled_views::main::audit_reads::attached_column_name::attached_column_name; -use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_kind::attached_target_kind; -use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_name::attached_target_name; -use sqlbuild_python_project::compiled_views::main::audit_reads::definition_name::definition_name; -use sqlbuild_python_project::compiled_views::main::audit_reads::description::description; -use sqlbuild_python_project::compiled_views::main::audit_reads::evaluation_mode::evaluation_mode; -use sqlbuild_python_project::compiled_views::main::audit_reads::evidence_limit::evidence_limit; -use sqlbuild_python_project::compiled_views::main::audit_reads::evidence_sql::evidence_sql; -use sqlbuild_python_project::compiled_views::main::audit_reads::measurement_contract::measurement_contract; -use sqlbuild_python_project::compiled_views::main::audit_reads::meta::meta; -use sqlbuild_python_project::compiled_views::main::audit_reads::minimum_samples::minimum_samples; -use sqlbuild_python_project::compiled_views::main::audit_reads::run_scope::run_scope; -use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; -use sqlbuild_python_project::compiled_views::main::audit_reads::sql_body::sql_body; -use sqlbuild_python_project::compiled_views::main::audit_reads::thresholds::thresholds; -use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::NativeProjectGraph; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::edges::edges; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::audit_reads::always_run::always_run; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_column_name::attached_column_name; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::definition_name::definition_name; +use sqlbuild_python_project::resource_views::main::audit_reads::description::description; +use sqlbuild_python_project::resource_views::main::audit_reads::evaluation_mode::evaluation_mode; +use sqlbuild_python_project::resource_views::main::audit_reads::evidence_limit::evidence_limit; +use sqlbuild_python_project::resource_views::main::audit_reads::evidence_sql::evidence_sql; +use sqlbuild_python_project::resource_views::main::audit_reads::measurement_contract::measurement_contract; +use sqlbuild_python_project::resource_views::main::audit_reads::meta::meta; +use sqlbuild_python_project::resource_views::main::audit_reads::minimum_samples::minimum_samples; +use sqlbuild_python_project::resource_views::main::audit_reads::run_scope::run_scope; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::resource_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::resource_views::main::audit_reads::thresholds::thresholds; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; @@ -130,7 +130,7 @@ fn plan_audit<'py>( let py = audit.py(); let audit = audit.clone(); let name: String = - sqlbuild_python_project::compiled_views::main::audit_reads::name::name(&audit)? + sqlbuild_python_project::resource_views::main::audit_reads::name::name(&audit)? .extract()?; let resolved = render_query( request, @@ -162,7 +162,7 @@ fn plan_audit<'py>( }; let requested = audit_enum(run_scope(&audit)?, "AuditRunScope", "final")?; let severity = audit_enum( - sqlbuild_python_project::compiled_views::main::audit_reads::severity::severity(&audit)?, + sqlbuild_python_project::resource_views::main::audit_reads::severity::severity(&audit)?, "AuditSeverity", "warn", )?; @@ -176,11 +176,11 @@ fn plan_audit<'py>( let reads: [(&str, AuditRead<'py>); 12] = [ ( "key", - sqlbuild_python_project::compiled_views::main::audit_reads::key::key, + sqlbuild_python_project::resource_views::main::audit_reads::key::key, ), ( "name", - sqlbuild_python_project::compiled_views::main::audit_reads::name::name, + sqlbuild_python_project::resource_views::main::audit_reads::name::name, ), ("definition_name", definition_name), ("evaluation_mode", evaluation_mode), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 9a11ddb71e..3c4e27c50a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -19,12 +19,12 @@ use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_kind::attached_target_kind; -use sqlbuild_python_project::compiled_views::main::audit_reads::attached_target_name::attached_target_name; -use sqlbuild_python_project::compiled_views::main::audit_reads::references::references; -use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_scope::main::edges::edges; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::references::references; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -111,7 +111,7 @@ pub(crate) fn attachment( &format!( "audit '{}': attached audit is missing an attached target name", text( - &sqlbuild_python_project::compiled_views::main::audit_reads::name::name( + &sqlbuild_python_project::resource_views::main::audit_reads::name::name( audit )? )? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 19479b4834..a2dd9a7f6a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -111,7 +111,9 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let source_map: &Bound<'py, PyAny> = &lookups.source_map; let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; let model_name: String = - sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name(model)?; + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?; for role in ["filter_inputs", "watermark_inputs"] { for item in mapping_items(&native_field(&roles, role)?)?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index c68388c7db..40fdaa4647 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -19,8 +19,8 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::compiled_views::main::model_reads::config::config; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::native_host::models::ModelLookup; const MICROBATCH_MODE: &str = "microbatch"; @@ -44,7 +44,7 @@ fn mapped_references<'py>( } fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { - sqlbuild_python_project::compiled_views::main::resource_reads::references::references(owner)? + sqlbuild_python_project::resource_views::main::resource_reads::references::references(owner)? .try_iter()? .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs index f21d6ca8e6..9153633367 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::records::replace::replace; +use sqlbuild_python_project::compiled_views::main::project::replace::replace; const ERROR_ACTION: &str = "error"; const SKIP_ACTION: &str = "skip"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 4a18d4b8f1..831cad5042 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -45,12 +45,12 @@ use sqlbuild_python_front::model_config::main::config::config_text_tuple::config use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_front::model_config::types::ConfigValues; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_rows::inferred_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::model_reads::relative_path::relative_path; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::relative_path::relative_path; use sqlbuild_python_project::planner_strategy::constants::CUSTOM_MATERIALIZATION; use sqlbuild_python_project::planner_strategy::constants::DDL_SEPARATOR; use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; @@ -119,7 +119,7 @@ fn schema_columns<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let entry: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::resource_reads::schema_entry::schema_entry( + sqlbuild_python_project::resource_views::main::resource_reads::schema_entry::schema_entry( model, )?; if entry.is_none() { @@ -215,7 +215,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); let entry: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; if !entry.is_none() { @@ -275,7 +275,7 @@ fn logical_ddl<'py>( let adapter: &Bound<'py, PyAny> = &request.adapter; let model: &Bound<'py, PyAny> = &request.model; let destination: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( model, )?; let qualified: Bound<'py, PyAny> = native_field(&destination, "qualified_name")?; @@ -340,7 +340,7 @@ fn logical_ddl<'py>( /// The output column names SQL analysis inferred for the model, when it inferred them. fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let inferred: Bound<'py, PyAny> = sqlbuild_python_project::compiled_views::main::model_reads::inferred_columns::inferred_columns(model)?; + let inferred: Bound<'py, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::inferred_columns::inferred_columns(model)?; if inferred.is_none() { return Ok(none(py)); } @@ -608,7 +608,7 @@ fn contract_fields<'py>( )?; entry.set_item("contract_columns", schema_columns(model, false)?)?; let schema_entry: Bound<'py, PyAny> = - sqlbuild_python_project::compiled_views::main::model_reads::schema_entry::schema_entry( + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( model, )?; entry.set_item( @@ -687,7 +687,9 @@ pub(crate) fn assemble_model_plan_entry<'py>( let table_type: Bound<'py, PyAny> = native_field( &native_field( &native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + model, + )?, "table_type", )?, "value", @@ -705,7 +707,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item("relative_path", relative_path(model)?)?; entry.set_item( "destination", - sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( model, )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 1752fef375..5ae6a39806 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -5,6 +5,8 @@ use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::planner_strategy::{audits, cursor_types, microbatches, strategy}; use sqlbuild_python_project::planner_strategy::main::register_fixture_completion::register_fixture_completion; +use sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context; +use sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -38,8 +40,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audits::register(module)?; cursor_types::register(module)?; register_fixture_completion(module)?; - sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context(module)?; - sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads(module)?; + register_fixture_context(module)?; + register_fixture_reads(module)?; microbatches::register(module)?; strategy::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index 6cc856aad8..d68f6cd890 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -81,9 +81,10 @@ pub(crate) fn resolve_model_plan_action<'py>( let py: Python<'py> = model.py(); let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; let strategy = ModelStrategy { - name: sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name( - model, - )?, + name: + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?, disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, materialization: model_materialization(model)?, incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index adf6e684b1..877eb665ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -40,9 +40,9 @@ use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::compiled_views::main::model_reads::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; use sqlbuild_python_project::planner_phases::main::by_name::by_name; use sqlbuild_python_project::planner_phases::main::config_str::config_str; use sqlbuild_python_project::planner_phases::main::model_key::model_key; @@ -54,6 +54,7 @@ use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -218,7 +219,7 @@ fn runtime_producer_names( let source = source?; if !native_field(&source_entry(&source)?, "loader")?.is_none() { loaded.insert( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? .extract()?, ); } @@ -246,7 +247,7 @@ fn upstream_relation<'py>( let name_text: String = name.extract()?; let is_selected = selected.is_some_and(|names| names.contains(&name_text)); let qualified = |node: Bound<'py, PyAny>| { - native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "qualified_name") + native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "qualified_name") }; if (kind.eq("ref")? || kind.eq("seed")?) && !deferred.is_none() @@ -387,10 +388,7 @@ fn collect_models<'py>( }; let runtime_names = runtime_producer_names(project, &producers)?; let mut models: Vec<_> = Vec::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; let name_value = name_object(&model)?; let model_name: String = name_value.extract()?; @@ -432,12 +430,7 @@ fn collect_models<'py>( None => None, }; let target_info = existing_relations.get_item(&name_value)?; - let qualified = native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?, - "qualified_name", - )?; + let qualified = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "qualified_name")?; if let Some(redirected) = redirected.filter(|value| !value.is_none()) { target_tag = Some(format!("{model_name}__target__max")); target_relation = Some(redirected.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index 9d2f515934..d0a0ae05f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -14,9 +14,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::references::references; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use std::collections::BTreeSet; @@ -196,7 +196,7 @@ pub(crate) fn semantic_source_columns<'py>( return Ok(columns.clone()); } let mut required: BTreeSet = BTreeSet::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; let analysis_off: bool = matches!( *config_node_of(&model_values(&model)?, SQL_ANALYSIS_KEY)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 34783f37ee..a16d161f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -29,12 +29,12 @@ use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::model_reads::references::references; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; -use sqlbuild_python_project::graph::main::keys::native_key::native_key; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_phases::main::by_name::by_name; @@ -45,6 +45,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::build_upstream_intermediate_source_map::build_upstream_intermediate_source_map; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. @@ -84,11 +85,9 @@ pub(crate) fn destination_nodes<'py>( ) -> PyResult>> { let mut nodes: Vec<_> = Vec::new(); let collections: [Bound<'py, PyAny>; 3] = [ - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)?, - sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds(project)?, - sqlbuild_python_project::compiled_views::main::project_reads::functions::functions( - project, - )?, + compiled_models(project)?, + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, ]; for collection in collections { for node in collection.try_iter()? { @@ -101,7 +100,7 @@ pub(crate) fn destination_nodes<'py>( fn selected_node(selected: &Bound<'_, PyAny>, node: &Bound<'_, PyAny>) -> PyResult { Ok(selected.is_none() || selected.contains( - sqlbuild_python_project::compiled_views::main::resource_reads::key::key(node)?, + sqlbuild_python_project::resource_views::main::resource_reads::key::key(node)?, )?) } @@ -132,7 +131,7 @@ fn resolve_database<'py>( if !selected_node(selected, &node)? { continue; } - let database = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -149,7 +148,7 @@ fn target_schemas( if !selected_node(selected, &node)? { continue; } - let schema = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -215,7 +214,7 @@ fn metadata_name_filter( let seeds = by_name(project, "seeds")?; let sources = PyDict::new(py); for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? + sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? .try_iter()? { let entry = source_entry(&source?)?; @@ -224,14 +223,14 @@ fn metadata_name_filter( for key in selected.try_iter()? { let name = native_field(&key?, "name")?; if let Some(model) = models.get_item(&name)? { - names.insert(native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "name")?.extract()?); names.extend(model_upstream_names( &model, (&models, &seeds), &selected_names, )?); } else if let Some(seed) = seeds.get_item(&name)? { - names.insert(native_field(&sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination(&seed)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?, "name")?.extract()?); } else if let Some(entry) = sources.get_item(&name)? { names.insert(source_table_name(&entry)?); } @@ -298,16 +297,15 @@ fn prefetch_schemas<'py>( for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); } - for source in sqlbuild_python_project::compiled_views::main::project_reads::sources::sources( - &request.project, - )? - .try_iter()? + for source in + sqlbuild_python_project::compiled_views::main::project::sources::sources(&request.project)? + .try_iter()? { let source = source?; let entry = source_entry(&source)?; if (!relevant.is_none() && !relevant.contains( - sqlbuild_python_project::compiled_views::main::source_reads::key::key(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, )?) || !native_field(&entry, "expression")?.is_none() || native_field(&entry, "schema")?.is_none() @@ -408,8 +406,8 @@ fn gather_relations<'py>( let logical = PyDict::new(py); for node in destination_nodes(&request.project)? { logical.set_item( - location_identity(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?)?, - sqlbuild_python_project::compiled_views::main::resource_reads::name::name(&node)?, + location_identity(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?)?, + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)?, )?; } let result = PyDict::new(py); @@ -460,7 +458,7 @@ fn gather_columns<'py>( let mut node_names: HashSet = HashSet::new(); for node in destination_nodes(&request.project)? { node_names.insert( - sqlbuild_python_project::compiled_views::main::resource_reads::name::name(&node)? + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? .extract()?, ); } @@ -503,10 +501,7 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> .collect::>()?; let mut all_selected = true; let mut any_migration = false; - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model = model?; if !names.contains(&model_name(&model)?) { all_selected = false; @@ -884,7 +879,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( &snapshot, &[( "target_relation", - native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "qualified_name")?, + native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "qualified_name")?, )], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 6210af027b..1047e704e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -12,11 +12,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; fn entries_of<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index f64f63abd5..5b0f7ff821 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -18,23 +18,24 @@ use sqlbuild_python_front::authored_values::main::render_project_var_text::rende use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::audit_reads::references::references; -use sqlbuild_python_project::compiled_views::main::audit_reads::scope_deps::scope_deps; -use sqlbuild_python_project::compiled_views::main::audit_reads::sql_body::sql_body; -use sqlbuild_python_project::compiled_views::main::function_reads::body_sql::body_sql; -use sqlbuild_python_project::compiled_views::main::model_reads::model_key::model_key; -use sqlbuild_python_project::compiled_views::main::model_reads::query_sql::query_sql; -use sqlbuild_python_project::compiled_views::main::project_reads::audits::audits; -use sqlbuild_python_project::compiled_views::main::project_reads::functions::functions; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::sources::sources; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; -use sqlbuild_python_project::compiled_views::main::source_reads::source_file::source_file; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::resource_views::main::audit_reads::references::references; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::resource_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::key::key; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_file::source_file; use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { @@ -99,7 +100,7 @@ fn selected_managed_refs( managed: &BTreeSet, ) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; if !selected.contains(model_key(&model)?)? { continue; @@ -109,9 +110,7 @@ fn selected_managed_refs( } for function in functions(project)?.try_iter()? { let function = function?; - if selected.contains( - sqlbuild_python_project::compiled_views::main::function_reads::key::key(&function)?, - )? { + if selected.contains(key(&function)?)? { names.extend(sql_source_reads(&body_sql(&function)?, managed)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 629dc90bda..5c8143ab3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -73,7 +73,7 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { for node in destination_nodes(project)? { - let database = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -87,7 +87,7 @@ fn state_schemas( ) -> PyResult> { let mut schemas: BTreeSet<_> = BTreeSet::new(); for node in destination_nodes(project)? { - let schema = native_field(&sqlbuild_python_project::compiled_views::main::resource_reads::destination::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -101,7 +101,7 @@ fn state_schemas( let Some(model) = data.model(scope.py(), name) else { continue; }; - let schema = native_field(&sqlbuild_python_project::compiled_views::main::model_reads::destination::destination(&model)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index be4a7dbf1b..ed0277c229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -22,12 +22,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::discovery::main::project::yaml_entry_line::yaml_entry_line; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; -use sqlbuild_python_project::compiled_views::main::source_reads::source_file::source_file; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_file::source_file; use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { @@ -87,7 +87,7 @@ fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { .extract()?; let line = yaml_entry_line( &native_field(&file, "contents")?.extract::()?, - &sqlbuild_python_project::compiled_views::main::source_reads::name::name(source)? + &sqlbuild_python_project::resource_views::main::source_reads::name::name(source)? .extract::()?, ); Ok(match line { @@ -108,13 +108,12 @@ pub(crate) fn check_selected_source_tables_exist<'py>( let read_map = native_field(relations, "source_read_map")?; let listed = native_field(relations, "listed_source_names")?; let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); - for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? - .try_iter()? + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? { let source = source?; let name: String = - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)? + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? .extract()?; let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { continue; @@ -169,7 +168,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let role = native_field(&probe, "role")?; missing.push(MissingSourceTable { - source_name: sqlbuild_python_project::compiled_views::main::source_reads::name::name( + source_name: sqlbuild_python_project::resource_views::main::source_reads::name::name( &source, )? .extract()?, @@ -193,13 +192,12 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); - for source in - sqlbuild_python_project::compiled_views::main::project_reads::sources::sources(project)? - .try_iter()? + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? { let source = source?; sources.set_item( - sqlbuild_python_project::compiled_views::main::source_reads::name::name(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, source, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index 3565911981..ea9d541ea8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -10,12 +10,12 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::seed_reads::schema_entry::schema_entry; -use sqlbuild_python_project::compiled_views::main::source_reads::source_entry::source_entry; use sqlbuild_python_project::planner_phases::main::config_str::config_str; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::HashMap; const CONTRACT_KEY: &str = "contract"; @@ -77,7 +77,9 @@ pub(crate) fn validate_watermark_column<'py>( let kind = native_field(reference, "ref_kind")?; let ref_name: String = native_field(reference, "ref_name")?.extract()?; let model_name: String = - sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name(model)?; + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?; let lowered = |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; let (names, types) = if kind.eq("ref")? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index d43b3032de..df99de24ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -12,12 +12,12 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::enum_member_value::enum_member_value; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_project::main::schema_facts::python_column_facts::python_column_facts; -use sqlbuild_python_project::compiled_views::main::model_input_view::attached_columns::attached_columns; -use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; -use sqlbuild_python_project::compiled_views::main::source_views::read_source_view::read_source_view; -use sqlbuild_python_project::compiled_views::types::AttachedSource; -use sqlbuild_python_project::compiled_views::types::SourceViews; +use sqlbuild_python_project::compiled_project::main::python_column_facts::python_column_facts; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_columns::attached_columns; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::resource_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::resource_views::types::AttachedSource; +use sqlbuild_python_project::resource_views::types::SourceViews; const UNKNOWN_TYPE: &str = "UNKNOWN"; const NOT_NULL_AUDIT_NAME: &str = "not_null"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 98360c9e15..4312270b18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -33,7 +33,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::authored_values::main::project_var_text::project_var_text; use sqlbuild_python_front::model_config::main::config::record_reads::record_reads; -use sqlbuild_python_project::compiled_views::main::model_input_view::input_reference_rows::input_reference_rows; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 95d50398cf..e05f8753e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -9,7 +9,7 @@ use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; -use sqlbuild_python_project::compiled_views::main::model_input_view::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index c2e3ccf08d..866850a577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::query_command::project::{ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { document diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index bfa7a3382d..e87eda9f4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -52,7 +52,7 @@ use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_rep use crate::bindings::_helpers::recall::sinks::InvocationEvents; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs index 020327e3c7..e238e4e34c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -10,7 +10,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::relation_keys::entry_relation_key_row::entry_relation_key_row; +use sqlbuild_python_project::compiled_views::main::compiled_models::entry_relation_key_row::entry_relation_key_row; /// `CompilerDiagnostic` positions. const DIAGNOSTIC_PHASE_FIELD: usize = 0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index fb0b307779..a7992a4914 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -9,7 +9,7 @@ use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::declarations::constant_declaration_sql::constant_declaration_sql; use sqlbuild_python_front::render::main::declarations::render_enum_member_value::render_enum_member_value; -use sqlbuild_python_project::graph::main::project_graph::project_graph::project_graph; +use sqlbuild_python_project::graph::main::project::project_graph::project_graph; use sqlbuild_python_project::graph::models::ResourceRow; /// The parameter name through which a macro receives its context. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs index 5391badc54..53bb674dcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs @@ -1,8 +1,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::references::main::register_relations::register_relations; +use sqlbuild_python_project::references::main::register_source_relations::register_source_relations; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - sqlbuild_python_project::references::main::register_relations::register_relations(module)?; - sqlbuild_python_project::references::main::register_source_relations::register_source_relations( - module, - ) + register_relations(module)?; + register_source_relations(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 545808be50..4ba0c73a2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -32,6 +32,7 @@ use sqlbuild_python_base::build_command::main::boolean::boolean; use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; const KEYED_CHECK: &str = "keyed comparison"; const MAX_EXAMPLE_LENGTH: usize = 80; @@ -77,10 +78,7 @@ fn models_named<'py>( names: &[String], ) -> PyResult)>> { let mut models: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'py, PyAny> = model?; let name: String = native_field(&model, "name")?.extract()?; if names.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs index 4e0c1ad6b2..e87d6ba7e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs @@ -12,8 +12,9 @@ use crate::bindings::_helpers::lineage::project_lineage::build_project_column_li use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; /// The columns of a changed model a comparison proved changed, or `None` when every column may. fn changed_columns(comparison: Option<&NodeComparison>) -> Option> { @@ -45,10 +46,7 @@ pub(crate) fn affected_columns( ) -> PyResult>> { let py: Python<'_> = project.py(); let mut models: BTreeMap> = BTreeMap::new(); - for model in - sqlbuild_python_project::compiled_views::main::project_reads::models::models(project)? - .try_iter()? - { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; let name: String = model_name(&model)?; if affected.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index 7c093111ac..0f2f9634b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -36,11 +36,11 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; use sqlbuild_python_front::discovery::main::project::configured_review::configured_review; -use sqlbuild_python_project::compiled_views::main::model_reads::dep_rows::dep_rows; -use sqlbuild_python_project::compiled_views::main::model_reads::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::project_reads::models::models; -use sqlbuild_python_project::compiled_views::main::project_reads::seeds::seeds; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const BASE_REF_ALIAS: &str = "@base"; @@ -215,7 +215,7 @@ fn key_columns(value: &Bound<'_, PyAny>) -> PyResult> { /// One model as a review node, critical by its `critical` key or one of the critical tags. fn model_node(model: &Bound<'_, PyAny>, critical_tags: &[String]) -> PyResult { let values: Bound<'_, PyAny> = native_field( - &sqlbuild_python_project::compiled_views::main::model_reads::config::config(model)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config(model)?, "values", )?; let tags: Vec = strings(&config_value(&values, "tags")?)?; @@ -384,10 +384,10 @@ fn review_graph( for seed in seeds(project)?.try_iter()? { let seed: Bound<'_, PyAny> = seed?; let name: String = - sqlbuild_python_project::compiled_views::main::seed_reads::name::name(&seed)? + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? .extract()?; if let Some(database) = destination_database( - &sqlbuild_python_project::compiled_views::main::seed_reads::destination::destination( + &sqlbuild_python_project::resource_views::main::seed_reads::destination::destination( &seed, )?, )? { @@ -407,7 +407,7 @@ fn review_graph( ); } let mut dependencies: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for model in models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; let node: ReviewNode = model_node(&model, critical_tags)?; if node.materialized == INCREMENTAL_MATERIALIZATION @@ -415,11 +415,7 @@ fn review_graph( { project_graph.stateful.insert(node.name.clone()); } - if let Some(database) = destination_database( - &sqlbuild_python_project::compiled_views::main::model_reads::destination::destination( - &model, - )?, - )? { + if let Some(database) = destination_database(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? { project_graph.databases.insert(node.name.clone(), database); } project_graph diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs index d77bdcda7a..372d122c2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs @@ -18,11 +18,11 @@ use crate::bindings::_helpers::rules::fact_replay::{ ModelEntry, project_attribute, project_models, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_views::main::model_reads::enum_columns::enum_columns; -use sqlbuild_python_project::compiled_views::main::records::compiled_audit_record::compiled_audit_record; -use sqlbuild_python_project::compiled_views::main::records::compiled_function_record::compiled_function_record; -use sqlbuild_python_project::compiled_views::main::records::compiled_seed_record::compiled_seed_record; -use sqlbuild_python_project::compiled_views::main::records::compiled_source_record::compiled_source_record; +use sqlbuild_python_project::compiled_views::main::compiled_models::enum_columns::enum_columns; +use sqlbuild_python_project::compiled_views::main::project::compiled_audit_record::compiled_audit_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_function_record::compiled_function_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_seed_record::compiled_seed_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_source_record::compiled_source_record; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; /// The first bytes of every payload file; hosts reject any other. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index cb1d9f364d..7a5a3a8d2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; /// One custom rule's cache identity from its metadata, source fingerprint and settings. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 4947b57411..4fbdffac47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -25,8 +25,8 @@ use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_views::main::records::compiled_project_facts::compiled_project_facts; -use sqlbuild_python_project::compiled_views::main::records::record_model_analyses::record_model_analyses; +use sqlbuild_python_project::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; const CANCELLED: &str = "native rules evaluation was cancelled"; const BUILD_PHASE: &str = "early_rules_build_ms"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 20d454a6ac..efb853e5fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -36,10 +36,10 @@ use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interru use sqlbuild_python_base::native_threads::models::Joined; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_front::render::models::SqlExpansionInputs; -use sqlbuild_python_project::compiled_views::main::relation_keys::model_key_row::model_key_row; -use sqlbuild_python_project::compiled_views::main::relation_keys::one_row_seed_names::one_row_seed_names; -use sqlbuild_python_project::compiled_views::main::relation_keys::relation_keys::relation_keys; -use sqlbuild_python_project::compiled_views::main::relation_keys::source_key_input::source_key_input; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key_row::model_key_row; +use sqlbuild_python_project::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::relation_keys::relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::source_key_input::source_key_input; use sqlbuild_python_project::compiled_views::models::RelationKeyRow; use sqlbuild_python_project::compiled_views::types::RelationKeys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 5a6e1e1415..948466a24e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -25,12 +25,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as record_field_or_attribute; use sqlbuild_python_base::records::main::fields::record_spec::record_spec; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::models::NativeCompiledAudit; -use sqlbuild_python_project::compiled_views::models::NativeCompiledFunction; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; -use sqlbuild_python_project::compiled_views::models::NativeCompiledSeed; -use sqlbuild_python_project::compiled_views::models::NativeCompiledSource; use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::resource_views::models::NativeCompiledAudit; +use sqlbuild_python_project::resource_views::models::NativeCompiledFunction; +use sqlbuild_python_project::resource_views::models::NativeCompiledSeed; +use sqlbuild_python_project::resource_views::models::NativeCompiledSource; const MODEL_RECORD: &str = "sqlbuild._native.Model"; const MATERIALIZED: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 6365bd65e7..2132e21dd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -40,7 +40,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::render::models::SqlExpansionInputs; -use sqlbuild_python_project::compiled_views::main::relation_keys::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; use sqlbuild_python_project::compiled_views::types::RelationKeys; const VERIFICATION_FAILURE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 07a6fa78f6..746b7fc8ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -8,7 +8,7 @@ use sqlbuild_rules::lint_text::types::PreparedSql; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::models::Positional; -use sqlbuild_python_project::compiled_views::main::relation_keys::register_relation_keys::register_relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::register_relation_keys::register_relation_keys; #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 50404e7c88..b175023977 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -43,11 +43,11 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::scopes::main::reports::metadata_projection_value::metadata_projection_value; -use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_views::main::records::compiled_project_facts::compiled_project_facts; -use sqlbuild_python_project::compiled_views::main::records::declaration_rows::declaration_rows; -use sqlbuild_python_project::compiled_views::models::NativeCompiledAudit; +use sqlbuild_python_project::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::project::declaration_rows::declaration_rows; +use sqlbuild_python_project::resource_views::models::NativeCompiledAudit; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index a0d61a71c6..7077efccbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -22,7 +22,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; const MODEL_RESOURCE_TYPE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 9cabbfb701..c7a16a07ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -48,8 +48,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::model_reads::output_proven::output_proven; -use sqlbuild_python_project::compiled_views::main::relation_keys::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::output_proven::output_proven; +use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; use sqlbuild_python_project::compiled_views::models::SqlRuleInputs; use sqlbuild_python_project::compiled_views::models::SqlRuleKey; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs index d0ac6723af..88c31f05a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -14,8 +14,8 @@ use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_project::main::values::plain_value::plain_value; -use sqlbuild_python_project::compiled_views::main::records::plain_typed_value::plain_typed_value; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; +use sqlbuild_python_project::compiled_views::main::project::plain_typed_value::plain_typed_value; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 7fa2339aa4..95b3ab4c7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -38,8 +38,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; -use sqlbuild_python_project::compiled_project::types::VerifiedRun; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index bddd0a01f5..e39ee991e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -20,9 +20,9 @@ use crate::bindings::_helpers::semantic_checks::type_recovery::{ use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; -use sqlbuild_python_project::compiled_views::main::model_reads::lineage_inputs::lineage_inputs; -use sqlbuild_python_project::compiled_views::main::records::with_binding_diagnostics::with_binding_diagnostics; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::project::with_binding_diagnostics::with_binding_diagnostics; use sqlbuild_python_project::compiled_views::types::LineageInput; use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index e639693b61..1e90ed2da9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -29,7 +29,7 @@ use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::model_config::models::AuditInstance; use sqlbuild_python_front::model_config::types::HeaderValue; use sqlbuild_python_front::model_config::types::PyConfigNode; -use sqlbuild_python_project::compiled_views::main::model_reads::binding_validated::binding_validated; +use sqlbuild_python_project::compiled_views::main::compiled_models::binding_validated::binding_validated; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index d94db0ea49..4ff5851ff8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -24,8 +24,8 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::models::ProjectCatalog; -use sqlbuild_python_project::compiled_views::main::audit_views::audit_reference_rows::audit_reference_rows; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::resource_views::main::audit_views::audit_reference_rows::audit_reference_rows; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index 6f968d24fb..ed96fc30a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -18,7 +18,7 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::analysis::models::ProjectCatalog; use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; -use sqlbuild_python_project::compiled_views::main::model_reads::output_proven::output_proven; +use sqlbuild_python_project::compiled_views::main::compiled_models::output_proven::output_proven; const NO_CATALOG_MESSAGE: &str = "NativeCompilerError: native semantic completion: the project has no analysis catalog"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index b5ca21d054..330bee76e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -10,7 +10,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::records::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 907d18bf37..cf87c9b8d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -32,9 +32,9 @@ use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::main::schema_validations::schema_validations; use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; -use sqlbuild_python_project::compiled_views::main::model_reads::inferred_names::inferred_names; -use sqlbuild_python_project::compiled_views::main::model_reads::lineage_inputs::lineage_inputs; -use sqlbuild_python_project::compiled_views::main::records::with_type_recovery::with_type_recovery; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::project::with_type_recovery::with_type_recovery; use sqlbuild_python_project::compiled_views::types::LineageInput; use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index 3a84bf7a30..a097921e9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -22,9 +22,9 @@ use sqlbuild_python_base::native_host::main::python_executable::python_executabl use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_project::graph::main::python_node_graph::python_node_graph::python_node_graph; -use sqlbuild_python_project::graph::main::python_node_graph::run_selection_from_inputs::run_selection_from_inputs; -use sqlbuild_python_project::graph::main::python_selection::python_node_names::python_node_names; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; /// The prefix of a Python test module's file name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index f887567e21..7b6a2236b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -69,8 +69,8 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; -use sqlbuild_python_project::compiled_project::models::WorkerBuild; -use sqlbuild_python_project::compiled_project::types::VerifiedRun; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 5d1dc7f379..52e8ff7e43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -4,12 +4,16 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::sql_test_glue::compiled_tests; use sqlbuild_python_front::sql_test_glue::main::register_authored_ctes::register_authored_ctes; +use sqlbuild_python_project::sql_test_glue::main::plan_compiled_sql_test_artifacts_function::plan_compiled_sql_test_artifacts_function; +use sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_function::plan_sql_tests_function; +use sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_natively_function::plan_sql_tests_natively_function; +use sqlbuild_python_project::sql_test_glue::main::sql_test_plan_error_messages_function::sql_test_plan_error_messages_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_function::plan_sql_tests_function(module)?)?; - module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_natively_function::plan_sql_tests_natively_function(module)?)?; - module.add_function(sqlbuild_python_project::sql_test_glue::main::plan_compiled_sql_test_artifacts_function::plan_compiled_sql_test_artifacts_function(module)?)?; - module.add_function(sqlbuild_python_project::sql_test_glue::main::sql_test_plan_error_messages_function::sql_test_plan_error_messages_function(module)?)?; + module.add_function(plan_sql_tests_function(module)?)?; + module.add_function(plan_sql_tests_natively_function(module)?)?; + module.add_function(plan_compiled_sql_test_artifacts_function(module)?)?; + module.add_function(sql_test_plan_error_messages_function(module)?)?; register_authored_ctes(module)?; compiled_tests::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 7a6e4e8166..2eb99d1af5 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -18,6 +18,7 @@ use sqlbuild_python_base::records::main::registry::capture::capture; use sqlbuild_python_base::records::main::registry::register_records::register_records; use sqlbuild_python_base::records::main::registry::verify_attributes::verify_attributes; use sqlbuild_python_base::records::main::registry::verify_tables::verify_tables; +use sqlbuild_python_project::analysis::main::register_normalization::register_normalization; use sqlbuild_python_project::analysis::models::BindingPositions; use sqlbuild_python_project::analysis::models::CompactAnalysisJob; use sqlbuild_python_project::analysis::models::ProjectCatalog; @@ -34,9 +35,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { native_host::registration::register(module)?; rules::registration::register(module)?; cli::registration::register(module)?; - sqlbuild_python_project::analysis::main::register_normalization::register_normalization( - module, - )?; + register_normalization(module)?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index 5ede471c66..57f9266ca1 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1469,11 +1469,11 @@ paths = ["crates/sqlbuild-python-base/src/boundary/models.rs"] reason = "Each fills a missing keyword argument with its own default, None or False." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs"] reason = "Each compiled view reads its own field set from the Python boundary." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python-project/src/compiled_project/_helpers/snapshot_execution.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs"] reason = "Each callback or worker report forwards one distinct build or check event with its own payload." [[dupes.allowlist]] @@ -1676,7 +1676,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147515 } +thresholds = { max_crate_code_lines = 147431 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -1794,6 +1794,66 @@ path = "crates/sqlbuild-python-front/src/render/types.rs" symbols = ["StateTuple"] reason = "Python passes the expansion state as one five-item tuple; StateInput reads it in a single tuple extraction and names each position, as the positional record it replaces did." +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/analysis_session/models.rs" +symbols = ["FamilyRow::extract_bound", "PivotRequestRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_project/models.rs" +symbols = ["FunctionRow::extract_bound", "FunctionSettings::extract_bound", "TestBlockRow::extract_bound", "TestFileRow::extract_bound", "ScenarioRow::extract_bound", "AuditBlockRow::extract_bound", "AuditFileRow::extract_bound", "PythonResourceRow::extract_bound", "ScenarioDescriptionRow::extract_bound", "HookDescriptionRow::extract_bound", "ProofPy::extract_bound", "AnalysisPy::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" +symbols = ["AnalysisProjections", "CompiledProjections", "ModelInputHandles", "InputProjections", "NativeProjectView"] +reason = "AnalysisProjections, CompiledProjections, ModelInputHandles, InputProjections, NativeProjectView implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" +symbols = ["RelationKeyRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/graph/models.rs" +symbols = ["CompiledProjectGraph"] +reason = "CompiledProjectGraph implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/graph/models.rs" +symbols = ["ResourceRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/planner_scope/models.rs" +symbols = ["RetainedScope"] +reason = "RetainedScope implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/planner_selection/models.rs" +symbols = ["Carrier"] +reason = "Carrier implements Debug manually because it holds the interpreter token, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/resource_views/models.rs" +symbols = ["AuditProjections", "FunctionProjections", "NativeCompiledFunction", "SeedHandles", "NativeCompiledSeed", "SourceProjections"] +reason = "AuditProjections, FunctionProjections, NativeCompiledFunction, SeedHandles, NativeCompiledSeed, SourceProjections implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/resource_views/models.rs" +symbols = ["SeedNamespace::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -2027,6 +2087,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/main/registry/native_attr.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2166,26 +2227,22 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs=1", "crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs=4", "crates/sqlbuild-python-project/src/compiled_project/models.rs=2", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs=2", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs=9", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs=3", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs=2", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs=7", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/records.rs=14", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs=9", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs=7", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs=14", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs=6", "crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs=15", "crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs=1", "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=16", "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs=1", - "crates/sqlbuild-python-project/src/graph/_helpers/algorithms.rs=3", - "crates/sqlbuild-python-project/src/graph/_helpers/execution_gates.rs=3", - "crates/sqlbuild-python-project/src/graph/_helpers/python_lifecycle.rs=2", - "crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs=3", - "crates/sqlbuild-python-project/src/graph/_helpers/python_selection.rs=5", + "crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs=2", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs=5", "crates/sqlbuild-python-project/src/graph/models.rs=2", "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=3", - "crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs=19", "crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs=1", "crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs=2", "crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs=2", @@ -2193,23 +2250,26 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs=3", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs=1", "crates/sqlbuild-python-project/src/planner_selection/models.rs=3", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs=2", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_reads.rs=4", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=2", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs=4", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=19", "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=5", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs=2", "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs=1", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs=1", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs=3", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs=13", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs=2", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/fixture_validation.rs=4", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_driver.rs=8", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_errors.rs=4", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_rendering.rs=2", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan_values.rs=2", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs=1", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs=8", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", @@ -2616,6 +2676,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2679,30 +2740,26 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/audit_views.rs=4", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_analysis.rs=3", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_model_view.rs=5", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/function_views.rs=7", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/inputs_view.rs=24", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input_view.rs=5", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/project_view.rs=26", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/seed_views.rs=4", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/source_views.rs=3", - "crates/sqlbuild-python-project/src/compiled_views/models.rs=44", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs=24", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs=26", + "crates/sqlbuild-python-project/src/compiled_views/models.rs=34", "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=12", - "crates/sqlbuild-python-project/src/graph/_helpers/python_node_graph.rs=4", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs=4", "crates/sqlbuild-python-project/src/graph/models.rs=10", "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=2", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs=1", "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_completion.rs=3", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_preparation.rs=6", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixture_sql.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=6", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", "crates/sqlbuild-python-project/src/planner_strategy/models.rs=30", "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=2", "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=2", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/cursor_window.rs=2", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/planning.rs=4", + "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs=3", + "crates/sqlbuild-python-project/src/resource_views/models.rs=33", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=4", "crates/sqlbuild-python-project/src/sql_test_glue/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 4c175d640f..ad2ad7572a 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -4,9 +4,9 @@ crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.prepare_analysis crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.schemas crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs .schemas -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs InferredNullability.value -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs BindingCatalog.expression_shapes -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs BindingCatalog.native +crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs InferredNullability.value +crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.expression_shapes +crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_result @@ -107,22 +107,22 @@ crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.entry_path crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.native crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.outcome -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs LayoutConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs LayoutConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs PythonHookEntry.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs PythonHookEntry.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleExemption.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleExemption.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleIgnore.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleIgnore.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesCacheConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesCacheConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlHookEntry.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlHookEntry.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlTestRulesConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlTestRulesConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs PythonHookEntry.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs PythonHookEntry.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleExemption.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleExemption.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleIgnore.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleIgnore.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesCacheConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesCacheConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEntry.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEntry.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dict__ crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs PreparedCompileArtifacts.start crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.schemas @@ -148,7 +148,7 @@ crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs CompileInp crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs PreparedCompileArtifacts.publish crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs PythonNodeGraph.selected_sql_refs crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs PlannerScopeResolution.stale_warning_scope -crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs CollectionRendering.value +crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs CollectionRendering.value crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs ConnectionProgressReporter.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs DbtInitProgressReporter.complete @@ -328,7 +328,7 @@ crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs Scena crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.model_locations crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.seed_locations crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs CompiledResourceType.value crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action @@ -423,14 +423,14 @@ crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnect crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnectionRelay.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs BindingCatalog.inferred_schema -crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs BindingPositions.position +crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs BindingCatalog.inferred_schema +crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs BindingPositions.position crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs PlanningProgressReporter.on_progress -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs CursorGrain.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs CursorType.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs CompiledResourceType.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs WarningSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs CursorGrain.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs CursorType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs WarningSeverity.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs CompiledResourceType.value crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs OldNameViewEventType.value crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs ColumnMigrationDecision.records_event crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs OldNameViewEventType.value From 7c1089c1f1ea212fb23862411c6ae304eb81c7bf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:11:04 +0000 Subject: [PATCH 05/10] wip: allowlist the duplicate clusters the project move touches Gives each cluster the move touches a reason in [dupes]; fensu dupes reports no new clusters against the slice base. Not tested. --- fensu.toml | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/fensu.toml b/fensu.toml index 57f9266ca1..42905e1d8a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3200,6 +3200,110 @@ reason = "Each host keeps its own first Python error and returns its own failure paths = ["crates/sqlbuild-python-front/src/authored_values/models.rs", "crates/sqlbuild-python-front/src/cursor/models.rs"] reason = "SqlValueClasses::load and CursorSession::new each load the record classes their own conversions need; CursorSession's to_utc and utc_offset each turn the zone call's result into their own value." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs", "crates/sqlbuild-python-project/src/graph/models.rs", "crates/sqlbuild-python-project/src/resource_views/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] +reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] +reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each getter reads its own cached projection through the view's shared read helper." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs"] +reason = "Each reads a resource key's type and name into its own native key; consolidating the key readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each records its own per-resource fact into its own resource list by index." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each returns a view of its own resource list by index after its own bounds check." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] +reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs", "crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs"] +reason = "Each Python entry reads its own keyword arguments and calls the shared planner with its own options." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs"] +reason = "One selects compiled models by name, the other keys a project field's nodes by destination; only the collecting loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/models.rs"] +reason = "Each builds its own resource's key from its own record; only the key construction is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/project_snapshot/models.rs"] +reason = "Each worker report forwards one distinct check event with its own fields." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs"] +reason = "One returns the first model destination's database, the other collects every model destination's schema; only the loop over compiled models is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each compiled view reads its own field set from the Python boundary." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs"] +reason = "Each converts the declaration row of its own consumer into its own declaration type." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs"] +reason = "The analysis session and the compiled model view each build lineage inputs from their own row types." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each getter reads its own cached projection through the view's shared read helper." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "The compiled model and the model input each borrow their own record from the same owner; only the borrow is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "The compiled model and the model input each read the reference rows of their own record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs", "crates/sqlbuild-python-project/src/references/main/match_project_relation.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs", "crates/sqlbuild-python-project/src/planner_phases/main/construct.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs"] +reason = "Each returns its own stored Python handle bound to the interpreter; only the bind-and-clone is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." From d84f8490e4c7c8aed19e43ea27e60a348b804a59 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:23:47 +0000 Subject: [PATCH 06/10] wip: keep schema handles' Debug quiet and drop stale fensu entries SchemaHandles prints no fields again, so audit arguments stay out of Debug output. Removes the budgets of the deleted bindings/models.rs and merges repeated duplicate allowlist entries. --- .../_helpers/project/project_host.rs | 9 ++++++++ .../src/compiled_views/models.rs | 1 - fensu.toml | 22 ++----------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs index 061e60262f..e59b937909 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs @@ -5,6 +5,7 @@ use std::fmt::{Debug, Formatter, Result as FmtResult}; use crate::compiled_views::models::AnalysisProjections; use crate::compiled_views::models::CompiledProjections; use crate::compiled_views::models::ModelInputHandles; +use crate::compiled_views::models::SchemaHandles; use crate::resource_views::models::AuditProjections; use crate::resource_views::models::FunctionProjections; use crate::resource_views::models::SeedHandles; @@ -18,6 +19,14 @@ impl Debug for ModelInputHandles { } } +impl Debug for SchemaHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SchemaHandles") + .finish_non_exhaustive() + } +} + impl Debug for CompiledProjections { fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { formatter diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index 75eb837deb..0f54ef21a8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -1012,7 +1012,6 @@ pub struct InputViewClasses<'py> { } /// A model schema's header columns and specs, which carry user-authored audit arguments. -#[derive(Debug)] pub struct SchemaHandles { pub columns: Arc, pub audits: Arc, diff --git a/fensu.toml b/fensu.toml index 42905e1d8a..500065cff9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1809,8 +1809,8 @@ reason = "Python passes each of these records as one positional tuple; extract_b [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" -symbols = ["AnalysisProjections", "CompiledProjections", "ModelInputHandles", "InputProjections", "NativeProjectView"] -reason = "AnalysisProjections, CompiledProjections, ModelInputHandles, InputProjections, NativeProjectView implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." +symbols = ["AnalysisProjections", "CompiledProjections", "ModelInputHandles", "InputProjections", "NativeProjectView", "SchemaHandles"] +reason = "AnalysisProjections, CompiledProjections, ModelInputHandles, InputProjections, NativeProjectView implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; SchemaHandles implements it manually so the user-authored audit arguments its specs carry stay out of logs; other models derive it." [[targets.rust.rule_exceptions]] rule = "FPRSS012" @@ -2664,7 +2664,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", - "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] @@ -2925,7 +2924,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=8", - "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB075] @@ -3212,10 +3210,6 @@ reason = "Each matches its own native class's attribute names to its own getters paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs"] reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] -reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." @@ -3236,14 +3230,6 @@ reason = "Each records its own per-resource fact into its own resource list by i paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] reason = "Each returns a view of its own resource list by index after its own bounds check." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs"] -reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] -reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs", "crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] reason = "Each builds its own index from its own node list; only the index-building loop is alike." @@ -3280,10 +3266,6 @@ reason = "Each converts the declaration row of its own consumer into its own dec paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs"] reason = "The analysis session and the compiled model view each build lineage inputs from their own row types." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] -reason = "Each getter reads its own cached projection through the view's shared read helper." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] reason = "The compiled model and the model input each borrow their own record from the same owner; only the borrow is alike." From af3661411dd2f2df6077febd482656fa01d94aba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:51:09 +0000 Subject: [PATCH 07/10] wip: move forwarded helper bodies into their main entries and regroup split imports --- .../attachments/_helpers/argument_values.rs | 53 +- .../attachments/_helpers/ceremonial_select.rs | 14 +- .../attachments/_helpers/cursor_intrinsics.rs | 133 +---- .../attachments/_helpers/function_headers.rs | 26 +- .../attachments/_helpers/reference_markers.rs | 52 +- .../main/analysis_cursor_splices.rs | 17 +- .../attachments/main/analysis_cursor_sql.rs | 13 +- .../src/attachments/main/argument_pairs.rs | 16 +- .../main/assert_no_unresolved_sql_markers.rs | 43 +- .../attachments/main/compile_input_error.rs | 11 +- .../attachments/main/config_argument_value.rs | 31 +- .../src/attachments/main/parsed_header.rs | 24 +- .../main/register_ceremonial_select.rs | 11 +- .../main/register_cursor_intrinsics.rs | 15 +- .../main/register_reference_markers.rs | 11 +- .../main/reserved_cursor_markers.rs | 8 +- .../main/resolve_cursor_intrinsics.rs | 24 +- .../main/validated_model_cursor_sql.rs | 31 +- .../src/authored_values/_helpers/fields.rs | 18 +- .../authored_values/_helpers/project_vars.rs | 66 +-- .../authored_values/_helpers/python_nodes.rs | 24 +- .../authored_values/_helpers/sql_values.rs | 47 +- .../authored_values/main/project_var_text.rs | 42 +- .../src/authored_values/main/raw_value.rs | 4 +- .../authored_values/main/register_fields.rs | 13 +- .../main/register_project_vars.rs | 11 +- .../main/register_python_nodes.rs | 17 +- .../main/register_sql_values.rs | 10 +- .../main/render_project_var_text.rs | 12 +- .../main/render_project_var_text_function.rs | 5 +- .../src/authored_values/main/sql_value.rs | 13 +- .../authored_values/main/sql_value_error.rs | 19 +- .../command_runtime/_helpers/concurrency.rs | 60 +-- .../main/resolve_concurrency.rs | 17 +- .../main/resolve_configured_concurrency.rs | 44 +- .../_helpers/effective_config.rs | 227 +-------- .../main/build_effective_connection.rs | 16 +- .../main/build_effective_connection_config.rs | 27 +- ...ld_effective_connection_config_function.rs | 5 +- .../build_effective_connection_function.rs | 5 +- .../main/build_effective_runtime_config.rs | 19 +- ...build_effective_runtime_config_function.rs | 5 +- .../main/build_effective_settings.rs | 39 +- .../main/build_effective_settings_config.rs | 16 +- ...uild_effective_settings_config_function.rs | 5 +- .../main/build_effective_settings_function.rs | 5 +- .../main/build_effective_target_config.rs | 12 +- .../build_effective_target_config_function.rs | 5 +- .../main/build_effective_vars.rs | 21 +- .../main/build_effective_vars_function.rs | 5 +- .../compile_attachment/main/project_vars.rs | 16 +- .../main/target_selection.rs | 21 +- .../_helpers/config_validation.rs | 50 +- ...reject_invalid_connection_configuration.rs | 7 +- .../reject_invalid_project_configuration.rs | 7 +- .../main/session_warehouse_support.rs | 19 +- .../contracts/_helpers/effective_config.rs | 149 +----- .../src/contracts/_helpers/spec_resolution.rs | 270 +--------- .../main/effective_config_settings.rs | 41 +- .../contracts/main/effective_config_vars.rs | 25 +- .../main/effective_connection_fields.rs | 71 ++- .../main/loader_destination_parts.rs | 13 +- .../src/contracts/main/loader_parts.rs | 52 +- .../main/matching_dynamic_families.rs | 19 +- .../main/register_effective_config.rs | 19 +- .../main/register_spec_resolution.rs | 31 +- .../main/resolve_effective_adapter_name.rs | 16 +- .../resolve_effective_collection_rendering.rs | 15 +- .../main/resolve_effective_scenario_config.rs | 68 ++- .../contracts/main/resolve_target_config.rs | 47 +- .../src/contracts/main/resolve_target_name.rs | 19 +- .../src/cursor/_helpers/conversion.rs | 11 - .../src/cursor/_helpers/durations.rs | 91 +--- .../src/cursor/_helpers/intervals.rs | 70 +-- .../src/cursor/_helpers/mod.rs | 1 - .../src/cursor/_helpers/overrides.rs | 58 +-- .../src/cursor/_helpers/scalars.rs | 187 +------ .../src/cursor/_helpers/values.rs | 39 +- .../cursor/main/conversion/algebra_error.rs | 9 +- .../main/durations/parse_cursor_duration.rs | 30 +- .../main/durations/register_durations.rs | 13 +- .../durations/shift_by_cursor_duration.rs | 13 +- .../cursor/main/durations/shifted_moment.rs | 42 +- .../main/intervals/interval_validation.rs | 20 +- .../src/cursor/main/intervals/partition.rs | 17 +- .../main/intervals/register_intervals.rs | 25 +- .../cursor/main/overrides/cursor_integer.rs | 34 +- .../cursor/main/overrides/cursor_timestamp.rs | 30 +- .../src/cursor/main/scalars/as_scalar.rs | 12 +- .../src/cursor/main/scalars/compare_values.rs | 21 +- .../src/cursor/main/scalars/parse.rs | 12 +- .../src/cursor/main/scalars/parse_value.rs | 43 +- .../cursor/main/scalars/register_scalars.rs | 35 +- .../src/cursor/main/scalars/render_value.rs | 16 +- .../cursor/main/scalars/sentinel_to_token.rs | 18 +- .../src/cursor/main/scalars/to_exclusive.rs | 13 +- .../src/cursor/main/scalars/to_inclusive.rs | 13 +- .../src/cursor/main/scalars/try_parse.rs | 23 +- .../src/cursor/main/values/local_date_time.rs | 16 +- .../src/cursor/main/values/replaced.rs | 27 +- .../src/cursor/models.rs | 6 +- .../discovery/_helpers/failures/guidance.rs | 89 +--- .../src/discovery/_helpers/failures/raised.rs | 12 +- .../_helpers/headers/header_declarations.rs | 10 +- .../_helpers/headers/header_values.rs | 18 +- .../_helpers/headers/hook_entries.rs | 55 +- .../discovery/_helpers/headers/key_checks.rs | 12 +- .../_helpers/project/configuration.rs | 60 +-- .../discovery/_helpers/project/locations.rs | 20 +- .../_helpers/project/project_config.rs | 135 +---- .../_helpers/project/project_facts.rs | 121 +---- .../_helpers/project/project_files.rs | 26 +- .../_helpers/project/project_validation.rs | 68 +-- .../_helpers/python_files/imported_modules.rs | 26 +- .../python_files/imported_providers.rs | 51 +- .../python_files/imported_python_nodes.rs | 148 +----- .../python_files/python_function_files.rs | 48 +- .../python_files/python_hook_files.rs | 25 +- .../python_files/python_module_files.rs | 8 +- .../_helpers/python_files/python_paths.rs | 30 +- .../_helpers/records/declaration_records.rs | 186 +------ .../_helpers/records/description_inputs.rs | 29 +- .../_helpers/records/model_file_record.rs | 12 - .../_helpers/records/model_records.rs | 110 +--- .../_helpers/records/sql_test_records.rs | 123 +---- .../_helpers/records/yaml_records.rs | 170 +------ .../_helpers/snapshots/scope_snapshot.rs | 49 +- .../discovery/_helpers/snapshots/snapshot.rs | 55 +- .../_helpers/sources/declaration_files.rs | 13 +- .../_helpers/sources/orchestration_files.rs | 33 +- .../_helpers/sources/provider_secrets.rs | 46 +- .../_helpers/sources/yaml_entries.rs | 11 +- .../main/failures/explicit_references_help.rs | 8 +- .../main/failures/microbatch_guidance.rs | 7 +- .../main/failures/register_guidance.rs | 19 +- .../main/failures/register_raised.rs | 10 +- .../discovery/main/failures/setting_help.rs | 32 +- .../main/failures/sql_analysis_guidance.rs | 22 +- .../discovery/main/headers/hook_entries.rs | 40 +- .../headers/register_header_declarations.rs | 10 +- .../main/headers/register_header_values.rs | 14 +- .../main/headers/register_hook_entries.rs | 11 +- .../main/headers/register_key_checks.rs | 12 +- .../src/discovery/main/project/column_name.rs | 9 +- .../main/project/configured_changes.rs | 10 +- .../main/project/configured_git_base_ref.rs | 10 +- .../main/project/configured_review.rs | 10 +- .../project/discover_configured_providers.rs | 37 +- .../discover_connection_configuration.rs | 11 +- .../project/discover_project_configuration.rs | 10 +- .../src/discovery/main/project/each.rs | 4 +- .../main/project/load_local_config_file.rs | 20 +- .../main/project/load_project_config_file.rs | 20 +- .../discovery/main/project/named_node_rows.rs | 24 +- .../discovery/main/project/project_facts.rs | 60 ++- .../main/project/register_locations.rs | 13 +- .../main/project/register_project_config.rs | 16 +- .../main/project/register_project_files.rs | 15 +- .../project/register_project_validation.rs | 13 +- .../main/project/resolve_config_file_path.rs | 19 +- .../main/project/seed_csv_settings.rs | 18 +- .../src/discovery/main/project/stem.rs | 10 +- .../project/validate_local_sql_analysis.rs | 18 +- .../main/project/validation_error.rs | 43 +- .../main/project/warn_deprecated_settings.rs | 19 +- .../discovery/main/project/yaml_entry_line.rs | 10 +- .../discover_event_exporter_functions.rs | 33 +- .../python_files/discover_hook_functions.rs | 4 +- .../discover_materialization_files.rs | 53 +- .../python_files/discover_provider_classes.rs | 4 +- .../discover_python_function_files.rs | 31 +- .../discover_python_node_functions.rs | 70 ++- .../python_files/install_recall_writer.rs | 5 +- .../main/python_files/node_identity.rs | 5 +- .../main/python_files/provider_by_name.rs | 16 +- .../main/python_files/provider_usages.rs | 17 +- .../register_imported_python_nodes.rs | 25 +- .../register_python_function_files.rs | 10 +- .../register_python_hook_files.rs | 10 +- .../register_python_module_files.rs | 10 +- .../reject_unsupported_python_paths.rs | 28 +- .../main/records/discover_adapter_file.rs | 26 +- .../main/records/discover_audit_files.rs | 18 +- .../main/records/discover_constant_files.rs | 18 +- .../main/records/discover_enum_files.rs | 18 +- .../main/records/discover_macro_files.rs | 18 +- .../main/records/discover_model_files.rs | 44 +- .../records/discover_model_schema_files.rs | 18 +- .../main/records/discover_scenario_files.rs | 40 +- .../main/records/discover_schema_files.rs | 78 ++- .../main/records/discover_seed_files.rs | 14 +- .../main/records/discover_source_files.rs | 62 ++- .../records/discover_sql_function_files.rs | 18 +- .../main/records/discover_sql_hook_files.rs | 18 +- .../main/records/discover_test_files.rs | 41 +- .../main/records/discovered_model_files.rs | 41 +- .../records/externally_described_models.rs | 16 +- .../main/records/parse_sql_test_file.rs | 23 +- .../discovery/main/records/projected_text.rs | 11 +- .../records/register_declaration_records.rs | 29 +- .../records/register_description_inputs.rs | 11 +- .../main/records/register_model_records.rs | 10 +- .../main/records/register_sql_test_records.rs | 16 +- .../main/records/register_yaml_records.rs | 16 +- .../records/retained_discovery_session.rs | 19 +- .../snapshots/directory_snapshot_scope.rs | 13 +- .../main/snapshots/discover_scope_snapshot.rs | 13 +- .../src/discovery/main/snapshots/escaped.rs | 23 +- .../discovery/main/snapshots/in_snapshot.rs | 4 +- .../main/snapshots/register_scope_snapshot.rs | 10 +- .../main/snapshots/register_snapshot.rs | 15 +- .../discovery/main/sources/consumer_uses.rs | 19 +- .../invocation_allows_missing_secrets.rs | 5 +- .../sources/orchestration_declarations.rs | 22 +- .../discovery/main/sources/provider_uses.rs | 12 +- .../sources/register_declaration_files.rs | 13 +- .../main/sources/register_yaml_entries.rs | 12 +- .../main/sources/require_used_secrets.rs | 4 +- .../src/discovery/main/sources/sink_uses.rs | 16 +- .../src/discovery/models.rs | 2 +- .../macro_bridge/_helpers/first_compile.rs | 20 +- .../macro_bridge/_helpers/module_digests.rs | 16 +- .../src/macro_bridge/_helpers/store_keys.rs | 114 +---- .../macro_bridge/main/context_store_token.rs | 56 ++- .../src/macro_bridge/main/keep_delta_claim.rs | 6 +- .../macro_bridge/main/macro_store_token.rs | 43 +- .../main/register_first_compile.rs | 10 +- .../main/register_module_digests.rs | 14 +- .../macro_bridge/main/register_store_keys.rs | 12 +- .../src/macro_bridge/main/text.rs | 13 +- .../_helpers/config/config_classes.rs | 83 +-- .../_helpers/config/config_errors.rs | 52 +- .../_helpers/config/config_nodes.rs | 59 +-- .../_helpers/config/config_templates.rs | 71 +-- .../_helpers/config/config_values.rs | 31 +- .../_helpers/config/template_data.rs | 183 ++----- .../_helpers/config/typed_config.rs | 198 +------- .../_helpers/headers/header_audits.rs | 4 +- .../_helpers/headers/header_columns.rs | 45 +- .../_helpers/headers/header_families.rs | 24 - .../_helpers/headers/header_metadata.rs | 113 +---- .../src/model_config/_helpers/headers/mod.rs | 1 - .../_helpers/model_nodes/column_migrations.rs | 36 +- .../_helpers/model_nodes/contract_classes.rs | 23 +- .../_helpers/model_nodes/described_models.rs | 12 +- .../model_nodes/model_config_builder.rs | 35 +- .../_helpers/model_nodes/model_validator.rs | 10 +- .../model_nodes/unicode_environment.rs | 13 +- .../model_config/main/config/carried_node.rs | 16 +- .../main/config/config_cursor_bound.rs | 19 +- .../model_config/main/config/config_fields.rs | 60 ++- .../model_config/main/config/config_flag.rs | 13 +- .../main/config/config_integer_object.rs | 19 +- .../model_config/main/config/config_node.rs | 14 +- .../main/config/config_node_of.rs | 5 +- .../model_config/main/config/config_object.rs | 37 +- .../model_config/main/config/config_str.rs | 17 +- .../model_config/main/config/config_text.rs | 17 +- .../main/config/config_text_is.rs | 22 +- .../main/config/config_text_object.rs | 19 +- .../main/config/config_text_tuple.rs | 23 +- .../model_config/main/config/config_value.rs | 14 +- .../main/config/early_model_templates.rs | 29 +- .../src/model_config/main/config/expanded.rs | 55 +- .../main/config/expanded_effective_vars.rs | 45 +- .../main/config/expanded_template_data.rs | 17 +- .../model_config/main/config/model_values.rs | 11 +- .../main/config/native_config_error.rs | 28 +- .../main/config/raised_config_exception.rs | 11 +- .../model_config/main/config/record_reads.rs | 17 +- .../config/record_template_reads_function.rs | 5 +- .../main/config/register_config_errors.rs | 10 +- .../main/config/register_config_values.rs | 19 +- .../main/config/register_template_data.rs | 17 +- .../model_config/main/config/retain_values.rs | 25 +- .../src/model_config/main/config/run_id.rs | 30 +- .../main/config/template_error.rs | 5 +- .../model_config/main/config/typed_values.rs | 43 +- .../src/model_config/main/config/view.rs | 16 +- .../model_config/main/headers/audit_tuple.rs | 9 +- .../model_config/main/headers/family_specs.rs | 22 +- .../main/headers/native_column_record.rs | 21 +- .../main/headers/native_schema_column.rs | 43 +- .../main/headers/parsed_audit_identity.rs | 12 +- .../main/headers/register_header_columns.rs | 10 +- .../main/headers/schema_columns.rs | 30 +- .../model_config/main/headers/thresholds.rs | 20 +- .../model_nodes/column_migration_error.rs | 29 +- .../main/model_nodes/contract_classes.rs | 12 +- .../externally_described_models.rs | 4 +- .../main/model_nodes/raised_config_error.rs | 16 +- .../register_model_config_builder.rs | 10 +- .../model_nodes/register_model_validator.rs | 10 +- .../register_unicode_environment.rs | 9 +- .../main/model_nodes/unicode_environment.rs | 10 +- .../main/model_nodes/yaml_entry_classes.rs | 12 +- .../src/model_config/models.rs | 12 +- .../src/model_config/types.rs | 2 +- .../_helpers/audit_results_state.rs | 13 +- .../native_host/_helpers/consumption_state.rs | 131 +---- .../_helpers/observations_state.rs | 90 +--- .../native_host/main/append_observations.rs | 68 ++- .../src/native_host/main/execute.rs | 17 +- .../native_host/main/fixed_state_table_sql.rs | 37 +- .../src/native_host/main/integer_literal.rs | 14 +- .../src/native_host/main/literal.rs | 14 +- .../src/native_host/main/optional_text.rs | 11 +- .../native_host/main/recorder_callbacks.rs | 11 +- .../main/register_audit_results_state.rs | 10 +- .../main/register_consumption_state.rs | 15 +- .../main/register_observations_state.rs | 15 +- .../src/native_host/main/state_table_name.rs | 22 +- .../src/refactor/_helpers/contract_compare.rs | 14 +- .../src/refactor/_helpers/contract_edits.rs | 158 +----- .../src/refactor/_helpers/python_strings.rs | 9 +- .../src/refactor/_helpers/refactor_facts.rs | 104 +--- .../src/refactor/main/compare_contracts.rs | 6 +- .../refactor/main/declaration_placement.rs | 4 +- .../src/refactor/main/edit_contract_model.rs | 54 +- .../src/refactor/main/edit_contract_source.rs | 38 +- .../src/refactor/main/model_facts.rs | 36 +- .../src/refactor/main/refactor_facts.rs | 50 +- .../main/register_contract_compare.rs | 13 +- .../refactor/main/register_contract_edits.rs | 15 +- .../refactor/main/register_python_strings.rs | 11 +- .../main/remaining_contract_findings.rs | 51 +- .../src/refactor/main/unit_test_rebindings.rs | 17 +- .../render/_helpers/calls/macro_arguments.rs | 13 +- .../render/_helpers/calls/macro_call_rows.rs | 34 +- .../src/render/_helpers/calls/macro_calls.rs | 55 +- .../_helpers/declarations/authored_sql.rs | 81 +-- .../declarations/declaration_expansion.rs | 20 +- .../declarations/declaration_indexes.rs | 95 +--- .../declarations/declaration_values.rs | 126 +---- .../_helpers/expansion/macro_evaluation.rs | 3 +- .../_helpers/expansion/macro_expansion.rs | 16 +- .../expansion/macro_expansion_result.rs | 26 +- .../_helpers/expansion/macro_modules.rs | 16 +- .../_helpers/sql_macros/sql_macro_host.rs | 8 +- .../sql_macros/sql_macro_inventory.rs | 13 +- .../_helpers/sql_macros/sql_macro_tests.rs | 43 +- .../main/calls/register_macro_arguments.rs | 11 +- .../render/main/calls/register_macro_calls.rs | 17 +- .../src/render/main/calls/span_expansion.rs | 9 +- .../src/render/main/calls/span_row.rs | 9 +- .../src/render/main/calls/span_tuple.rs | 9 +- .../main/calls/top_level_macro_call_names.rs | 30 +- .../declarations/constant_declaration_sql.rs | 53 +- .../declaration_scope_resolver.rs | 48 +- .../main/declarations/expand_with_inputs.rs | 25 +- .../public_declaration_indexes.rs | 21 +- .../declarations/public_model_schema_index.rs | 53 +- .../declarations/register_authored_sql.rs | 15 +- .../register_declaration_indexes.rs | 12 +- .../register_declaration_values.rs | 15 +- .../declarations/render_enum_member_value.rs | 16 +- .../src/render/main/declarations/span_rows.rs | 13 +- .../src/render/main/expansion/expanded.rs | 4 +- .../render/main/expansion/expansion_result.rs | 4 +- .../main/expansion/macro_expansion_classes.rs | 12 +- .../expansion/register_macro_expansion.rs | 10 +- .../register_macro_expansion_result.rs | 10 +- .../main/expansion/register_macro_modules.rs | 13 +- .../register_sql_macro_inventory.rs | 10 +- .../main/sql_macros/with_sql_macro_mocks.rs | 4 +- .../sql_macros/without_with_list_calls.rs | 38 +- .../src/render/models.rs | 6 +- .../scopes/_helpers/checks/scope_tolerant.rs | 9 +- .../_helpers/checks/scope_validation.rs | 175 +------ .../_helpers/checks/scope_visibility.rs | 38 +- .../declarations/declaration_contexts.rs | 8 +- .../src/scopes/_helpers/index/scope_cache.rs | 88 +--- .../src/scopes/_helpers/index/scope_index.rs | 83 +-- .../src/scopes/_helpers/index/scope_lookup.rs | 128 +---- .../scopes/_helpers/index/scope_objects.rs | 13 +- .../_helpers/relationships/compile_usages.rs | 11 +- .../relationships/relationship_names.rs | 14 +- .../_helpers/reports/scope_report_objects.rs | 85 +--- .../scopes/_helpers/reports/scope_reports.rs | 269 ++-------- .../main/checks/register_scope_validation.rs | 15 +- .../main/checks/register_scope_visibility.rs | 20 +- .../main/checks/relocated_declarations.rs | 57 ++- .../checks/scope_index_with_compile_usages.rs | 60 ++- .../main/checks/scope_tolerant_index.rs | 6 +- .../main/checks/validate_scope_index.rs | 39 +- .../register_declaration_contexts.rs | 10 +- .../scopes/main/index/build_scope_lookup.rs | 106 +++- .../main/index/open_native_scope_index.rs | 41 +- .../scopes/main/index/project_fingerprint.rs | 24 +- .../main/index/read_scope_index_cache.rs | 25 +- .../scopes/main/index/register_scope_cache.rs | 11 +- .../scopes/main/index/register_scope_index.rs | 15 +- .../main/index/register_scope_lookup.rs | 15 +- .../main/index/write_scope_index_cache.rs | 26 +- .../relationships/register_compile_usages.rs | 11 +- .../register_relationship_names.rs | 13 +- .../src/scopes/main/reports/json_to_python.rs | 41 +- .../main/reports/metadata_projection_value.rs | 10 +- .../src/scopes/main/reports/read_result.rs | 37 +- .../main/reports/register_scope_reports.rs | 23 +- .../main/reports/report_scope_index_errors.rs | 4 +- .../src/scopes/main/reports/scope_browse.rs | 25 +- .../src/scopes/main/reports/scope_explain.rs | 47 +- .../src/scopes/main/reports/scope_list.rs | 37 +- .../scopes/main/reports/scope_move_preview.rs | 43 +- .../main/reports/scope_rebound_usages.rs | 19 +- .../src/scopes/main/reports/scope_report.rs | 48 +- .../main/reports/serialize_scope_report.rs | 26 +- .../sql_test_glue/_helpers/authored_ctes.rs | 66 +-- .../src/sql_test_glue/main/prefixed_ctes.rs | 19 +- .../main/register_authored_ctes.rs | 10 +- .../main/validate_authored_sql_test_ctes.rs | 38 +- .../src/sqltext/_helpers/authored_values.rs | 78 +-- .../src/sqltext/_helpers/code_scanner.rs | 9 - .../src/sqltext/_helpers/ddl_effects.rs | 20 +- .../_helpers/declaration_references.rs | 21 +- .../src/sqltext/_helpers/lexical_syntax.rs | 66 --- .../src/sqltext/_helpers/mod.rs | 1 - .../src/sqltext/_helpers/model_headers.rs | 100 +--- .../src/sqltext/_helpers/placeholders.rs | 11 +- .../src/sqltext/_helpers/sql_references.rs | 63 +-- .../src/sqltext/_helpers/static_sql.rs | 65 +-- .../sqltext/main/authored_value_to_python.rs | 50 +- .../src/sqltext/main/declaration_scan.rs | 15 +- .../src/sqltext/main/extract_references.rs | 46 +- .../src/sqltext/main/header_failure.rs | 25 +- .../src/sqltext/main/interpolated.rs | 19 +- .../src/sqltext/main/interpolation_batch.rs | 33 +- .../src/sqltext/main/location_rows.rs | 13 +- .../src/sqltext/main/map_to_python.rs | 15 +- .../src/sqltext/main/model_output_columns.rs | 15 +- .../main/optional_authored_value_to_python.rs | 12 +- .../sqltext/main/optional_record_syntax.rs | 12 +- .../src/sqltext/main/parse_model_headers.rs | 26 +- .../src/sqltext/main/record_syntax.rs | 19 +- .../src/sqltext/main/register_code_scanner.rs | 10 +- .../src/sqltext/main/register_ddl_effects.rs | 15 +- .../sqltext/main/register_model_headers.rs | 15 +- .../src/sqltext/main/register_placeholders.rs | 11 +- .../sqltext/main/register_sql_references.rs | 15 +- .../src/sqltext/main/sorted_texts.rs | 13 +- .../src/sqltext/main/syntax_record.rs | 32 +- .../state_rows/_helpers/event_rows/events.rs | 26 +- .../_helpers/event_rows/freshness.rs | 176 +------ .../src/state_rows/_helpers/event_rows/mod.rs | 1 - .../_helpers/event_rows/timestamps.rs | 35 -- .../_helpers/migrations/migration_io.rs | 151 +----- .../_helpers/migrations/migration_objects.rs | 49 +- .../migrations/migration_projections.rs | 139 +---- .../_helpers/migrations/migration_rows.rs | 23 +- .../_helpers/tables/state_tables.rs | 18 +- .../state_rows/main/event_rows/age_policy.rs | 44 +- .../main/event_rows/date_time_typed.rs | 10 +- .../state_rows/main/event_rows/equivalent.rs | 28 +- .../src/state_rows/main/event_rows/in_utc.rs | 13 +- .../state_rows/main/event_rows/normalize.rs | 67 ++- .../main/event_rows/offset_aware.rs | 9 +- .../main/event_rows/presumed_utc.rs | 14 +- .../main/event_rows/register_events.rs | 15 +- .../main/event_rows/register_freshness.rs | 17 +- .../main/event_rows/version_hash.rs | 9 +- .../src/state_rows/main/migrations/layout.rs | 45 +- .../main/migrations/newest_column.rs | 30 +- .../state_rows/main/migrations/newest_move.rs | 27 +- .../main/migrations/old_name_retention.rs | 18 +- .../main/migrations/old_name_views.rs | 48 +- .../state_rows/main/migrations/read_events.rs | 30 +- .../main/migrations/read_histories.rs | 34 +- .../main/migrations/register_migration_io.rs | 19 +- .../register_migration_projections.rs | 15 +- .../migrations/register_migration_rows.rs | 11 +- .../main/migrations/stored_columns_binding.rs | 15 +- .../migrations/stored_old_name_columns.rs | 28 +- .../main/tables/register_state_tables.rs | 15 +- .../main/tables/state_table_create_sql.rs | 4 +- .../src/state_rows/models.rs | 3 +- .../src/analysis/_helpers/binding_catalog.rs | 42 +- .../src/analysis/_helpers/catalog.rs | 3 +- .../src/analysis/_helpers/normalization.rs | 31 +- .../src/analysis/_helpers/relation_names.rs | 27 +- .../src/analysis/_helpers/schema_request.rs | 58 +-- .../main/extract_sql_relation_names.rs | 22 +- .../src/analysis/main/ignores_quoted_case.rs | 26 +- .../analysis/main/normalize_analysis_sql.rs | 16 +- .../analysis/main/register_binding_catalog.rs | 13 +- .../analysis/main/register_normalization.rs | 17 +- .../main/schema_validation_request.rs | 53 +- .../src/analysis/main/schema_validations.rs | 5 +- .../_helpers/fixture_columns.rs | 11 +- .../_helpers/profile_inputs.rs | 99 +--- .../_helpers/query_columns.rs | 31 +- .../src/analysis_session/_helpers/session.rs | 113 +---- .../_helpers/stage_functions.rs | 125 +---- .../main/case_sensitive_shapes.rs | 17 +- .../main/dialect_or_generic.rs | 12 +- .../main/expression_source_shapes.rs | 4 +- .../infer_native_expression_source_shapes.rs | 16 +- .../main/infer_native_query_columns.rs | 66 ++- .../main/nullability_rules.rs | 40 +- .../analysis_session/main/profile_dialect.rs | 10 +- .../main/prove_dynamic_column_contracts.rs | 39 +- .../main/query_column_rows.rs | 4 +- .../main/query_columns_error.rs | 12 +- .../main/register_fixture_columns.rs | 12 +- .../main/register_query_columns.rs | 11 +- .../analysis_session/main/register_session.rs | 10 +- .../main/register_stage_functions.rs | 20 +- .../src/analysis_session/main/return_types.rs | 10 +- .../src/analysis_session/main/shape_rows.rs | 15 +- .../main/start_model_analysis_session.rs | 62 ++- .../src/analysis_session/main/text_pairs.rs | 14 +- .../src/auditing/_helpers/render.rs | 72 +-- .../src/auditing/main/register_render.rs | 11 +- .../src/auditing/main/render_audit_sql.rs | 55 +- .../_helpers/cursor_roles.rs | 72 +-- .../main/resolve_cursor_input_roles.rs | 59 ++- .../resolve_cursor_input_roles_function.rs | 5 +- .../_helpers/attachment_inputs.rs | 7 +- .../compiled_project/_helpers/job_catalog.rs | 80 +-- .../src/compiled_project/_helpers/mod.rs | 1 - .../src/compiled_project/_helpers/project.rs | 122 +---- .../src/compiled_project/_helpers/rows.rs | 20 - .../compiled_project/_helpers/schema_facts.rs | 135 +---- .../compiled_project/main/config_values.rs | 11 +- .../src/compiled_project/main/declarations.rs | 5 +- .../main/entry_schema_fact.rs | 46 +- .../src/compiled_project/main/family_fact.rs | 13 +- .../compiled_project/main/graph_with_jobs.rs | 12 +- .../src/compiled_project/main/install.rs | 5 +- .../src/compiled_project/main/job_names.rs | 10 +- .../compiled_project/main/materialization.rs | 15 +- .../src/compiled_project/main/model_facts.rs | 61 ++- .../src/compiled_project/main/needed_jobs.rs | 15 +- .../src/compiled_project/main/plain_value.rs | 4 +- .../main/python_column_fact.rs | 47 +- .../main/python_column_facts.rs | 12 +- .../main/reference_row_tuple.rs | 15 +- .../compiled_project/main/register_project.rs | 29 +- .../src/compiled_project/main/text_value.rs | 12 +- .../compiled_project/main/triggered_jobs.rs | 32 +- .../compiled_models/compiled_model_view.rs | 99 +--- .../_helpers/compiled_models/model_reads.rs | 288 +---------- .../_helpers/compiled_models/relation_keys.rs | 189 +------ .../_helpers/model_inputs/model_input_view.rs | 146 +----- .../_helpers/project/getters.rs | 12 +- .../_helpers/project/project_host.rs | 3 +- .../_helpers/project/project_reads.rs | 86 +--- .../_helpers/project/records.rs | 354 +------------ .../main/compiled_models/audit_key_rows.rs | 25 +- .../main/compiled_models/binding_validated.rs | 13 +- .../main/compiled_models/config.rs | 14 +- .../main/compiled_models/declared_rows.rs | 13 +- .../compiled_models/defaulted_destination.rs | 26 +- .../main/compiled_models/dep_rows.rs | 13 +- .../main/compiled_models/deps.rs | 14 +- .../main/compiled_models/destination.rs | 13 +- .../dynamic_columns_declared.rs | 13 +- .../compiled_models/entry_relation_key_row.rs | 24 +- .../main/compiled_models/enum_columns.rs | 14 +- .../main/compiled_models/inferred_columns.rs | 14 +- .../main/compiled_models/inferred_names.rs | 10 +- .../main/compiled_models/inferred_rows.rs | 23 +- .../main/compiled_models/lineage_facts.rs | 51 +- .../main/compiled_models/lineage_inputs.rs | 30 +- .../main/compiled_models/model_graph_row.rs | 34 +- .../main/compiled_models/model_key.rs | 13 +- .../main/compiled_models/model_key_row.rs | 21 +- .../main/compiled_models/model_name.rs | 13 +- .../main/compiled_models/name_object.rs | 14 +- .../main/compiled_models/object_key.rs | 12 +- .../compiled_models/one_row_seed_names.rs | 32 +- .../main/compiled_models/output_proven.rs | 14 +- .../compiled_models/project_relation_keys.rs | 40 +- .../main/compiled_models/proven_inputs.rs | 17 +- .../main/compiled_models/query_sql.rs | 13 +- .../main/compiled_models/references.rs | 14 +- .../compiled_models/register_relation_keys.rs | 12 +- .../main/compiled_models/relation_keys.rs | 43 +- .../main/compiled_models/relation_location.rs | 20 +- .../main/compiled_models/relative_path.rs | 14 +- .../main/compiled_models/schema_entry.rs | 14 +- .../main/compiled_models/source_key_input.rs | 13 +- .../main/compiled_models/star_flags.rs | 19 +- .../main/compiled_models/unchecked_columns.rs | 16 +- .../main/model_inputs/attached_columns.rs | 24 +- .../main/model_inputs/attached_schema.rs | 32 +- .../main/model_inputs/input_analysis_sql.rs | 16 +- .../main/model_inputs/input_model_name.rs | 18 +- .../main/model_inputs/input_reference_rows.rs | 20 +- .../model_inputs/input_sql_macro_checks.rs | 14 +- .../main/model_inputs/reference_tuple.rs | 29 +- .../main/model_inputs/view_classes.rs | 8 +- .../src/compiled_views/main/project/audits.rs | 15 +- .../compiled_views/main/project/collection.rs | 26 +- .../main/project/compile_inputs_view.rs | 34 +- .../main/project/compiled_audit_record.rs | 12 +- .../main/project/compiled_function_record.rs | 12 +- .../main/project/compiled_model_record.rs | 12 +- .../main/project/compiled_models.rs | 15 +- .../main/project/compiled_project_facts.rs | 20 +- .../main/project/compiled_seed_record.rs | 12 +- .../main/project/compiled_source_record.rs | 12 +- .../main/project/declaration_rows.rs | 24 +- .../main/project/declaration_rows_function.rs | 5 +- .../compiled_views/main/project/functions.rs | 15 +- .../main/project/inputs_with.rs | 13 +- .../main/project/plain_typed_value.rs | 36 +- .../main/project/project_with.rs | 13 +- .../main/project/record_model_analyses.rs | 73 ++- .../main/project/register_records.rs | 43 +- .../compiled_views/main/project/replace.rs | 16 +- .../src/compiled_views/main/project/seeds.rs | 15 +- .../compiled_views/main/project/sources.rs | 15 +- .../compiled_views/main/project/sql_tests.rs | 14 +- .../main/project/with_binding_diagnostics.rs | 18 +- .../main/project/with_type_recovery.rs | 55 +- .../src/compiled_views/models.rs | 10 +- .../_helpers/contract_diagnostics.rs | 79 +-- .../src/contracts/_helpers/exception_types.rs | 8 - .../main/evaluate_model_contracts.rs | 5 +- .../main/promotion_conflict_diagnostics.rs | 63 ++- .../main/register_contract_diagnostics.rs | 13 +- .../main/register_exception_types.rs | 12 +- .../src/fingerprints/_helpers/latest_rows.rs | 99 +--- .../src/fingerprints/_helpers/query_hashes.rs | 30 +- .../src/fingerprints/_helpers/storage_io.rs | 33 +- .../fingerprints/main/compute_query_hash.rs | 14 +- .../main/read_latest_fingerprints.rs | 4 +- .../fingerprints/main/register_latest_rows.rs | 11 +- .../main/register_query_hashes.rs | 15 +- .../fingerprints/main/register_storage_io.rs | 19 +- .../src/fingerprints/main/row_fingerprint.rs | 41 +- .../main/select_latest_fingerprints.rs | 38 +- .../src/graph/_helpers/nodes/algorithms.rs | 90 +--- .../graph/_helpers/nodes/execution_gates.rs | 72 +-- .../src/graph/_helpers/nodes/keys.rs | 12 - .../src/graph/_helpers/nodes/mod.rs | 1 - .../graph/_helpers/project/project_graph.rs | 36 -- .../src/graph/_helpers/project/views.rs | 118 +---- .../_helpers/python_nodes/python_lifecycle.rs | 71 +-- .../python_nodes/python_node_graph.rs | 96 +--- .../_helpers/python_nodes/python_selection.rs | 57 +-- .../src/graph/main/nodes/closure.rs | 27 +- .../src/graph/main/nodes/inverted.rs | 31 +- .../main/nodes/model_python_hook_names.rs | 41 +- .../src/graph/main/nodes/native_key.rs | 12 +- .../graph/main/nodes/register_algorithms.rs | 17 +- .../main/nodes/register_execution_gates.rs | 13 +- .../src/graph/main/nodes/sql_ref_key.rs | 27 +- .../main/nodes/transitive_closure_many.rs | 16 +- .../graph/main/project/compiled_graph_keys.rs | 12 +- .../src/graph/main/project/edge_dict.rs | 14 +- .../main/project/graph_from_view_mappings.rs | 34 +- .../main/project/native_graph_from_edges.rs | 20 +- .../src/graph/main/project/project_graph.rs | 25 +- .../main/project/project_lineage_views.rs | 16 +- .../main/project/register_project_graph.rs | 11 +- .../src/graph/main/project/register_views.rs | 23 +- .../build_python_sql_run_lifecycle.rs | 63 ++- .../main/python_nodes/python_node_graph.rs | 49 +- .../main/python_nodes/python_node_names.rs | 18 +- .../python_nodes/register_python_lifecycle.rs | 10 +- .../register_python_node_graph.rs | 11 +- .../python_nodes/register_python_selection.rs | 16 +- .../resolve_python_check_excludes.rs | 21 +- .../python_nodes/run_selection_from_inputs.rs | 30 +- .../src/graph/models.rs | 4 +- .../src/lineage/_helpers/lineage_facts.rs | 112 +---- .../src/lineage/main/confidence_member.rs | 12 +- .../src/lineage/main/facts_eq.rs | 27 +- .../src/lineage/main/facts_getitem.rs | 29 +- .../src/lineage/main/facts_iter.rs | 16 +- .../src/lineage/main/facts_len.rs | 16 +- .../src/lineage/main/facts_projected.rs | 16 +- .../src/lineage/main/facts_resource_name.rs | 15 +- .../src/lineage/main/transform_kind_member.rs | 12 +- .../_helpers/orchestration_triggers.rs | 154 +----- .../src/native_host/main/compile_calendars.rs | 79 ++- .../src/native_host/main/compile_sensors.rs | 70 ++- .../src/planner_identity/_helpers/graph.rs | 176 +------ .../src/planner_identity/_helpers/hashing.rs | 72 +-- .../baselined_function_definition_hash.rs | 21 +- .../main/baselined_model_query_hash.rs | 15 +- .../planner_identity/main/execution_deps.rs | 53 +- .../planner_identity/main/execution_order.rs | 27 +- .../main/execution_origins.rs | 75 ++- .../main/function_definition_hash.rs | 11 +- .../src/planner_identity/main/graph_key.rs | 12 +- .../main/model_definition_hash.rs | 11 +- .../planner_identity/main/register_graph.rs | 25 +- .../planner_identity/main/register_hashing.rs | 10 +- .../src/planner_phases/_helpers/mod.rs | 1 - .../src/planner_phases/_helpers/objects.rs | 187 ------- .../src/planner_phases/main/by_name.rs | 18 +- .../src/planner_phases/main/config_str.rs | 10 +- .../src/planner_phases/main/construct.rs | 11 +- .../src/planner_phases/main/frozenset.rs | 10 +- .../src/planner_phases/main/member.rs | 10 +- .../src/planner_phases/main/model_key.rs | 22 +- .../src/planner_phases/main/native.rs | 10 +- .../src/planner_phases/main/native_member.rs | 10 +- .../src/planner_phases/main/object.rs | 12 +- .../src/planner_phases/main/optional_text.rs | 9 +- .../src/planner_phases/main/planner_error.rs | 24 +- .../src/planner_phases/main/planner_model.rs | 10 +- .../src/planner_phases/main/replace.rs | 10 +- .../src/planner_phases/main/sorted_names.rs | 14 +- .../src/planner_phases/main/strings.rs | 11 +- .../src/planner_phases/main/text.rs | 9 +- .../src/planner_phases/main/tuple.rs | 10 +- .../src/planner_phases/mod.rs | 1 - .../src/planner_scope/_helpers/loaders.rs | 60 +-- .../src/planner_scope/_helpers/retained.rs | 42 +- .../src/planner_scope/_helpers/scope.rs | 4 +- .../src/planner_scope/_helpers/sources.rs | 53 +- .../src/planner_scope/_helpers/values.rs | 34 -- .../main/build_intermediate_source_map.rs | 12 +- .../build_intermediate_source_map_function.rs | 5 +- .../planner_scope/main/build_planner_scope.rs | 5 +- .../main/build_source_load_entries.rs | 7 +- .../build_source_load_entries_function.rs | 5 +- .../main/build_source_load_map_function.rs | 5 +- .../build_upstream_intermediate_source_map.rs | 10 +- .../src/planner_scope/main/edges.rs | 17 +- ...d_selected_loader_dependencies_function.rs | 5 +- .../src/planner_scope/main/keys.rs | 12 +- .../src/planner_scope/main/project_graph.rs | 20 +- .../src/planner_scope/main/project_scope.rs | 42 +- .../src/planner_scope/main/scope_data.rs | 4 +- .../src/planner_scope/main/source_load_map.rs | 34 +- .../main/upstream_loader_dependency_names.rs | 14 +- ...stream_loader_dependency_names_function.rs | 5 +- .../src/planner_selection/_helpers/carrier.rs | 9 - .../_helpers/model_selectors.rs | 88 +--- .../planner_selection/_helpers/selectors.rs | 149 +----- .../planner_selection/_helpers/unit_tests.rs | 32 +- .../main/expand_project_upstream_keys.rs | 12 +- .../main/match_project_selector.rs | 38 +- .../main/parse_project_selector.rs | 15 +- .../src/planner_selection/main/py_strings.rs | 11 +- .../main/register_model_selectors.rs | 11 +- .../main/reject_unit_test_selectors.rs | 9 +- .../main/resolve_graph_keys.rs | 22 +- .../main/resolve_graph_selection.rs | 10 +- .../main/resolve_project_selector_tokens.rs | 32 +- .../main/resolve_project_selectors.rs | 26 +- .../resolve_sqlbuild_model_selector_names.rs | 41 +- .../main/select_project_graph.rs | 14 +- .../main/split_project_unit_test_selectors.rs | 14 +- .../main/split_selector_expansion.rs | 39 +- .../main/split_unit_test_selectors.rs | 9 +- .../_helpers/fixtures/fixture_completion.rs | 44 +- .../_helpers/fixtures/fixture_context.rs | 39 +- .../_helpers/fixtures/fixture_metadata.rs | 3 +- .../_helpers/fixtures/fixture_preparation.rs | 5 +- .../_helpers/fixtures/fixture_reads.rs | 10 +- .../_helpers/fixtures/fixture_selection.rs | 4 +- .../_helpers/fixtures/fixture_sql.rs | 63 +-- .../_helpers/fixtures/fixture_types.rs | 6 +- .../main/complete_empty_fixture_sql.rs | 31 +- .../main/complete_fixtures.rs | 18 +- .../main/complete_typed_null_columns.rs | 33 +- .../main/count_chain_resolution.rs | 9 +- .../planner_strategy/main/planning_context.rs | 4 +- .../main/register_fixture_completion.rs | 10 +- .../main/register_fixture_context.rs | 14 +- .../main/register_fixture_reads.rs | 10 +- .../src/planner_strategy/models.rs | 15 +- .../_helpers/snapshot/snapshot.rs | 10 +- .../_helpers/snapshot/snapshot_boundary.rs | 9 +- .../snapshot/snapshot_declarations.rs | 21 - .../_helpers/snapshot/snapshot_models.rs | 6 +- .../_helpers/snapshot/snapshot_resources.rs | 14 +- .../_helpers/snapshot/snapshot_roles.rs | 39 -- .../main/compile_declarations.rs | 5 +- .../main/compile_project_roles.rs | 33 +- .../src/project_snapshot/main/model_nodes.rs | 6 +- .../main/python_declarations.rs | 22 +- .../main/register_snapshot.rs | 11 +- .../project_snapshot/main/relative_path.rs | 10 +- .../project_snapshot/main/resource_nodes.rs | 3 +- .../_helpers/explicit_references.rs | 77 +-- .../src/references/_helpers/relations.rs | 51 +- .../references/_helpers/source_relations.rs | 85 +--- .../main/explicit_reference_report.rs | 5 +- .../main/hard_coded_relation_remedy.rs | 25 +- .../references/main/match_project_relation.rs | 35 +- .../main/native_execution_gate_edges.rs | 23 +- .../main/old_name_reference_message.rs | 8 +- .../main/register_explicit_references.rs | 15 +- .../src/references/main/register_relations.rs | 15 +- .../main/register_source_relations.rs | 11 +- .../src/references/main/render_source_read.rs | 23 +- .../references/main/render_source_relation.rs | 51 +- .../src/relation_naming/_helpers/fitting.rs | 34 +- .../main/fitted_logical_name.rs | 10 +- .../relation_naming/main/register_fitting.rs | 13 +- .../src/relation_naming/main/shared_error.rs | 9 +- .../resource_views/_helpers/audit_reads.rs | 222 +------- .../resource_views/_helpers/audit_views.rs | 34 +- .../resource_views/_helpers/function_reads.rs | 167 +----- .../resource_views/_helpers/function_views.rs | 15 - .../src/resource_views/_helpers/mod.rs | 1 - .../resource_views/_helpers/resource_reads.rs | 151 ------ .../src/resource_views/_helpers/seed_reads.rs | 68 +-- .../src/resource_views/_helpers/seed_views.rs | 68 +-- .../resource_views/_helpers/source_reads.rs | 48 +- .../resource_views/_helpers/source_views.rs | 28 +- .../main/audit_reads/always_run.rs | 17 +- .../main/audit_reads/attached_column_name.rs | 17 +- .../main/audit_reads/attached_target_kind.rs | 17 +- .../main/audit_reads/attached_target_name.rs | 17 +- .../main/audit_reads/definition_name.rs | 17 +- .../main/audit_reads/description.rs | 17 +- .../main/audit_reads/evaluation_mode.rs | 17 +- .../main/audit_reads/evidence_limit.rs | 17 +- .../main/audit_reads/evidence_sql.rs | 17 +- .../resource_views/main/audit_reads/key.rs | 15 +- .../main/audit_reads/measurement_contract.rs | 17 +- .../resource_views/main/audit_reads/meta.rs | 15 +- .../main/audit_reads/minimum_samples.rs | 17 +- .../resource_views/main/audit_reads/name.rs | 15 +- .../main/audit_reads/references.rs | 17 +- .../main/audit_reads/run_scope.rs | 17 +- .../main/audit_reads/scope_deps.rs | 17 +- .../main/audit_reads/severity.rs | 17 +- .../main/audit_reads/sql_body.rs | 17 +- .../main/audit_reads/thresholds.rs | 17 +- .../main/audit_views/audit_record.rs | 13 +- .../main/audit_views/audit_reference_rows.rs | 23 +- .../main/function_reads/arguments.rs | 17 +- .../main/function_reads/body_sql.rs | 17 +- .../main/function_reads/deps.rs | 15 +- .../main/function_reads/destination.rs | 17 +- .../main/function_reads/entry_point.rs | 17 +- .../function_reads/fingerprint_destination.rs | 17 +- .../resource_views/main/function_reads/key.rs | 15 +- .../main/function_reads/language.rs | 17 +- .../main/function_reads/name.rs | 15 +- .../main/function_reads/packages.rs | 17 +- .../main/function_reads/relative_path.rs | 17 +- .../main/function_reads/return_columns.rs | 17 +- .../main/function_reads/returns.rs | 17 +- .../main/function_reads/runtime_version.rs | 17 +- .../main/function_reads/source_file_path.rs | 17 +- .../main/function_views/function_record.rs | 15 +- .../main/resource_reads/destination.rs | 35 +- .../main/resource_reads/graph_row.rs | 39 +- .../resource_views/main/resource_reads/key.rs | 34 +- .../main/resource_reads/name.rs | 34 +- .../main/resource_reads/references.rs | 28 +- .../main/resource_reads/schema_entry.rs | 28 +- .../main/seed_reads/destination.rs | 17 +- .../main/seed_reads/external.rs | 15 +- .../src/resource_views/main/seed_reads/key.rs | 15 +- .../resource_views/main/seed_reads/name.rs | 15 +- .../main/seed_reads/schema_entry.rs | 17 +- .../main/seed_reads/seed_file.rs | 17 +- .../main/seed_views/paired_seeds.rs | 62 ++- .../resource_views/main/source_reads/key.rs | 15 +- .../resource_views/main/source_reads/name.rs | 15 +- .../main/source_reads/source_entry.rs | 17 +- .../main/source_reads/source_file.rs | 17 +- .../main/source_views/read_source_view.rs | 5 +- .../main/source_views/replaced.rs | 11 +- .../main/source_views/retain_sources.rs | 14 +- .../src/semantic_shapes/_helpers/coverage.rs | 8 +- .../semantic_shapes/_helpers/declarations.rs | 48 +- .../src/semantic_shapes/_helpers/shapes.rs | 48 +- .../_helpers/source_bindings.rs | 19 +- .../semantic_shapes/main/declared_types.rs | 32 +- .../semantic_shapes/main/function_names.rs | 17 +- .../semantic_shapes/main/project_shapes.rs | 4 +- .../semantic_shapes/main/register_coverage.rs | 10 +- .../semantic_shapes/main/register_shapes.rs | 14 +- .../main/register_source_bindings.rs | 10 +- .../semantic_shapes/main/semantic_coverage.rs | 4 +- .../main/snapshot_validity_columns.rs | 32 +- .../main/source_binding_diagnostics.rs | 6 +- .../_helpers/inputs/cursor_window.rs | 14 +- .../_helpers/plan/plan_assembly.rs | 10 +- .../_helpers/plan/plan_driver.rs | 137 +---- .../_helpers/plan/plan_entries.rs | 3 +- .../_helpers/plan/plan_rendering.rs | 9 +- .../main/assemble_sql_test_plans.rs | 4 +- ...an_compiled_sql_test_artifacts_function.rs | 5 +- .../main/plan_sql_tests_function.rs | 5 +- .../main/plan_sql_tests_natively_function.rs | 5 +- .../src/sql_test_glue/main/planned_rows.rs | 84 +++- .../main/sql_test_model_chain_names_by_key.rs | 20 +- ..._test_model_chain_names_by_key_function.rs | 5 +- .../sql_test_plan_error_messages_function.rs | 5 +- .../src/sql_test_glue/models.rs | 3 +- .../src/sql_test_glue/types.rs | 3 +- .../src/type_system/_helpers/nullability.rs | 24 +- .../type_system/main/register_nullability.rs | 18 +- .../analysis_session/model_analyses.rs | 14 +- .../compile_driver/assembly_analysis.rs | 3 +- .../_helpers/compile_driver/assembly_facts.rs | 6 +- .../_helpers/compile_driver/discovery.rs | 6 +- .../compile_driver/pipeline_entries.rs | 3 +- .../bindings/_helpers/consumption/recorder.rs | 6 +- .../_helpers/exported_constants/constants.rs | 8 +- .../_helpers/input_loops/audit_loop.rs | 9 +- .../_helpers/input_loops/function_loop.rs | 3 +- .../_helpers/input_loops/model_loop.rs | 12 +- .../_helpers/input_loops/model_loop_config.rs | 6 +- .../input_loops/model_loop_contracts.rs | 3 +- .../_helpers/input_loops/model_loop_header.rs | 4 +- .../_helpers/input_loops/model_loop_hooks.rs | 6 +- .../_helpers/input_loops/model_loop_host.rs | 15 +- .../_helpers/input_loops/model_loop_macros.rs | 4 +- .../input_loops/resource_attachment.rs | 18 +- .../input_loops/resource_diagnostics.rs | 10 +- .../_helpers/input_loops/source_loop.rs | 13 +- .../_helpers/input_loops/sql_test_loop.rs | 6 +- .../_helpers/observations/publication.rs | 3 +- .../_helpers/planner_entries/model_entries.rs | 3 +- .../planner_migrations/state_inspection.rs | 4 +- .../_helpers/planner_scenario/execution.rs | 7 +- .../planner_strategy/audit_entries.rs | 7 +- .../planner_strategy/cursor_inputs.rs | 10 +- .../planner_strategy/model_entries.rs | 11 +- .../project_assembly/declared_relations.rs | 3 +- .../src/bindings/_helpers/rules/sql_rules.rs | 6 +- .../bindings/_helpers/scopes/scope_command.rs | 4 +- fensu.toml | 475 +++++++++++++++--- tests/native_field_fallbacks.txt | 5 +- 928 files changed, 11656 insertions(+), 16273 deletions(-) delete mode 100644 crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs delete mode 100644 crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs delete mode 100644 crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs delete mode 100644 crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs delete mode 100644 crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs delete mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs delete mode 100644 crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs delete mode 100644 crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs delete mode 100644 crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs index 4b5621ff57..caca7e0466 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs @@ -1,59 +1,14 @@ //! Python audit argument values read as plain argument values. -use crate::model_config::main::config::config_object::config_object; -use crate::model_config::types::PyConfigNode; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, - PyStringMethods, PyTuple, PyTupleMethods, + PyBool, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, }; use sqlbuild_attachments::audits::models::ArgumentValue; -use sqlbuild_model_config::config_values::models::ConfigNode; - -/// Native-built argument containers need no intermediate Python list or tuple. -pub(crate) fn config_argument_value( - py: Python<'_>, - value: &PyConfigNode, -) -> PyResult { - Ok(match value { - ConfigNode::Null => ArgumentValue::Null, - ConfigNode::Bool(value) => ArgumentValue::Boolean(*value), - ConfigNode::Int(value) => ArgumentValue::Number(value.to_string()), - ConfigNode::Str(value) => ArgumentValue::Text(value.clone()), - ConfigNode::List(items) => ArgumentValue::List( - items - .iter() - .map(|item| config_argument_value(py, item)) - .collect::>()?, - ), - ConfigNode::Tuple(items) => ArgumentValue::Tuple( - items - .iter() - .map(|item| config_argument_value(py, item)) - .collect::>()?, - ), - ConfigNode::Map(_) => ArgumentValue::Opaque, - ConfigNode::Float(_) | ConfigNode::Host(_) => { - return argument_value(&config_object(py, value)?); - } - }) -} - -/// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -pub(crate) fn argument_pairs( - arguments: &Bound<'_, PyDict>, -) -> PyResult> { - let mut pairs: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.len()); - for (key, value) in arguments.iter() { - if let Ok(key) = key.downcast::() { - pairs.push((key.to_string_lossy().into_owned(), argument_value(&value)?)); - } - } - Ok(pairs) -} /// One plain value; text with lone surrogates, rejected where it enters a compile, reads lossily. -fn argument_value(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn argument_value(value: &Bound<'_, PyAny>) -> PyResult { if value.is_none() { return Ok(ArgumentValue::Null); } diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs index 9e154442e5..f9756ebd37 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs @@ -1,7 +1,7 @@ //! Where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::PyResult; +use pyo3::pyfunction; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -11,7 +11,10 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The code-point offset to insert `SELECT 1` at, or `None`. #[pyfunction] -fn omitted_ceremonial_select(sql: &str, syntax: LexicalSyntaxInput) -> PyResult> { +pub(crate) fn omitted_ceremonial_select( + sql: &str, + syntax: LexicalSyntaxInput, +) -> PyResult> { let syntax: LexicalSyntax = syntax.into(); compiler_guard(|| { Ok(match omitted_select_offset(sql, &syntax) { @@ -20,8 +23,3 @@ fn omitted_ceremonial_select(sql: &str, syntax: LexicalSyntaxInput) -> PyResult< }) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(omitted_ceremonial_select, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs index a6222f0c50..c26925d74b 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs @@ -2,49 +2,31 @@ use crate::attachments::constants::MICROBATCH_END_SENTINEL; use crate::attachments::constants::MICROBATCH_START_SENTINEL; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use crate::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use crate::attachments::main::compile_input_error::compile_input_error; +use crate::attachments::main::validated_model_cursor_sql::validated_model_cursor_sql; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{Python, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_macros::models::TextSplice; +use pyo3::{Python, pyfunction}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; -use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; -use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; -use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::ErrorId; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; -const CURSOR_START_CALL: &str = "__cursor_start()"; -const CURSOR_END_CALL: &str = "__cursor_end()"; +pub(crate) const CURSOR_START_CALL: &str = "__cursor_start()"; +pub(crate) const CURSOR_END_CALL: &str = "__cursor_end()"; const INTEGER_CURSOR_TYPE: &str = "integer"; const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; -const MODEL_SQL_CONTEXT: &str = "Model SQL"; -const SQL_CONTEXT: &str = "SQL"; +pub(crate) const MODEL_SQL_CONTEXT: &str = "Model SQL"; +pub(crate) const SQL_CONTEXT: &str = "SQL"; -const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; - -/// The reserved executor bound markers no authored SQL may contain. -pub(crate) fn reserved_cursor_markers() -> Vec { - RESERVED_CURSOR_MARKERS - .iter() - .map(|marker| (*marker).to_owned()) - .collect() -} - -/// A `CompileInputError` carrying `message`. -pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyResult { - Ok(PyErr::from_value( - error_class(py, ErrorId::CompileInput)?.call1((message,))?, - )) -} +pub(crate) const RESERVED_CURSOR_MARKERS: [&str; 2] = + [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; /// `text` when it is a string, else `None`. -fn text_value(value: &Bound<'_, PyAny>) -> Option { +pub(crate) fn text_value(value: &Bound<'_, PyAny>) -> Option { match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, @@ -70,60 +52,8 @@ pub(crate) fn replaced( outcome.or_else(|message| Err(compile_input_error(py, message)?)) } -/// One model query with canonical intrinsic calls, raising when the model may not use them. -pub(crate) fn validated_model_cursor_sql( - py: Python<'_>, - sql: &str, - (model_name, config_values): (&str, &Bound<'_, PyAny>), -) -> PyResult { - let incremental: bool = text_value(&mapping_get(config_values, "materialized")?) - .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); - let cursor: Option = text_value(&mapping_get(config_values, "cursor")?); - let reserved_markers: Vec = reserved_cursor_markers(); - let outcome: Result = compiler_guard(|| { - Ok(validated_model_intrinsics( - sql, - &reserved_markers, - &IntrinsicModel { - name: model_name, - incremental, - cursor: cursor.as_deref(), - }, - )) - })?; - outcome.or_else(|message| Err(compile_input_error(py, message)?)) -} - -/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -pub(crate) fn analysis_cursor_sql( - py: Python<'_>, - sql: &str, - cursor_type: &Bound<'_, PyAny>, -) -> PyResult { - let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); - Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) -} - -/// `analysis_cursor_sql` with each replaced call as a splice of `sql`. -pub(crate) fn analysis_cursor_splices( - sql: &str, - cursor_type: Option<&str>, -) -> Result<(String, Vec), String> { - let literal: &str = analysis_bound(cursor_type); - let (replaced, spans) = - replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; - let splices: Vec = spans - .iter() - .map(|span| TextSplice { - source: (span.source_start, span.source_end), - output: (span.output_start, span.output_end), - }) - .collect(); - Ok((replaced, splices)) -} - /// The literal static analysis reads for every cursor intrinsic of a model with `cursor_type`. -fn analysis_bound(cursor_type: Option<&str>) -> &'static str { +pub(crate) fn analysis_bound(cursor_type: Option<&str>) -> &'static str { if cursor_type == Some(INTEGER_CURSOR_TYPE) { INTEGER_ANALYSIS_BOUND } else { @@ -134,7 +64,7 @@ fn analysis_bound(cursor_type: Option<&str>) -> &'static str { /// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. #[pyfunction] #[pyo3(signature = (*, sql, config_values, model_name))] -fn validated_model_cursor_intrinsics( +pub(crate) fn validated_model_cursor_intrinsics( py: Python<'_>, sql: &str, config_values: &Bound<'_, PyAny>, @@ -146,41 +76,10 @@ fn validated_model_cursor_intrinsics( /// `sql` with intrinsics replaced by typed literals of `cursor_type` for static analysis. #[pyfunction] #[pyo3(signature = (*, sql, cursor_type))] -fn cursor_intrinsics_analysis_sql( +pub(crate) fn cursor_intrinsics_analysis_sql( py: Python<'_>, sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { analysis_cursor_sql(py, sql, cursor_type) } - -/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -#[pyfunction] -#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] -pub(crate) fn resolve_cursor_intrinsics( - py: Python<'_>, - sql: &str, - start_sql: Option<&str>, - end_sql: Option<&str>, -) -> PyResult<(String, bool)> { - let (rendered, found) = replaced( - py, - sql, - SQL_CONTEXT, - ( - start_sql.unwrap_or(CURSOR_START_CALL), - end_sql.unwrap_or(CURSOR_END_CALL), - ), - )?; - if !found || start_sql.is_none() || end_sql.is_none() { - return Ok((sql.to_owned(), found)); - } - Ok((rendered, true)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; - module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; - module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs index 98c8087ea6..2eeedb15d6 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs @@ -5,30 +5,10 @@ use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; -use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; - -/// One function file's parsed header; non-string keys and `None` values are never read. -pub(crate) fn parsed_header( - header_values: &Bound<'_, PyDict>, - language: FunctionLanguage, - relative_path: &str, -) -> FunctionHeader { - let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); - for (key, value) in header_values.iter() { - let Some(name) = text_value(&key) else { - continue; - }; - if value.is_none() { - continue; - } - header.push((name, header_value(&value))); - } - parse_function_header(&header, language, relative_path) -} +use sqlbuild_attachments::functions::models::HeaderValue; /// One header value. -fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { +pub(crate) fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { if let Some(text) = text_value(value) { return HeaderValue::Text(text); } @@ -50,7 +30,7 @@ fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { } /// A `str` (or subclass) as text; lone surrogates, rejected at compile entry, read as U+FFFD. -fn text_value(value: &Bound<'_, PyAny>) -> Option { +pub(crate) fn text_value(value: &Bound<'_, PyAny>) -> Option { match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs index 8f151e1d20..029661b7d9 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs @@ -1,18 +1,10 @@ //! Reject executable SQL that still contains an unresolved reference call. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; -use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; /// The reference function and diagnostic code of each marker kind, in the scanner's order. -const MARKER_KINDS: [(&str, &str); 5] = [ +pub(crate) const MARKER_KINDS: [(&str, &str); 5] = [ ("__ref", "R001"), ("__seed", "R002"), ("__source", "R003"), @@ -21,48 +13,10 @@ const MARKER_KINDS: [(&str, &str); 5] = [ ]; /// A `ValueError` carrying `message` and `code` attributes. -fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult { +pub(crate) fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult { let error: PyErr = PyValueError::new_err(message.clone()); let value = error.value(py); value.setattr("message", message)?; value.setattr("code", code)?; Ok(error) } - -/// Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -#[pyfunction] -#[pyo3(signature = (*, sql, context))] -pub(crate) fn assert_no_unresolved_sql_markers( - py: Python<'_>, - sql: &Bound<'_, PyString>, - context: &str, -) -> PyResult<()> { - let text = sql.to_string_lossy(); - let outcome: (Option, Option) = compiler_guard(|| { - Ok(match unresolved_reference_marker(&text) { - Ok(marker) => (marker, None), - Err(construct) => (None, Some(unclosed_message(context, construct))), - }) - })?; - match outcome { - (_, Some(unclosed)) => Err(PyErr::from_value( - error_class(py, ErrorId::CompileInput)?.call1((unclosed,))?, - )), - (Some(marker), None) => { - let (function, code) = MARKER_KINDS.get(marker).copied().ok_or_else(|| { - PyValueError::new_err(format!("unknown reference marker kind {marker}")) - })?; - Err(coded_value_error( - py, - format!("{context} still contains unresolved {function}() markers"), - code, - )?) - } - (None, None) => Ok(()), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assert_no_unresolved_sql_markers, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs index f7caf3f420..77a4e6f834 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs @@ -1,13 +1,24 @@ //! `analysis_cursor_sql` with each replaced call as a splice of `sql`. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::MODEL_SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::analysis_bound; use sqlbuild_analysis::sql_macros::models::TextSplice; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; /// `analysis_cursor_sql` with each replaced call as a splice of `sql`. -#[inline] pub fn analysis_cursor_splices( sql: &str, cursor_type: Option<&str>, ) -> Result<(String, Vec), String> { - cursor_intrinsics::analysis_cursor_splices(sql, cursor_type) + let literal: &str = analysis_bound(cursor_type); + let (replaced, spans) = + replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; + let splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + Ok((replaced, splices)) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs index 6aa9f8b277..235f6c8234 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs @@ -1,17 +1,18 @@ //! `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::MODEL_SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::analysis_bound; +use crate::attachments::_helpers::cursor_intrinsics::replaced; +use crate::attachments::_helpers::cursor_intrinsics::text_value; use pyo3::Python; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -#[inline] pub fn analysis_cursor_sql( py: Python<'_>, sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { - cursor_intrinsics::analysis_cursor_sql(py, sql, cursor_type) + let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs index 9b2119bd47..42d5fb8473 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs @@ -1,13 +1,17 @@ //! Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -use crate::attachments::_helpers::argument_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::attachments::_helpers::argument_values::argument_value; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_attachments::audits::models::ArgumentValue; /// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -#[inline] pub fn argument_pairs(arguments: &Bound<'_, PyDict>) -> PyResult> { - argument_values::argument_pairs(arguments) + let mut pairs: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.len()); + for (key, value) in arguments.iter() { + if let Ok(key) = key.downcast::() { + pairs.push((key.to_string_lossy().into_owned(), argument_value(&value)?)); + } + } + Ok(pairs) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs index 67a209e017..f54e584918 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs @@ -1,17 +1,46 @@ //! Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -use crate::attachments::_helpers::reference_markers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyString; +use crate::attachments::_helpers::reference_markers::MARKER_KINDS; +use crate::attachments::_helpers::reference_markers::coded_value_error; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_attachments::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; +use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, sql, context))] pub fn assert_no_unresolved_sql_markers( py: Python<'_>, sql: &Bound<'_, PyString>, context: &str, ) -> PyResult<()> { - reference_markers::assert_no_unresolved_sql_markers(py, sql, context) + let text = sql.to_string_lossy(); + let outcome: (Option, Option) = compiler_guard(|| { + Ok(match unresolved_reference_marker(&text) { + Ok(marker) => (marker, None), + Err(construct) => (None, Some(unclosed_message(context, construct))), + }) + })?; + match outcome { + (_, Some(unclosed)) => Err(PyErr::from_value( + error_class(py, ErrorId::CompileInput)?.call1((unclosed,))?, + )), + (Some(marker), None) => { + let (function, code) = MARKER_KINDS.get(marker).copied().ok_or_else(|| { + PyValueError::new_err(format!("unknown reference marker kind {marker}")) + })?; + Err(coded_value_error( + py, + format!("{context} still contains unresolved {function}() markers"), + code, + )?) + } + (None, None) => Ok(()), + } } diff --git a/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs index 72483c1606..8c4a8e715e 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs @@ -1,12 +1,13 @@ //! A `CompileInputError` carrying `message`. -use crate::attachments::_helpers::cursor_intrinsics; use pyo3::Python; -use pyo3::prelude::PyErr; -use pyo3::prelude::PyResult; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult}; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// A `CompileInputError` carrying `message`. -#[inline] pub fn compile_input_error(py: Python<'_>, message: String) -> PyResult { - cursor_intrinsics::compile_input_error(py, message) + Ok(PyErr::from_value( + error_class(py, ErrorId::CompileInput)?.call1((message,))?, + )) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs index e7f68a7202..f8f08cfaa6 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs @@ -1,13 +1,34 @@ //! Native-built argument containers need no intermediate Python list or tuple. -use crate::attachments::_helpers::argument_values; +use crate::attachments::_helpers::argument_values::argument_value; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_attachments::audits::models::ArgumentValue; +use sqlbuild_model_config::config_values::models::ConfigNode; /// Native-built argument containers need no intermediate Python list or tuple. -#[inline] pub fn config_argument_value(py: Python<'_>, value: &PyConfigNode) -> PyResult { - argument_values::config_argument_value(py, value) + Ok(match value { + ConfigNode::Null => ArgumentValue::Null, + ConfigNode::Bool(value) => ArgumentValue::Boolean(*value), + ConfigNode::Int(value) => ArgumentValue::Number(value.to_string()), + ConfigNode::Str(value) => ArgumentValue::Text(value.clone()), + ConfigNode::List(items) => ArgumentValue::List( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Tuple(items) => ArgumentValue::Tuple( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Map(_) => ArgumentValue::Opaque, + ConfigNode::Float(_) | ConfigNode::Host(_) => { + return argument_value(&config_object(py, value)?); + } + }) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs index ca1c8b5033..d7c6d9fda0 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs @@ -1,17 +1,27 @@ //! One function file's parsed header; non-string keys and `None` values are never read. -use crate::attachments::_helpers::function_headers; -use pyo3::prelude::Bound; -use pyo3::types::PyDict; -use sqlbuild_attachments::functions::models::FunctionHeader; -use sqlbuild_attachments::functions::models::FunctionLanguage; +use crate::attachments::_helpers::function_headers::header_value; +use crate::attachments::_helpers::function_headers::text_value; +use pyo3::prelude::{Bound, PyAnyMethods}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; +use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; /// One function file's parsed header; non-string keys and `None` values are never read. -#[inline] pub fn parsed_header( header_values: &Bound<'_, PyDict>, language: FunctionLanguage, relative_path: &str, ) -> FunctionHeader { - function_headers::parsed_header(header_values, language, relative_path) + let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); + for (key, value) in header_values.iter() { + let Some(name) = text_value(&key) else { + continue; + }; + if value.is_none() { + continue; + } + header.push((name, header_value(&value))); + } + parse_function_header(&header, language, relative_path) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs index a84d8eb71e..3a29a82770 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs @@ -1,12 +1,11 @@ //! Register the ceremonial select functions and classes. -use crate::attachments::_helpers::ceremonial_select; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::_helpers::ceremonial_select::omitted_ceremonial_select; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the ceremonial select functions and classes. -#[inline] pub fn register_ceremonial_select(module: &Bound<'_, PyModule>) -> PyResult<()> { - ceremonial_select::register(module) + module.add_function(wrap_pyfunction!(omitted_ceremonial_select, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs index 4efba4d901..88076527c0 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs @@ -1,12 +1,15 @@ //! Register the cursor intrinsics functions and classes. -use crate::attachments::_helpers::cursor_intrinsics; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::_helpers::cursor_intrinsics::cursor_intrinsics_analysis_sql; +use crate::attachments::_helpers::cursor_intrinsics::validated_model_cursor_intrinsics; +use crate::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor intrinsics functions and classes. -#[inline] pub fn register_cursor_intrinsics(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_intrinsics::register(module) + module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; + module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs index 9f79437e34..9e9a25e353 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs @@ -1,12 +1,11 @@ //! Register the reference markers functions and classes. -use crate::attachments::_helpers::reference_markers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the reference markers functions and classes. -#[inline] pub fn register_reference_markers(module: &Bound<'_, PyModule>) -> PyResult<()> { - reference_markers::register(module) + module.add_function(wrap_pyfunction!(assert_no_unresolved_sql_markers, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs index 1b3973bc25..cd44c68744 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs @@ -1,9 +1,11 @@ //! The reserved executor bound markers no authored SQL may contain. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::RESERVED_CURSOR_MARKERS; /// The reserved executor bound markers no authored SQL may contain. -#[inline] pub fn reserved_cursor_markers() -> Vec { - cursor_intrinsics::reserved_cursor_markers() + RESERVED_CURSOR_MARKERS + .iter() + .map(|marker| (*marker).to_owned()) + .collect() } diff --git a/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs index 752c0d4d25..a6b1663486 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs @@ -1,16 +1,32 @@ //! Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -use crate::attachments::_helpers::cursor_intrinsics; -use pyo3::Python; +use crate::attachments::_helpers::cursor_intrinsics::CURSOR_END_CALL; +use crate::attachments::_helpers::cursor_intrinsics::CURSOR_START_CALL; +use crate::attachments::_helpers::cursor_intrinsics::SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::replaced; use pyo3::prelude::PyResult; +use pyo3::{Python, pyfunction}; /// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] pub fn resolve_cursor_intrinsics( py: Python<'_>, sql: &str, start_sql: Option<&str>, end_sql: Option<&str>, ) -> PyResult<(String, bool)> { - cursor_intrinsics::resolve_cursor_intrinsics(py, sql, start_sql, end_sql) + let (rendered, found) = replaced( + py, + sql, + SQL_CONTEXT, + ( + start_sql.unwrap_or(CURSOR_START_CALL), + end_sql.unwrap_or(CURSOR_END_CALL), + ), + )?; + if !found || start_sql.is_none() || end_sql.is_none() { + return Ok((sql.to_owned(), found)); + } + Ok((rendered, true)) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs index e93c8a78c7..3214f4162c 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs @@ -1,17 +1,36 @@ //! One model query with canonical intrinsic calls, raising when the model may not use them. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::INCREMENTAL_MATERIALIZATION; +use crate::attachments::_helpers::cursor_intrinsics::text_value; +use crate::attachments::main::compile_input_error::compile_input_error; +use crate::attachments::main::reserved_cursor_markers::reserved_cursor_markers; use pyo3::Python; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; /// One model query with canonical intrinsic calls, raising when the model may not use them. -#[inline] pub fn validated_model_cursor_sql( py: Python<'_>, sql: &str, (model_name, config_values): (&str, &Bound<'_, PyAny>), ) -> PyResult { - cursor_intrinsics::validated_model_cursor_sql(py, sql, (model_name, config_values)) + let incremental: bool = text_value(&mapping_get(config_values, "materialized")?) + .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); + let cursor: Option = text_value(&mapping_get(config_values, "cursor")?); + let reserved_markers: Vec = reserved_cursor_markers(); + let outcome: Result = compiler_guard(|| { + Ok(validated_model_intrinsics( + sql, + &reserved_markers, + &IntrinsicModel { + name: model_name, + incremental, + cursor: cursor.as_deref(), + }, + )) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) } diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs index 3c31b6c879..0a3fc1e0c5 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs @@ -1,8 +1,8 @@ //! Optional authored string and boolean settings, rejected with the caller's error class. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::Request; @@ -42,7 +42,9 @@ impl<'py> Setting<'py> { /// An optional non-empty string setting, or `None` when absent. #[pyfunction] #[pyo3(signature = (**request))] -fn optional_named_string<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn optional_named_string<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let setting: Setting<'py> = Setting::read(&Request::new("optional_named_string", request)?)?; if setting.raw_value.is_none() { return Ok(setting.raw_value); @@ -60,7 +62,9 @@ fn optional_named_string<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< /// An optional boolean setting, or `default` when absent. #[pyfunction] #[pyo3(signature = (**request))] -fn optional_named_bool<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn optional_named_bool<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("optional_named_bool", request)?; let setting: Setting<'py> = Setting::read(&request)?; if setting.raw_value.is_none() { @@ -71,9 +75,3 @@ fn optional_named_bool<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(optional_named_string, module)?)?; - module.add_function(wrap_pyfunction!(optional_named_bool, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs index 605701b73f..730cb3eb5e 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs @@ -3,11 +3,8 @@ use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_python_base::native_host::main::host_error::host_error; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDict; const JSON_MODULE: &str = "json"; /// Characters of a structured value's JSON preview before it is cut short. @@ -15,12 +12,12 @@ const PREVIEW_LIMIT: usize = 120; const PREVIEW_ELLIPSIS: &str = "..."; /// `SharedInputError(message)`. -fn shared_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn shared_input_error(py: Python<'_>, message: String) -> PyErr { host_error(py, FacadeHost::SharedInputError, &message) } /// Compact, key-sorted JSON of `value`, cut to the preview limit. -fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = value.py(); let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("separators", (",", ":"))?; @@ -39,58 +36,3 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { .collect(); Ok(format!("{kept}{PREVIEW_ELLIPSIS}")) } - -/// One project var value as interpolated text; structured values are rejected. -#[pyfunction] -#[pyo3(signature = (*, value, label))] -pub(crate) fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_var_text(value, label) -} - -/// One project var value as interpolated text; structured values are rejected. -pub(crate) fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - let py: Python<'_> = value.py(); - if value.is_none() { - return Ok(String::new()); - } - if value.is_instance_of::() { - return Ok(if value.is_truthy()? { "true" } else { "false" }.to_owned()); - } - if value.is_instance_of::() - || value.is_instance_of::() - || value.is_instance_of::() - { - return value.str()?.extract(); - } - let structure: Option<&str> = if value.is_instance_of::() { - Some("an object") - } else if value.is_instance_of::() { - Some("an array") - } else { - None - }; - let message: String = match structure { - Some(structure) => format!( - "{label} is {structure} and cannot be interpolated as text: {}. Use a macro to \ - consume structured vars.", - preview_json(value)? - ), - None => format!( - "{label} has unsupported type {} and cannot be interpolated as text", - value.get_type().name()?.repr()? - ), - }; - Err(shared_input_error(py, message)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_project_var_text, module)?)?; - Ok(()) -} - -/// `render_project_var_text` as a Python function. -pub(crate) fn render_project_var_text_function( - py: pyo3::Python<'_>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(render_project_var_text, py) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs index d6f912de6c..21df75d7b3 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs @@ -1,14 +1,12 @@ //! Python-node authoring checks: retry policies, audit cases and hook reads, in Python's operators. use pyo3::exceptions::{PyBaseException, PyTypeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyDict, PyDictMethods, PyInt, PyMemoryView, PyString, PyStringMethods, PyTuple, PyType, }; use pyo3::types::{PyTupleMethods, PyTypeMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; use sqlbuild_model_config::header_metadata::main::json_meta_problem::json_meta_problem; @@ -57,7 +55,7 @@ fn exception_class(value: &Bound<'_, PyAny>) -> PyResult { /// The validated `retry_on` exception classes of a retry policy. #[pyfunction] #[pyo3(signature = (**request))] -fn retry_policy_exceptions<'py>( +pub(crate) fn retry_policy_exceptions<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request: Request<'_, 'py> = Request::new("retry_policy_exceptions", request)?; @@ -100,7 +98,7 @@ fn retry_policy_exceptions<'py>( /// The delay before retry `retry_index`: exponential backoff, capped, and jittered by `random`. #[pyfunction] #[pyo3(signature = (*, retry_policy, retry_index))] -fn retry_delay_seconds<'py>( +pub(crate) fn retry_delay_seconds<'py>( retry_policy: &Bound<'py, PyAny>, retry_index: &Bound<'py, PyAny>, ) -> PyResult> { @@ -145,7 +143,7 @@ fn valid_count(value: &Bound<'_, PyAny>) -> PyResult { /// Reject an audit case without a lowercase name, a definition, valid counts, or JSON meta. #[pyfunction] #[pyo3(signature = (**request))] -fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> { +pub(crate) fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> { let request = Request::new("validate_audit_case", request)?; let py: Python<'_> = request.py(); let name: &Bound<'_, PyAny> = &request.get("name")?; @@ -179,7 +177,9 @@ fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> /// `@hook(reads=...)` references in first-seen order; a leading invalid `None` is tolerated. #[pyfunction] -fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn normalize_hook_reads<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = value.py(); let reference: Bound<'py, PyAny> = host_entry(py, UserCodeHost::SqlResourceRef)?; let reads: Vec> = if value.is_instance(&reference)? { @@ -209,11 +209,3 @@ fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; - module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; - module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; - module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs index b0be7fdc1d..3ffdb4e9aa 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs @@ -1,22 +1,19 @@ //! Typed SQL value normalization between Python authored values and native `SqlValue`s. +use crate::authored_values::main::sql_value::sql_value; +use crate::authored_values::main::sql_value_error::sql_value_error; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, }; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::models::{SqlLogicalType, SqlValue}; -use sqlbuild_model_config::sql_values::errors::SqlValueFailure; use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value as normalized; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; fn is_exactly(value: &Bound<'_, PyAny>, class: &Bound<'_, PyAny>) -> bool { @@ -110,7 +107,10 @@ pub(crate) fn logical_type<'py>( classes.logical_type.call1((kind, element)) } -fn payload<'py>(value: &SqlValue, classes: &SqlValueClasses<'py>) -> PyResult> { +pub(crate) fn payload<'py>( + value: &SqlValue, + classes: &SqlValueClasses<'py>, +) -> PyResult> { let py = classes.kind.py(); Ok(match value { SqlValue::Null => py.None().into_bound(py), @@ -140,35 +140,10 @@ fn payload<'py>(value: &SqlValue, classes: &SqlValueClasses<'py>) -> PyResult( - value: &SqlValue, - classes: &SqlValueClasses<'py>, -) -> PyResult> { - classes.sql_value.call1(( - logical_type(&value.logical_type(), classes)?, - payload(value, classes)?, - )) -} - -/// The exception a normalization failure raises. -pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { - Ok(match failure { - SqlValueFailure::Invalid(message) => PyErr::from_value( - native_attr(py, NativeAttr::SqlValueValidationError)?.call1((message,))?, - ), - SqlValueFailure::DecimalOverflow => { - let overflow = py.import("decimal")?.getattr("Overflow")?; - let classes = pyo3::types::PyList::new(py, [overflow.clone()])?; - PyErr::from_value(overflow.call1((classes,))?) - } - }) -} - /// `normalize_sql_value`: the canonical `SqlValue` of an authored value. #[pyfunction] #[pyo3(signature = (*, raw_value, context, explicit_type = None, limits = None))] -fn normalize_sql_value<'py>( +pub(crate) fn normalize_sql_value<'py>( raw_value: &Bound<'py, PyAny>, context: &str, explicit_type: Option<&str>, @@ -191,7 +166,3 @@ fn normalize_sql_value<'py>( Err(failure) => Err(sql_value_error(py, failure)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs index 81300b47ea..8f92f84bb9 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs @@ -1,12 +1,42 @@ //! One project var value as interpolated text; structured values are rejected. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::project_vars::preview_json; +use crate::authored_values::_helpers::project_vars::shared_input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; /// One project var value as interpolated text; structured values are rejected. -#[inline] pub fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_vars::project_var_text(value, label) + let py: Python<'_> = value.py(); + if value.is_none() { + return Ok(String::new()); + } + if value.is_instance_of::() { + return Ok(if value.is_truthy()? { "true" } else { "false" }.to_owned()); + } + if value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + return value.str()?.extract(); + } + let structure: Option<&str> = if value.is_instance_of::() { + Some("an object") + } else if value.is_instance_of::() { + Some("an array") + } else { + None + }; + let message: String = match structure { + Some(structure) => format!( + "{label} is {structure} and cannot be interpolated as text: {}. Use a macro to \ + consume structured vars.", + preview_json(value)? + ), + None => format!( + "{label} has unsupported type {} and cannot be interpolated as text", + value.get_type().name()?.repr()? + ), + }; + Err(shared_input_error(py, message)) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs index 1415e788e2..c244fd23ec 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs @@ -2,9 +2,7 @@ use crate::authored_values::_helpers::sql_values; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_model_config::sql_values::models::RawSqlValue; /// An authored Python value read by exact type, as `normalize_sql_value` dispatches. diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs index a47e495da2..b1680e2e8f 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs @@ -1,12 +1,13 @@ //! Register the fields functions and classes. -use crate::authored_values::_helpers::fields; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::fields::optional_named_bool; +use crate::authored_values::_helpers::fields::optional_named_string; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fields functions and classes. -#[inline] pub fn register_fields(module: &Bound<'_, PyModule>) -> PyResult<()> { - fields::register(module) + module.add_function(wrap_pyfunction!(optional_named_string, module)?)?; + module.add_function(wrap_pyfunction!(optional_named_bool, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs index 52821b1733..a1ce916e50 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs @@ -1,12 +1,11 @@ //! Register the project vars functions and classes. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::main::render_project_var_text::render_project_var_text; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project vars functions and classes. -#[inline] pub fn register_project_vars(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_vars::register(module) + module.add_function(wrap_pyfunction!(render_project_var_text, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs index 7db72c6561..846d622ee1 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs @@ -1,12 +1,17 @@ //! Register the python nodes functions and classes. -use crate::authored_values::_helpers::python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::python_nodes::normalize_hook_reads; +use crate::authored_values::_helpers::python_nodes::retry_delay_seconds; +use crate::authored_values::_helpers::python_nodes::retry_policy_exceptions; +use crate::authored_values::_helpers::python_nodes::validate_audit_case; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python nodes functions and classes. -#[inline] pub fn register_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_nodes::register(module) + module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; + module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; + module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; + module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs index 07a16cc1d4..2de7dbd65e 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs @@ -1,12 +1,10 @@ //! Register the sql values functions and classes. -use crate::authored_values::_helpers::sql_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::sql_values::normalize_sql_value; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql values functions and classes. -#[inline] pub fn register_sql_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_values::register(module) + module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs index d3a2aac8df..7dce4c2620 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs @@ -1,12 +1,12 @@ //! One project var value as interpolated text; structured values are rejected. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::authored_values::main::project_var_text::project_var_text; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// One project var value as interpolated text; structured values are rejected. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, value, label))] pub fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_vars::render_project_var_text(value, label) + project_var_text(value, label) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs index 9cfdde9bd9..d220e241d4 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs @@ -1,11 +1,10 @@ //! `render_project_var_text` as a Python function. -use crate::authored_values::_helpers::project_vars; +use crate::authored_values::main::render_project_var_text::render_project_var_text; /// `render_project_var_text` as a Python function. -#[inline] pub fn render_project_var_text_function( py: pyo3::Python<'_>, ) -> pyo3::PyResult> { - project_vars::render_project_var_text_function(py) + pyo3::wrap_pyfunction!(render_project_var_text, py) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs index 24b7318920..1661b9edd9 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs @@ -1,17 +1,18 @@ //! A Python `SqlValue` for a native one. -use crate::authored_values::_helpers::sql_values; +use crate::authored_values::_helpers::sql_values::logical_type; +use crate::authored_values::_helpers::sql_values::payload; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_core::models::SqlValue; /// A Python `SqlValue` for a native one. -#[inline] pub fn sql_value<'py>( value: &SqlValue, classes: &SqlValueClasses<'py>, ) -> PyResult> { - sql_values::sql_value(value, classes) + classes.sql_value.call1(( + logical_type(&value.logical_type(), classes)?, + payload(value, classes)?, + )) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs index a3fbc75484..33ee056f1b 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs @@ -1,13 +1,20 @@ //! The exception a normalization failure raises. -use crate::authored_values::_helpers::sql_values; -use pyo3::prelude::PyErr; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_model_config::sql_values::errors::SqlValueFailure; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::NativeAttr; /// The exception a normalization failure raises. -#[inline] pub fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { - sql_values::sql_value_error(py, failure) + Ok(match failure { + SqlValueFailure::Invalid(message) => PyErr::from_value( + native_attr(py, NativeAttr::SqlValueValidationError)?.call1((message,))?, + ), + SqlValueFailure::DecimalOverflow => { + let overflow = py.import("decimal")?.getattr("Overflow")?; + let classes = pyo3::types::PyList::new(py, [overflow.clone()])?; + PyErr::from_value(overflow.call1((classes,))?) + } + }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs index 41f7b25158..9f5038fdf0 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs @@ -1,61 +1,3 @@ //! The worker limit of one command: `--concurrency`, the target, then `[settings]`. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; -use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; - -use crate::compile_attachment::main::build_effective_settings::build_effective_settings; -use crate::contracts::main::resolve_target_config::resolve_target_config; -use crate::contracts::main::resolve_target_name::resolve_target_name; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const CONCURRENCY: &str = "concurrency"; - -/// The effective concurrency of a command on `selected_target`, and the layer that set it. -pub(crate) fn resolve_concurrency( - requested: &Bound<'_, PyAny>, - discovered: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, -) -> PyResult { - resolve_configured_concurrency( - requested, - ( - &native_field(discovered, "project_config")?, - &native_field(discovered, "local_config")?, - ), - selected_target, - ) -} - -/// `resolve_concurrency` from the project and local configs themselves. -pub(crate) fn resolve_configured_concurrency( - requested: &Bound<'_, PyAny>, - (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - selected_target: &Bound<'_, PyAny>, -) -> PyResult { - let py = project_config.py(); - let target: Option<(String, i64)> = - match resolve_target_name(py, project_config, local_config, selected_target)? { - Some(name) => { - let config: Bound<'_, PyAny> = - resolve_target_config(py, project_config, local_config, name.as_any())?; - let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; - limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) - } - None => None, - } - .transpose()?; - let settings: i64 = native_field( - &build_effective_settings(project_config, local_config)?, - CONCURRENCY, - )? - .extract()?; - let local_override: bool = - native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; - Ok(effective_concurrency(&ConcurrencyLayers { - requested: requested.extract()?, - target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), - settings, - local_override, - })) -} +pub(crate) const CONCURRENCY: &str = "concurrency"; diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs index 0c69f9bdda..da83e1347d 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs @@ -1,17 +1,22 @@ //! The effective concurrency of a command on `selected_target`, and the layer that set it. -use crate::command_runtime::_helpers::concurrency; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_config::effective::models::EffectiveConcurrency; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The effective concurrency of a command on `selected_target`, and the layer that set it. -#[inline] pub fn resolve_concurrency( requested: &Bound<'_, PyAny>, discovered: &Bound<'_, PyAny>, selected_target: &Bound<'_, PyAny>, ) -> PyResult { - concurrency::resolve_concurrency(requested, discovered, selected_target) + resolve_configured_concurrency( + requested, + ( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + ), + selected_target, + ) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs index 5faa1fac25..5a40e5d3a8 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs @@ -1,21 +1,43 @@ //! `resolve_concurrency` from the project and local configs themselves. -use crate::command_runtime::_helpers::concurrency; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_config::effective::models::EffectiveConcurrency; +use crate::command_runtime::_helpers::concurrency::CONCURRENCY; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; +use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `resolve_concurrency` from the project and local configs themselves. -#[inline] pub fn resolve_configured_concurrency( requested: &Bound<'_, PyAny>, (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), selected_target: &Bound<'_, PyAny>, ) -> PyResult { - concurrency::resolve_configured_concurrency( - requested, - (project_config, local_config), - selected_target, - ) + let py = project_config.py(); + let target: Option<(String, i64)> = + match resolve_target_name(py, project_config, local_config, selected_target)? { + Some(name) => { + let config: Bound<'_, PyAny> = + resolve_target_config(py, project_config, local_config, name.as_any())?; + let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; + limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + } + None => None, + } + .transpose()?; + let settings: i64 = native_field( + &build_effective_settings(project_config, local_config)?, + CONCURRENCY, + )? + .extract()?; + let local_override: bool = + native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; + Ok(effective_concurrency(&ConcurrencyLayers { + requested: requested.extract()?, + target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + settings, + local_override, + })) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs index 30d675df80..a3b8cd8894 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs @@ -1,48 +1,21 @@ //! The effective connection, settings and vars of a project, its local config and target. +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::pyfunction; -use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::contracts::main::effective_config_settings::effective_config_settings; -use crate::contracts::main::effective_config_vars::effective_config_vars; use crate::contracts::main::effective_connection_fields::effective_connection_fields; -use crate::contracts::main::resolve_target_config::resolve_target_config; -use crate::contracts::main::resolve_target_name::resolve_target_name; -use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; use crate::model_config::main::config::expanded_template_data::expanded_template_data; -use crate::model_config::main::config::run_id::run_id; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; const CONNECTION_LABEL: &str = "effective connection"; -/// Merge base project connection with the selected environment overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] -pub(crate) fn build_effective_connection<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, - effective_vars: &Bound<'py, PyAny>, -) -> PyResult> { - expanded_connection( - (project_config, local_config), - target_config, - effective_vars, - false, - ) -} - /// Expand only the connection selected for the caller's writer or catalog role. -fn expanded_connection<'py>( +pub(crate) fn expanded_connection<'py>( (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, @@ -72,77 +45,8 @@ fn expanded_connection<'py>( ) } -/// Merge project settings with local developer overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn build_effective_settings<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let settings_class: Bound<'py, PyAny> = record_class(py, RecordId::SettingsConfig)?; - let names: Vec> = imported(py, "dataclasses", "fields")? - .call1((&settings_class,))? - .try_iter()? - .map(|field| native_field(&field?, "name")) - .collect::>()?; - let values: Bound<'py, PyDict> = effective_config_settings( - py, - ( - native_field(project_config, "settings")?, - native_field(local_config, "settings")?, - ), - &native_field(local_config, "setting_overrides")?, - names - .into_iter() - .map(|name| Ok(name.cast_into::()?)) - .collect::>()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .downcast_into()?; - new_record(py, RecordId::SettingsConfig, (), Some(&values)) -} - -/// Merge effective vars using the locked precedence order. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] -pub(crate) fn build_effective_vars<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, - cli_vars: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let merged: Bound<'py, PyAny> = effective_config_vars( - py, - (project_config.clone(), local_config.clone()), - target_config, - cli_vars, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - expanded_effective_vars(merged.downcast()?) -} - -/// The selected target name of a discovered project and its effective config, if any. -pub(crate) fn target_selection<'py>( - discovered: &Bound<'py, PyAny>, - selected: Option>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py: Python<'py> = discovered.py(); - let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; - let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); - let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { - return Ok((py.None().into_bound(py), py.None().into_bound(py))); - }; - let name: Bound<'py, PyAny> = name.into_any(); - let config: Bound<'py, PyAny> = - resolve_target_config(py, &project_config, &local_config, &name)?; - Ok((name, config)) -} - /// The effective vars of a discovered project for `target_config` and optional CLI vars. -fn discovered_vars<'py>( +pub(crate) fn discovered_vars<'py>( discovered: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, cli_vars: Option>, @@ -159,126 +63,3 @@ fn discovered_vars<'py>( &cli_vars, ) } - -/// Build effective target config without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None))] -pub(crate) fn build_effective_target_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, -) -> PyResult> { - Ok(target_selection(discovered_inputs, selected_target)?.1) -} - -/// Build the effective project connection config without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] -pub(crate) fn build_effective_connection_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, - cli_vars: Option>, - catalog: bool, -) -> PyResult> { - let (_, target_config) = target_selection(discovered_inputs, selected_target)?; - let effective_vars: Bound<'py, PyAny> = - discovered_vars(discovered_inputs, &target_config, cli_vars)?; - expanded_connection( - ( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, - ), - &target_config, - &effective_vars, - catalog, - ) -} - -/// Build environment, vars and invocation id without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] -pub(crate) fn build_effective_runtime_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, - cli_vars: Option>, -) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; - let effective_vars: Bound<'py, PyAny> = - discovered_vars(discovered_inputs, &target_config, cli_vars)?; - let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; - PyTuple::new(py, [target_name, effective_vars, run_id]) -} - -/// Build effective settings without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs))] -pub(crate) fn build_effective_settings_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - build_effective_settings( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, - ) -} - -/// `build_effective_vars` for a discovered project and target, without CLI vars. -pub(crate) fn project_vars<'py>( - discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, -) -> PyResult> { - build_effective_vars( - &native_field(discovered, "project_config")?, - &native_field(discovered, "local_config")?, - target_config, - PyDict::new(discovered.py()).as_any(), - ) -} - -/// `build_effective_connection` as a function of `module`. -pub(crate) fn build_effective_connection_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_connection, module) -} - -/// `build_effective_connection_config` as a function of `module`. -pub(crate) fn build_effective_connection_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_connection_config, module) -} - -/// `build_effective_runtime_config` as a function of `module`. -pub(crate) fn build_effective_runtime_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_runtime_config, module) -} - -/// `build_effective_settings` as a function of `module`. -pub(crate) fn build_effective_settings_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_settings, module) -} - -/// `build_effective_settings_config` as a function of `module`. -pub(crate) fn build_effective_settings_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_settings_config, module) -} - -/// `build_effective_target_config` as a function of `module`. -pub(crate) fn build_effective_target_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_target_config, module) -} - -/// `build_effective_vars` as a function of `module`. -pub(crate) fn build_effective_vars_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_vars, module) -} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs index 1e07462d8f..8829856f5c 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs @@ -1,22 +1,22 @@ //! Merge base project connection with the selected environment overrides. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::expanded_connection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Merge base project connection with the selected environment overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] pub fn build_effective_connection<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_connection( - project_config, - local_config, + expanded_connection( + (project_config, local_config), target_config, effective_vars, + false, ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs index 6bde72927b..163d413662 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs @@ -1,22 +1,31 @@ //! Build the effective project connection config without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::discovered_vars; +use crate::compile_attachment::_helpers::effective_config::expanded_connection; +use crate::compile_attachment::main::target_selection::target_selection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Build the effective project connection config without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] pub fn build_effective_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, catalog: bool, ) -> PyResult> { - effective_config::build_effective_connection_config( - discovered_inputs, - selected_target, - cli_vars, + let (_, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + expanded_connection( + ( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ), + &target_config, + &effective_vars, catalog, ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs index 74e098cc13..baa40b6a27 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_connection_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; /// `build_effective_connection_config` as a function of `module`. -#[inline] pub fn build_effective_connection_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_connection_config_function(module) + pyo3::wrap_pyfunction!(build_effective_connection_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs index 1fd50f688d..7bc55e4d04 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs @@ -1,11 +1,10 @@ //! `build_effective_connection` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_connection::build_effective_connection; /// `build_effective_connection` as a function of `module`. -#[inline] pub fn build_effective_connection_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_connection_function(module) + pyo3::wrap_pyfunction!(build_effective_connection, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs index c4eeb120e1..a9d2c10702 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs @@ -1,17 +1,24 @@ //! Build environment, vars and invocation id without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::discovered_vars; +use crate::compile_attachment::main::target_selection::target_selection; +use crate::model_config::main::config::run_id::run_id; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// Build environment, vars and invocation id without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] pub fn build_effective_runtime_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, ) -> PyResult> { - effective_config::build_effective_runtime_config(discovered_inputs, selected_target, cli_vars) + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + PyTuple::new(py, [target_name, effective_vars, run_id]) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs index 2626522098..2ce5569a41 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_runtime_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; /// `build_effective_runtime_config` as a function of `module`. -#[inline] pub fn build_effective_runtime_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_runtime_config_function(module) + pyo3::wrap_pyfunction!(build_effective_runtime_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs index d5b3f04d24..beaf407f2e 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs @@ -1,15 +1,42 @@ //! Merge project settings with local developer overrides. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::effective_config_settings::effective_config_settings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Merge project settings with local developer overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn build_effective_settings<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_settings(project_config, local_config) + let py: Python<'py> = project_config.py(); + let settings_class: Bound<'py, PyAny> = record_class(py, RecordId::SettingsConfig)?; + let names: Vec> = imported(py, "dataclasses", "fields")? + .call1((&settings_class,))? + .try_iter()? + .map(|field| native_field(&field?, "name")) + .collect::>()?; + let values: Bound<'py, PyDict> = effective_config_settings( + py, + ( + native_field(project_config, "settings")?, + native_field(local_config, "settings")?, + ), + &native_field(local_config, "setting_overrides")?, + names + .into_iter() + .map(|name| Ok(name.cast_into::()?)) + .collect::>()?, + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? + .downcast_into()?; + new_record(py, RecordId::SettingsConfig, (), Some(&values)) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs index 7212e9a1bc..95f9b3632f 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs @@ -1,14 +1,18 @@ //! Build effective settings without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Build effective settings without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs))] pub fn build_effective_settings_config<'py>( discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_settings_config(discovered_inputs) + build_effective_settings( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs index 2bc134c68c..ffc2ba7d9d 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_settings_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; /// `build_effective_settings_config` as a function of `module`. -#[inline] pub fn build_effective_settings_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_settings_config_function(module) + pyo3::wrap_pyfunction!(build_effective_settings_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs index 4031736371..5699648601 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs @@ -1,11 +1,10 @@ //! `build_effective_settings` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; /// `build_effective_settings` as a function of `module`. -#[inline] pub fn build_effective_settings_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_settings_function(module) + pyo3::wrap_pyfunction!(build_effective_settings, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs index e7f58c2b4e..b6176d69e9 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs @@ -1,15 +1,15 @@ //! Build effective target config without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::target_selection::target_selection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Build effective target config without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None))] pub fn build_effective_target_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, ) -> PyResult> { - effective_config::build_effective_target_config(discovered_inputs, selected_target) + Ok(target_selection(discovered_inputs, selected_target)?.1) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs index c15c82fa8c..1b53f61b51 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_target_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_target_config::build_effective_target_config; /// `build_effective_target_config` as a function of `module`. -#[inline] pub fn build_effective_target_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_target_config_function(module) + pyo3::wrap_pyfunction!(build_effective_target_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs index 5225b0caad..f4b0350869 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs @@ -1,17 +1,26 @@ //! Merge effective vars using the locked precedence order. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::effective_config_vars::effective_config_vars; +use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; /// Merge effective vars using the locked precedence order. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] pub fn build_effective_vars<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, cli_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_vars(project_config, local_config, target_config, cli_vars) + let py: Python<'py> = project_config.py(); + let merged: Bound<'py, PyAny> = effective_config_vars( + py, + (project_config.clone(), local_config.clone()), + target_config, + cli_vars, + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + expanded_effective_vars(merged.downcast()?) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs index fa6d8a7d07..d566e756f8 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs @@ -1,11 +1,10 @@ //! `build_effective_vars` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; /// `build_effective_vars` as a function of `module`. -#[inline] pub fn build_effective_vars_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_vars_function(module) + pyo3::wrap_pyfunction!(build_effective_vars, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs index 3215cf81e9..70c5470939 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs @@ -1,15 +1,19 @@ //! `build_effective_vars` for a discovered project and target, without CLI vars. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_vars` for a discovered project and target, without CLI vars. -#[inline] pub fn project_vars<'py>( discovered: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::project_vars(discovered, target_config) + build_effective_vars( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target_config, + PyDict::new(discovered.py()).as_any(), + ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs index db99738dd0..a84523391c 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs @@ -1,15 +1,24 @@ //! The selected target name of a discovered project and its effective config, if any. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The selected target name of a discovered project and its effective config, if any. -#[inline] pub fn target_selection<'py>( discovered: &Bound<'py, PyAny>, selected: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - effective_config::target_selection(discovered, selected) + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); + let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + }; + let name: Bound<'py, PyAny> = name.into_any(); + let config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &name)?; + Ok((name, config)) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs index 3cc8fdb394..24d8b4bfd4 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs @@ -1,21 +1,17 @@ //! Validate loaded project and local configuration before discovery reads project files. +use crate::compile_driver::main::session_warehouse_support::session_warehouse_support; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_target_warehouse; -use sqlbuild_host::entries::models::AdapterHost; - -use sqlbuild_python_base::boundary::main::call_with::call_with; - use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::contracts::main::resolve_target_config::resolve_target_config; use crate::discovery::main::project::resolve_config_file_path::resolve_config_file_path; use crate::discovery::main::project::validate_local_sql_analysis::validate_local_sql_analysis; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -45,7 +41,9 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { } /// Reject a local SQL analysis opt-out the project forbids. -fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_local_sql_analysis_opt_out( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); let local_settings: Bound<'_, PyAny> = native_field(config.local_config, "settings")?; let flags: Bound<'_, PyTuple> = PyTuple::new( @@ -78,22 +76,6 @@ fn first_configured_group(config: &Bound<'_, PyAny>) -> PyResult, - adapter_name: &Bound<'_, PyAny>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult { - call_with( - &host_entry(py, AdapterHost::AdapterSupportsSessionWarehouse)?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir.clone()), - ], - )? - .is_truthy() -} - fn reject_group( py: Python<'_>, (local, adapter_name): (bool, &Bound<'_, PyAny>), @@ -120,7 +102,9 @@ fn reject_group( } /// Reject command-group warehouses under adapters without a session warehouse. -fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_unsupported_target_warehouses( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); if let Some(group) = first_configured_group(config.project_config)? { let adapter_name: Bound<'_, PyAny> = native_field(config.project_config, "adapter")?; @@ -153,7 +137,9 @@ fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) - } /// Resolve every configured target so invalid connections fail discovery. -fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_invalid_target_connections( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); let mut names: Vec = target_names(config.project_config)?; names.extend(target_names(config.local_config)?); @@ -181,19 +167,3 @@ fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> P } Ok(()) } - -/// Validate the SQL analysis policy, target warehouses and target connections, in that order. -pub(crate) fn reject_invalid_project_configuration( - config: &ProjectConfiguration<'_, '_>, -) -> PyResult<()> { - reject_local_sql_analysis_opt_out(config)?; - reject_invalid_connection_configuration(config) -} - -/// Validate the target warehouses and target connections a warehouse connection depends on. -pub(crate) fn reject_invalid_connection_configuration( - config: &ProjectConfiguration<'_, '_>, -) -> PyResult<()> { - reject_unsupported_target_warehouses(config)?; - reject_invalid_target_connections(config) -} diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs index 4dab78eccb..17960d30ae 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs @@ -1,13 +1,14 @@ //! Validate the target warehouses and target connections a warehouse connection depends on. -use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::_helpers::config_validation::reject_invalid_target_connections; +use crate::compile_driver::_helpers::config_validation::reject_unsupported_target_warehouses; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::PyResult; /// Validate the target warehouses and target connections a warehouse connection depends on. -#[inline] pub fn reject_invalid_connection_configuration( config: &ProjectConfiguration<'_, '_>, ) -> PyResult<()> { - config_validation::reject_invalid_connection_configuration(config) + reject_unsupported_target_warehouses(config)?; + reject_invalid_target_connections(config) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs index f7afdf5744..c9523744fa 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs @@ -1,11 +1,12 @@ //! Validate the SQL analysis policy, target warehouses and target connections, in that order. -use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::_helpers::config_validation::reject_local_sql_analysis_opt_out; +use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::PyResult; /// Validate the SQL analysis policy, target warehouses and target connections, in that order. -#[inline] pub fn reject_invalid_project_configuration(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { - config_validation::reject_invalid_project_configuration(config) + reject_local_sql_analysis_opt_out(config)?; + reject_invalid_connection_configuration(config) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs index 72ca9d4c1e..c5e3947899 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs @@ -1,17 +1,22 @@ //! Whether the adapter can switch warehouses per session, as the adapter host answers. -use crate::compile_driver::_helpers::config_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// Whether the adapter can switch warehouses per session, as the adapter host answers. -#[inline] pub fn session_warehouse_support( py: Python<'_>, adapter_name: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, ) -> PyResult { - config_validation::session_warehouse_support(py, adapter_name, project_dir) + call_with( + &host_entry(py, AdapterHost::AdapterSupportsSessionWarehouse)?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )? + .is_truthy() } diff --git a/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs index 4539fa9c21..ad700a9f8a 100644 --- a/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs @@ -1,23 +1,18 @@ //! Effective target, vars, connection and settings precedence over the loaded config objects. +use crate::contracts::main::effective_connection_fields::effective_connection_fields; use std::hash::{Hash, Hasher}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, PyStringMethods, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, PyStringMethods, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::effective::errors::{EffectiveConfigError, MessagePart}; use sqlbuild_config::effective::main::default_connection_name::default_connection_name; -use sqlbuild_config::effective::main::effective_connection::effective_connection; -use sqlbuild_config::effective::main::effective_settings::effective_settings; use sqlbuild_config::effective::main::effective_target_name::effective_target_name; -use sqlbuild_config::effective::main::effective_vars::effective_vars; use sqlbuild_config::effective::main::merged_target::merged_target; use sqlbuild_config::effective::main::target_fields::target_fields; use sqlbuild_config::effective::models::{ - ConnectionChoice, ConnectionLayers, FieldValue, MergeRule, TargetChoice, TargetField, - TargetLayer, + ConnectionChoice, FieldValue, MergeRule, TargetChoice, TargetField, TargetLayer, }; use sqlbuild_config::effective::types::Entries; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -27,7 +22,7 @@ const SURROGATE_PASS: &str = "surrogatepass"; /// A `str` key compared exactly: by its UTF-8 bytes, lone surrogates passed through. #[derive(Clone)] -struct Key<'py> { +pub(crate) struct Key<'py> { bytes: Vec, handle: Bound<'py, PyString>, } @@ -58,15 +53,15 @@ impl Ord for Key<'_> { } } -type Value<'py> = Bound<'py, PyAny>; +pub(crate) type Value<'py> = Bound<'py, PyAny>; type Layer<'py> = TargetLayer, Value<'py>>; -type Pairs<'py> = Entries, Value<'py>>; +pub(crate) type Pairs<'py> = Entries, Value<'py>>; /// `(group, name, value)` of each merged `TargetConfig` field. type FieldRow<'py> = (Option<&'static str>, &'static str, Value<'py>); /// A result, else the precedence failure message. pub(crate) type Outcome<'py, T> = (Option, Option>); -fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { let handle: Bound<'py, PyString> = value.downcast::()?.clone(); if let Ok(text) = handle.to_str() { return Ok(Key { @@ -82,14 +77,14 @@ fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { Ok(Key { bytes, handle }) } -fn optional_key<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn optional_key<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { if value.is_none() { return Ok(None); } key(value).map(Some) } -fn pairs<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn pairs<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { mapping .downcast::()? .iter() @@ -106,7 +101,10 @@ fn keys<'py>(mapping: &Bound<'py, PyAny>) -> PyResult>> { .collect() } -fn dictionary<'py>(py: Python<'py>, entries: Pairs<'py>) -> PyResult> { +pub(crate) fn dictionary<'py>( + py: Python<'py>, + entries: Pairs<'py>, +) -> PyResult> { let dictionary: Bound<'py, PyDict> = PyDict::new(py); for (name, value) in entries { dictionary.set_item(name.handle, value)?; @@ -162,7 +160,7 @@ fn is_set(value: &FieldValue, Value<'_>>) -> bool { } /// The failure message as a Python `str`, authored names inserted unchanged. -fn message<'py>( +pub(crate) fn message<'py>( py: Python<'py>, error: &EffectiveConfigError>, ) -> PyResult> { @@ -251,27 +249,7 @@ pub(crate) fn effective_config_target_fields<'py>( Ok((Some(rows), None)) } -/// `build_effective_vars` before template expansion. -#[pyfunction] -pub(crate) fn effective_config_vars<'py>( - py: Python<'py>, - configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), - target_config: &Bound<'py, PyAny>, - cli_vars: &Bound<'py, PyAny>, -) -> PyResult> { - let (project_config, local_config) = configs; - let project: Pairs<'py> = pairs(&native_field(&project_config, "vars")?)?; - let target: Pairs<'py> = if target_config.is_none() { - Vec::new() - } else { - pairs(&native_field(target_config, "vars")?)? - }; - let local: Pairs<'py> = pairs(&native_field(&local_config, "vars")?)?; - let cli: Pairs<'py> = pairs(cli_vars)?; - dictionary(py, effective_vars(&[&project, &target, &local, &cli])) -} - -fn named<'py>(mapping: &Bound<'py, PyAny>) -> PyResult, Pairs<'py>>> { +pub(crate) fn named<'py>(mapping: &Bound<'py, PyAny>) -> PyResult, Pairs<'py>>> { mapping .downcast::()? .iter() @@ -289,98 +267,3 @@ pub(crate) fn effective_config_connection<'py>( ) -> PyResult>> { effective_connection_fields(py, (project_config, local_config), target_config, false) } - -/// Catalog selection uses the same named-layer resolver, with no writer layers. -pub(crate) fn effective_connection_fields<'py>( - py: Python<'py>, - (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - target_config: &Bound<'py, PyAny>, - catalog: bool, -) -> PyResult>> { - let catalog_name = if catalog && !target_config.is_none() { - optional_key(&native_field(target_config, "catalog_connection")?)? - } else { - None - }; - let independent = catalog_name.is_some(); - let target: Option<(Option>, Pairs<'py>)> = if independent { - Some((catalog_name, Vec::new())) - } else if target_config.is_none() { - None - } else { - Some(( - optional_key(&native_field(target_config, "connection_name")?)?, - pairs(&native_field(target_config, "connection")?)?, - )) - }; - let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { - project: if independent { - Vec::new() - } else { - pairs(&native_field(project_config, "connection")?)? - }, - target, - project_named: named(&native_field(project_config, "connections")?)?, - local_named: named(&native_field(local_config, "connections")?)?, - local: if independent { - Vec::new() - } else { - pairs(&native_field(local_config, "connection")?)? - }, - }; - Ok(match effective_connection(layers) { - Ok(connection) => (Some(dictionary(py, connection)?), None), - Err(error) => { - let message = message(py, &error)?; - ( - None, - Some(if independent { - PyString::new(py, &format!("catalog_connection: {message}")) - } else { - message - }), - ) - } - }) -} - -/// `build_effective_settings`: setting values by field name, local overrides applied. -#[pyfunction] -pub(crate) fn effective_config_settings<'py>( - py: Python<'py>, - settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), - overrides: &Bound<'py, PyAny>, - names: Vec>, -) -> PyResult> { - let read = |settings: &Bound<'py, PyAny>| -> PyResult> { - names - .iter() - .map(|name| { - Ok(( - key(name.as_any())?, - native_field(settings, &name.to_cow()?)?, - )) - }) - .collect() - }; - let overridden: Vec> = overrides - .try_iter()? - .map(|name| key(&name?)) - .collect::>()?; - let (project_settings, local_settings) = settings; - let settings: Pairs<'py> = effective_settings( - read(&project_settings)?, - &read(&local_settings)?, - &overridden, - ); - dictionary(py, settings) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(effective_config_target_name, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_target_fields, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_vars, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_connection, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_settings, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs index 6fd358ee29..a989a2f477 100644 --- a/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs @@ -4,206 +4,57 @@ use std::cell::RefCell; use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; use sqlbuild_model_config::name_patterns::models::NamePattern; -use crate::contracts::_helpers::effective_config::effective_config_target_fields; -use crate::contracts::_helpers::effective_config::effective_config_target_name; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const TARGET_GROUPS: [(&str, RecordId); 4] = [ +pub(crate) const TARGET_GROUPS: [(&str, RecordId); 4] = [ ("clone", RecordId::ClonePolicy), ("execution_limits", RecordId::ExecutionLimitsConfig), ("diff", RecordId::TargetDiffConfig), ("warehouses", RecordId::TargetWarehousesConfig), ]; -const SNAPSHOT_LIMITS: [&str; 4] = [ +pub(crate) const SNAPSHOT_LIMITS: [&str; 4] = [ "max_rows_per_relation", "max_total_rows", "max_bytes_per_relation", "max_total_bytes", ]; const ALL_DIALECTS: &str = "*"; -const LOADER_SCHEMA_TABLE_PARTS: usize = 2; -const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; +pub(crate) const LOADER_SCHEMA_TABLE_PARTS: usize = 2; +pub(crate) const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; const NAME_PATTERN_CACHE_LIMIT: usize = 256; thread_local! { static NAME_PATTERNS: RefCell> = RefCell::new(HashMap::new()); } -fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { +pub(crate) fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { Ok(PyErr::from_value( error_class(py, ErrorId::SpecConfig)?.call1((message,))?, )) } -/// Resolve the effective target name. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, selected_target))] -pub(crate) fn resolve_target_name<'py>( - py: Python<'py>, - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - selected_target: &Bound<'py, PyAny>, -) -> PyResult>> { - let (name, error) = - effective_config_target_name(py, project_config, local_config, selected_target)?; - match error { - Some(message) => Err(spec_error(py, message)?), - None => Ok(name), - } -} - -/// Merge project target config with local developer overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_name))] -pub(crate) fn resolve_target_config<'py>( - py: Python<'py>, - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_name: &Bound<'py, PyAny>, -) -> PyResult> { - let (rows, error) = effective_config_target_fields( - py, - (project_config.clone(), local_config.clone()), - target_name, - &new_record(py, RecordId::TargetConfig, (), None)?, - )?; - let Some(rows) = rows else { - let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); - return Err(spec_error(py, message)?); - }; - let values: Bound<'py, PyDict> = PyDict::new(py); - let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS - .iter() - .map(|(group, _)| (*group, PyDict::new(py))) - .collect(); - for (group, name, value) in rows { - match group { - None => values.set_item(name, value)?, - Some(group) => { - if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { - fields.set_item(name, value)?; - } - } - } - } - for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { - let nested: Bound<'py, PyAny> = new_record(py, *class_name, (), Some(fields))?; - values.set_item(*group, nested)?; - } - new_record(py, RecordId::TargetConfig, (), Some(&values)) -} - -/// Resolve the effective adapter name, allowing local override. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn resolve_effective_adapter_name<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let local: Bound<'py, PyAny> = native_field(local_config, "adapter")?; - if local.is_none() { - return native_field(project_config, "adapter"); - } - Ok(local) -} - -/// Resolve declaration rendering over project configuration. -#[pyfunction] -#[pyo3(signature = (*, project_config, declaration_override))] -pub(crate) fn resolve_effective_collection_rendering<'py>( - project_config: &Bound<'py, PyAny>, - declaration_override: &Bound<'py, PyAny>, -) -> PyResult> { - if !declaration_override.is_none() { - return Ok(declaration_override.clone()); - } - native_field(project_config, "constants")?.getattr("collection_rendering") -} - /// `local` when set, else `project`. -fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bound<'py, PyAny> { +pub(crate) fn override_or<'py>( + local: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, +) -> Bound<'py, PyAny> { if local.is_none() { project } else { local } } -/// Resolve scenario config, allowing local overrides to replace project rules. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn resolve_effective_scenario_config<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let project: Bound<'py, PyAny> = native_field(project_config, "scenario")?; - let local: Bound<'py, PyAny> = native_field(local_config, "scenario")?; - let overrides: Bound<'py, PyDict> = PyDict::new(py); - for item in native_field(&project, "local_type_overrides")? - .call_method0("items")? - .try_iter()? - { - let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; - } - for item in native_field(&local, "local_type_overrides")? - .call_method0("items")? - .try_iter()? - { - let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - overrides - .call_method1("setdefault", (dialect, PyDict::new(py)))? - .call_method1("update", (rules,))?; - } - let project_limits: Bound<'py, PyAny> = native_field(&project, "snapshot_limits")?; - let local_limits: Bound<'py, PyAny> = native_field(&local, "snapshot_limits")?; - let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS - .iter() - .map(|field| { - Ok(( - *field, - override_or( - native_field(&local_limits, field)?, - native_field(&project_limits, field)?, - ), - )) - }) - .collect::>()?; - new_record_with( - py, - RecordId::ScenarioConfig, - &[ - ( - "run_namespace", - override_or( - native_field(&local, "run_namespace")?, - native_field(&project, "run_namespace")?, - ), - ), - ("local_type_overrides", overrides.into_any()), - ( - "snapshot_limits", - new_record_with(py, RecordId::ScenarioSnapshotLimitsConfig, &limits)?, - ), - ], - ) -} - /// Return global and dialect-specific scenario local type override rules. #[pyfunction] #[pyo3(signature = (*, scenario_config, sql_analysis_dialect))] -fn scenario_local_type_overrides_for_dialect<'py>( +pub(crate) fn scenario_local_type_overrides_for_dialect<'py>( scenario_config: &Bound<'py, PyAny>, sql_analysis_dialect: &Bound<'py, PyAny>, ) -> PyResult> { @@ -228,69 +79,13 @@ fn scenario_local_type_overrides_for_dialect<'py>( } /// Whether `text` is empty or only whitespace as Python's `str.strip` sees it. -fn blank(text: &str) -> bool { +pub(crate) fn blank(text: &str) -> bool { text.chars() .all(|character| character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character)) } -/// Parse a one-, two-, or three-part loader destination. -#[pyfunction] -#[pyo3(signature = (*, destination, default_database, default_schema))] -pub(crate) fn loader_destination_parts<'py>( - py: Python<'py>, - destination: &str, - default_database: Option>, - default_schema: Option>, -) -> PyResult> { - loader_parts(py, destination, (default_database, default_schema)) -} - -pub(crate) fn loader_parts<'py>( - py: Python<'py>, - destination: &str, - (default_database, default_schema): (Option>, Option>), -) -> PyResult> { - let parts: Vec<&str> = destination.split('.').collect(); - if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) - || parts.iter().any(|part| blank(part)) - { - return Err(spec_error( - py, - PyString::new( - py, - &format!( - "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ - dot-separated parts (table, schema.table, or database.schema.table)" - ), - ), - )?); - } - let none = || py.None().into_bound(py); - let text = |part: &str| PyString::new(py, part).into_any(); - let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = - match parts.as_slice() { - [table] => ( - default_database.unwrap_or_else(none), - default_schema.unwrap_or_else(none), - text(table), - ), - [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( - default_database.unwrap_or_else(none), - text(schema), - text(table), - ), - _ => (text(parts[0]), text(parts[1]), text(parts[2])), - }; - new_record( - py, - RecordId::LoaderDestinationParts, - (database, schema, table), - None, - ) -} - /// Whether a family's optional `name_pattern` accepts `column_name`. -fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { +pub(crate) fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { let pattern: Option = native_field(family, "name_pattern")?.extract()?; let Some(pattern) = pattern else { return Ok(true); @@ -310,38 +105,3 @@ fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult( - families: &Bound<'py, PyAny>, - column_name: &str, -) -> PyResult> { - let mut matching: Vec> = Vec::new(); - for family in families.try_iter()? { - let family: Bound<'py, PyAny> = family?; - if family_accepts(&family, column_name)? { - matching.push(family); - } - } - PyTuple::new(families.py(), matching) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; - module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_effective_collection_rendering, - module - )?)?; - module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; - module.add_function(wrap_pyfunction!( - scenario_local_type_overrides_for_dialect, - module - )?)?; - module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; - module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs index bfe3c29867..942f305cf2 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs @@ -1,20 +1,43 @@ //! `build_effective_settings`: setting values by field name, local overrides applied. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyString; +use crate::contracts::_helpers::effective_config::Key; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, PyStringMethods, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_config::effective::main::effective_settings::effective_settings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_settings`: setting values by field name, local overrides applied. -#[inline] +#[pyfunction] pub fn effective_config_settings<'py>( py: Python<'py>, settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), overrides: &Bound<'py, PyAny>, names: Vec>, ) -> PyResult> { - effective_config::effective_config_settings(py, settings, overrides, names) + let read = |settings: &Bound<'py, PyAny>| -> PyResult> { + names + .iter() + .map(|name| { + Ok(( + key(name.as_any())?, + native_field(settings, &name.to_cow()?)?, + )) + }) + .collect() + }; + let overridden: Vec> = overrides + .try_iter()? + .map(|name| key(&name?)) + .collect::>()?; + let (project_settings, local_settings) = settings; + let settings: Pairs<'py> = effective_settings( + read(&project_settings)?, + &read(&local_settings)?, + &overridden, + ); + dictionary(py, settings) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs index f1cad2a5b6..a9df435341 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs @@ -1,19 +1,30 @@ //! `build_effective_vars` before template expansion. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_config::effective::main::effective_vars::effective_vars; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_vars` before template expansion. -#[inline] +#[pyfunction] pub fn effective_config_vars<'py>( py: Python<'py>, configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, cli_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::effective_config_vars(py, configs, target_config, cli_vars) + let (project_config, local_config) = configs; + let project: Pairs<'py> = pairs(&native_field(&project_config, "vars")?)?; + let target: Pairs<'py> = if target_config.is_none() { + Vec::new() + } else { + pairs(&native_field(target_config, "vars")?)? + }; + let local: Pairs<'py> = pairs(&native_field(&local_config, "vars")?)?; + let cli: Pairs<'py> = pairs(cli_vars)?; + dictionary(py, effective_vars(&[&project, &target, &local, &cli])) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs index 24e86bbc01..715778081b 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs @@ -1,25 +1,70 @@ //! Catalog selection uses the same named-layer resolver, with no writer layers. -use crate::contracts::_helpers::effective_config; +use crate::contracts::_helpers::effective_config::Key; use crate::contracts::_helpers::effective_config::Outcome; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::Value; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::message; +use crate::contracts::_helpers::effective_config::named; +use crate::contracts::_helpers::effective_config::optional_key; +use crate::contracts::_helpers::effective_config::pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_config::effective::main::effective_connection::effective_connection; +use sqlbuild_config::effective::models::ConnectionLayers; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Catalog selection uses the same named-layer resolver, with no writer layers. -#[inline] pub fn effective_connection_fields<'py>( py: Python<'py>, (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, catalog: bool, ) -> PyResult>> { - effective_config::effective_connection_fields( - py, - (project_config, local_config), - target_config, - catalog, - ) + let catalog_name = if catalog && !target_config.is_none() { + optional_key(&native_field(target_config, "catalog_connection")?)? + } else { + None + }; + let independent = catalog_name.is_some(); + let target: Option<(Option>, Pairs<'py>)> = if independent { + Some((catalog_name, Vec::new())) + } else if target_config.is_none() { + None + } else { + Some(( + optional_key(&native_field(target_config, "connection_name")?)?, + pairs(&native_field(target_config, "connection")?)?, + )) + }; + let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { + project: if independent { + Vec::new() + } else { + pairs(&native_field(project_config, "connection")?)? + }, + target, + project_named: named(&native_field(project_config, "connections")?)?, + local_named: named(&native_field(local_config, "connections")?)?, + local: if independent { + Vec::new() + } else { + pairs(&native_field(local_config, "connection")?)? + }, + }; + Ok(match effective_connection(layers) { + Ok(connection) => (Some(dictionary(py, connection)?), None), + Err(error) => { + let message = message(py, &error)?; + ( + None, + Some(if independent { + PyString::new(py, &format!("catalog_connection: {message}")) + } else { + message + }), + ) + } + }) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs index 77dd1720fb..29cd7607b7 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs @@ -1,18 +1,17 @@ //! Parse a one-, two-, or three-part loader destination. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::main::loader_parts::loader_parts; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; /// Parse a one-, two-, or three-part loader destination. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, destination, default_database, default_schema))] pub fn loader_destination_parts<'py>( py: Python<'py>, destination: &str, default_database: Option>, default_schema: Option>, ) -> PyResult> { - spec_resolution::loader_destination_parts(py, destination, default_database, default_schema) + loader_parts(py, destination, (default_database, default_schema)) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs index b710005f65..52d16022d7 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs @@ -1,17 +1,55 @@ //! Loader parts. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::spec_resolution::LOADER_QUALIFIED_TABLE_PARTS; +use crate::contracts::_helpers::spec_resolution::LOADER_SCHEMA_TABLE_PARTS; +use crate::contracts::_helpers::spec_resolution::blank; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// Loader parts. -#[inline] pub fn loader_parts<'py>( py: Python<'py>, destination: &str, (default_database, default_schema): (Option>, Option>), ) -> PyResult> { - spec_resolution::loader_parts(py, destination, (default_database, default_schema)) + let parts: Vec<&str> = destination.split('.').collect(); + if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) + || parts.iter().any(|part| blank(part)) + { + return Err(spec_error( + py, + PyString::new( + py, + &format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ), + ), + )?); + } + let none = || py.None().into_bound(py); + let text = |part: &str| PyString::new(py, part).into_any(); + let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + match parts.as_slice() { + [table] => ( + default_database.unwrap_or_else(none), + default_schema.unwrap_or_else(none), + text(table), + ), + [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( + default_database.unwrap_or_else(none), + text(schema), + text(table), + ), + _ => (text(parts[0]), text(parts[1]), text(parts[2])), + }; + new_record( + py, + RecordId::LoaderDestinationParts, + (database, schema, table), + None, + ) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs index 22ee8aafaf..93dc5b6819 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs @@ -1,16 +1,23 @@ //! Return declared dynamic families accepting a physical column name. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::family_accepts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// Return declared dynamic families accepting a physical column name. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, families, column_name))] pub fn matching_dynamic_families<'py>( families: &Bound<'py, PyAny>, column_name: &str, ) -> PyResult> { - spec_resolution::matching_dynamic_families(families, column_name) + let mut matching: Vec> = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + if family_accepts(&family, column_name)? { + matching.push(family); + } + } + PyTuple::new(families.py(), matching) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs index 489cc16675..b0b7119ceb 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs @@ -1,12 +1,19 @@ //! Register the effective config functions and classes. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::effective_config::effective_config_connection; +use crate::contracts::_helpers::effective_config::effective_config_target_fields; +use crate::contracts::_helpers::effective_config::effective_config_target_name; +use crate::contracts::main::effective_config_settings::effective_config_settings; +use crate::contracts::main::effective_config_vars::effective_config_vars; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the effective config functions and classes. -#[inline] pub fn register_effective_config(module: &Bound<'_, PyModule>) -> PyResult<()> { - effective_config::register(module) + module.add_function(wrap_pyfunction!(effective_config_target_name, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_target_fields, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_vars, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_connection, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_settings, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs index f866a7667f..18e9f742cf 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs @@ -1,12 +1,31 @@ //! Register the spec resolution functions and classes. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::scenario_local_type_overrides_for_dialect; +use crate::contracts::main::loader_destination_parts::loader_destination_parts; +use crate::contracts::main::matching_dynamic_families::matching_dynamic_families; +use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use crate::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; +use crate::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the spec resolution functions and classes. -#[inline] pub fn register_spec_resolution(module: &Bound<'_, PyModule>) -> PyResult<()> { - spec_resolution::register(module) + module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; + module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_collection_rendering, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; + module.add_function(wrap_pyfunction!( + scenario_local_type_overrides_for_dialect, + module + )?)?; + module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; + module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs index 5da624dbdf..6888e0382c 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs @@ -1,15 +1,19 @@ //! Resolve the effective adapter name, allowing local override. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Resolve the effective adapter name, allowing local override. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn resolve_effective_adapter_name<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_adapter_name(project_config, local_config) + let local: Bound<'py, PyAny> = native_field(local_config, "adapter")?; + if local.is_none() { + return native_field(project_config, "adapter"); + } + Ok(local) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs index 9c29dfa1aa..ddff31f880 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs @@ -1,15 +1,18 @@ //! Resolve declaration rendering over project configuration. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Resolve declaration rendering over project configuration. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, declaration_override))] pub fn resolve_effective_collection_rendering<'py>( project_config: &Bound<'py, PyAny>, declaration_override: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_collection_rendering(project_config, declaration_override) + if !declaration_override.is_none() { + return Ok(declaration_override.clone()); + } + native_field(project_config, "constants")?.getattr("collection_rendering") } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs index edce6d7b7b..0ea5527f76 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs @@ -1,15 +1,71 @@ //! Resolve scenario config, allowing local overrides to replace project rules. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::SNAPSHOT_LIMITS; +use crate::contracts::_helpers::spec_resolution::override_or; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Resolve scenario config, allowing local overrides to replace project rules. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn resolve_effective_scenario_config<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_scenario_config(project_config, local_config) + let py: Python<'py> = project_config.py(); + let project: Bound<'py, PyAny> = native_field(project_config, "scenario")?; + let local: Bound<'py, PyAny> = native_field(local_config, "scenario")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + for item in native_field(&project, "local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; + } + for item in native_field(&local, "local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides + .call_method1("setdefault", (dialect, PyDict::new(py)))? + .call_method1("update", (rules,))?; + } + let project_limits: Bound<'py, PyAny> = native_field(&project, "snapshot_limits")?; + let local_limits: Bound<'py, PyAny> = native_field(&local, "snapshot_limits")?; + let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS + .iter() + .map(|field| { + Ok(( + *field, + override_or( + native_field(&local_limits, field)?, + native_field(&project_limits, field)?, + ), + )) + }) + .collect::>()?; + new_record_with( + py, + RecordId::ScenarioConfig, + &[ + ( + "run_namespace", + override_or( + native_field(&local, "run_namespace")?, + native_field(&project, "run_namespace")?, + ), + ), + ("local_type_overrides", overrides.into_any()), + ( + "snapshot_limits", + new_record_with(py, RecordId::ScenarioSnapshotLimitsConfig, &limits)?, + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs index 1b48b4863a..dc176befff 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs @@ -1,18 +1,51 @@ //! Merge project target config with local developer overrides. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::effective_config_target_fields; +use crate::contracts::_helpers::spec_resolution::TARGET_GROUPS; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// Merge project target config with local developer overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_name))] pub fn resolve_target_config<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_name: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_target_config(py, project_config, local_config, target_name) + let (rows, error) = effective_config_target_fields( + py, + (project_config.clone(), local_config.clone()), + target_name, + &new_record(py, RecordId::TargetConfig, (), None)?, + )?; + let Some(rows) = rows else { + let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); + return Err(spec_error(py, message)?); + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS + .iter() + .map(|(group, _)| (*group, PyDict::new(py))) + .collect(); + for (group, name, value) in rows { + match group { + None => values.set_item(name, value)?, + Some(group) => { + if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { + fields.set_item(name, value)?; + } + } + } + } + for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { + let nested: Bound<'py, PyAny> = new_record(py, *class_name, (), Some(fields))?; + values.set_item(*group, nested)?; + } + new_record(py, RecordId::TargetConfig, (), Some(&values)) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs index ea408772b6..aa3d143d22 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs @@ -1,19 +1,24 @@ //! Resolve the effective target name. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::effective_config_target_name; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyString; /// Resolve the effective target name. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, selected_target))] pub fn resolve_target_name<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, selected_target: &Bound<'py, PyAny>, ) -> PyResult>> { - spec_resolution::resolve_target_name(py, project_config, local_config, selected_target) + let (name, error) = + effective_config_target_name(py, project_config, local_config, selected_target)?; + match error { + Some(message) => Err(spec_error(py, message)?), + None => Ok(name), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs deleted file mode 100644 index 3e3dd1995e..0000000000 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! One call's cursor values read from and built as the native cursor value records. - -use sqlbuild_python_base::native_host::main::host_error::host_error; - -use pyo3::prelude::{PyErr, Python}; -use sqlbuild_host::entries::models::FacadeHost; - -/// `CursorAlgebraError(message)`. -pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - host_error(py, FacadeHost::CursorAlgebraError, message) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs index 93150ef70a..a2994c6e93 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs @@ -1,50 +1,13 @@ //! Calendar durations parsed from text and applied to dates and timestamps. -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyInt, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; -use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; -use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan, Moment}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_cursor::algebra::models::DurationSpan; -use crate::cursor::_helpers::values::civil_date; -use crate::cursor::_helpers::values::local_date_time; -use crate::cursor::_helpers::values::replaced; -use crate::cursor::constants::DATETIME_MODULE; -use crate::cursor::models::CursorSession; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MONTHS_PER_YEAR: i64 = 12; -/// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -#[pyfunction] -#[pyo3(signature = (value))] -pub(crate) fn parse_cursor_duration<'py>( - py: Python<'py>, - value: &str, -) -> PyResult>> { - let Some(digits): Option = parse_duration(value) else { - return Ok(None); - }; - let int = py.get_type::(); - let parts: Vec> = [ - digits.years, - digits.months, - digits.days, - digits.hours, - digits.minutes, - digits.seconds, - ] - .iter() - .map(|text| int.call1((text.as_str(),))) - .collect::>()?; - PyTuple::new(py, parts).map(Some) -} - -fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { let field = |name: &str| -> PyResult { native_field(duration, name)?.extract() }; Ok(DurationSpan { months: field("years")? * MONTHS_PER_YEAR + field("months")?, @@ -54,51 +17,3 @@ fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { seconds: field("seconds")?, }) } - -/// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -pub(crate) fn shifted_moment<'py>( - moment: &Bound<'py, PyAny>, - span: &DurationSpan, - direction: i64, -) -> PyResult> { - let py: Python<'py> = moment.py(); - let session: CursorSession<'py> = CursorSession::new(py)?; - let is_timestamp: bool = moment.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)?; - let typed: Moment = if is_timestamp { - Moment::Timestamp { - local: local_date_time(moment)?, - fold: native_field(moment, "fold")?.extract::()? != 0, - } - } else { - Moment::Date(civil_date(moment)?) - }; - let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; - match shifted { - Moment::Timestamp { local, fold } => replaced(moment, &local, fold), - Moment::Date(date) => call_with( - &native_field(moment, "replace")?, - &[ - ("year", date.year.into_pyobject(py)?.into_any()), - ("month", date.month.into_pyobject(py)?.into_any()), - ("day", date.day.into_pyobject(py)?.into_any()), - ], - ), - } -} - -/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[pyfunction] -#[pyo3(signature = (*, moment, duration, direction))] -pub(crate) fn shift_by_cursor_duration<'py>( - moment: &Bound<'py, PyAny>, - duration: &Bound<'py, PyAny>, - direction: i64, -) -> PyResult> { - shifted_moment(moment, &duration_span(duration)?, direction) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; - module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs index a63c908d17..a709a0380b 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs @@ -1,24 +1,20 @@ //! Building, intersecting, merging, splitting and capping half-open cursor intervals. +use crate::cursor::main::scalars::as_scalar::as_scalar; use std::cmp::Ordering; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PySlice, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::cap_from_end::cap_from_end; use sqlbuild_cursor::algebra::main::cap_from_start::cap_from_start; use sqlbuild_cursor::algebra::main::clamp::clamp; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::merge::merge; -use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; use sqlbuild_cursor::algebra::main::split_into_batches::split_into_batches; -use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; use sqlbuild_cursor::algebra::models::{Interval, Scalar}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::cursor::_helpers::scalars::as_scalar; use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -44,40 +40,10 @@ fn python_intervals<'py>( PyTuple::new(session.py(), built) } -/// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -#[pyfunction(name = "validate_cursor_interval")] -#[pyo3(signature = (*, start, end, grain))] -pub(crate) fn interval_validation( - start: &Bound<'_, PyAny>, - end: &Bound<'_, PyAny>, - grain: &Bound<'_, PyAny>, -) -> PyResult<()> { - let session: CursorSession<'_> = CursorSession::new(start.py())?; - let interval = Interval { - start: session.scalar(start)?, - end: session.scalar(end)?, - grain: session.grain(grain)?, - }; - session.checked(validate_interval(&interval, &session)) -} - -/// The canonical partition containing an observed physical value. -#[pyfunction(name = "cursor_observed_partition")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn partition<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - let session: CursorSession<'py> = CursorSession::new(value.py())?; - let scalar: Scalar = session.scalar(value)?; - let interval: Interval = session.checked(observed_partition(&scalar, session.grain(grain)?))?; - session.python_interval(&interval) -} - /// The intersection of two compatible half-open intervals, or `None`. #[pyfunction(name = "cursor_clamp")] #[pyo3(signature = (*, interval, bounds))] -fn clamp_intervals<'py>( +pub(crate) fn clamp_intervals<'py>( interval: &Bound<'py, PyAny>, bounds: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -93,7 +59,7 @@ fn clamp_intervals<'py>( /// Overlapping or adjacent canonical intervals merged. #[pyfunction(name = "cursor_merge")] #[pyo3(signature = (*, intervals))] -fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult> { let session: CursorSession<'py> = CursorSession::new(intervals.py())?; let typed: Vec = intervals .try_iter()? @@ -106,7 +72,7 @@ fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn merge_raw_intervals<'py>( intervals: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -167,7 +133,7 @@ fn merge_raw_intervals<'py>( /// An interval split into batches of a positive number of grain units. #[pyfunction(name = "cursor_split_into_batches")] #[pyo3(signature = (*, interval, step))] -fn split<'py>(interval: &Bound<'py, PyAny>, step: i64) -> PyResult> { +pub(crate) fn split<'py>(interval: &Bound<'py, PyAny>, step: i64) -> PyResult> { let session: CursorSession<'py> = CursorSession::new(interval.py())?; let typed: Interval = session.interval(interval)?; let batches: Vec = session.checked(split_into_batches(&typed, step, &session))?; @@ -185,28 +151,22 @@ fn sliced<'py>( /// At most `count` intervals from the start. #[pyfunction(name = "cursor_cap_from_start")] #[pyo3(signature = (*, intervals, count))] -fn cap_start<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { +pub(crate) fn cap_start<'py>( + intervals: &Bound<'py, PyAny>, + count: i64, +) -> PyResult> { sliced(intervals, cap_from_start(intervals.len()?, count)) } /// At most `count` intervals from the end; an empty tuple when `count` is not positive. #[pyfunction(name = "cursor_cap_from_end")] #[pyo3(signature = (*, intervals, count))] -fn cap_end<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { +pub(crate) fn cap_end<'py>( + intervals: &Bound<'py, PyAny>, + count: i64, +) -> PyResult> { if count <= 0 { return Ok(PyTuple::empty(intervals.py()).into_any()); } sliced(intervals, cap_from_end(intervals.len()?, count)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(interval_validation, module)?)?; - module.add_function(wrap_pyfunction!(partition, module)?)?; - module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; - module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; - module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; - module.add_function(wrap_pyfunction!(split, module)?)?; - module.add_function(wrap_pyfunction!(cap_start, module)?)?; - module.add_function(wrap_pyfunction!(cap_end, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs index fffa7c18fe..c5e240fb0a 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod conversion; pub(crate) mod durations; pub(crate) mod intervals; pub(crate) mod overrides; diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs index 05e7930c02..55b10209da 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs @@ -1,62 +1,14 @@ //! Executor cursor bounds from `--start-cursor`/`--end-cursor` overrides. -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyInt, PyString, PyStringMethods}; -use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use crate::cursor::_helpers::conversion::algebra_error; -use crate::cursor::constants::DATETIME_MODULE; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -const DECIMAL_MODULE: &str = "decimal"; -const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; -const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; +pub(crate) const DECIMAL_MODULE: &str = "decimal"; +pub(crate) const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; +pub(crate) const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; -fn instance_of(value: &Bound<'_, PyAny>, class: RecordId) -> PyResult { +pub(crate) fn instance_of(value: &Bound<'_, PyAny>, class: RecordId) -> PyResult { value.is_instance(&record_class(value.py(), class)?) } - -/// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -pub(crate) fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - if value.is_none() { - return Ok(value.clone()); - } - if instance_of(value, RecordId::TimestampValue)? { - return native_field(value, "value"); - } - let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - if instance_of(value, RecordId::DateValue)? { - let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; - return datetime.call_method1("combine", (native_field(value, "value")?, midnight)); - } - if instance_of(value, RecordId::IntegerValue)? { - return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); - } - datetime.call_method1("fromisoformat", (value,)) -} - -/// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -pub(crate) fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - if value.is_none() { - return Ok(value.clone()); - } - if instance_of(value, RecordId::IntegerValue)? { - return native_field(value, "value"); - } - if instance_of(value, RecordId::DateValue)? || instance_of(value, RecordId::TimestampValue)? { - return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); - } - if let Ok(text) = value.downcast::() - && let Some(parsed) = parse_integer_text(text.to_str()?) - { - return Ok(parsed.into_pyobject(py)?.into_any()); - } - let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; - py.get_type::().call1((decimal,)) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs index a7cdb7c2f3..7db875203d 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs @@ -1,31 +1,25 @@ //! Parsing, rendering, comparing and stepping typed cursor values. -use std::cmp::Ordering; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use crate::cursor::main::scalars::as_scalar::as_scalar; use pyo3::IntoPyObject; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyString; use pyo3::types::PyStringMethods; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::errors::CursorError; -use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::cursor_sort_key::cursor_sort_key as sort_key; use sqlbuild_cursor::algebra::main::exclusive_end_from_observed_max::exclusive_end_from_observed_max; -use sqlbuild_cursor::algebra::main::exclusive_to_inclusive::exclusive_to_inclusive; use sqlbuild_cursor::algebra::main::extreme_position::extreme_position; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; -use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; use sqlbuild_cursor::algebra::main::normalize_timestamp_text::normalize_timestamp_text; use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; -use sqlbuild_cursor::algebra::main::render::render; use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, SortKey}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::cursor::_helpers::conversion::algebra_error; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::imported::imported; @@ -35,8 +29,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const DECIMAL_MODULE: &str = "decimal"; -const INTEGER_CURSOR: &str = "integer"; -const TIMESTAMP_CURSOR: &str = "timestamp"; +pub(crate) const INTEGER_CURSOR: &str = "integer"; +pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; /// `CursorAlgebraError(message)` raised from `cause`. fn chained(py: Python<'_>, message: String, cause: PyErr) -> PyErr { @@ -46,7 +40,7 @@ fn chained(py: Python<'_>, message: String, cause: PyErr) -> PyErr { } /// `IntegerValue` parsed from `str(raw)` the way `int(Decimal(str(raw)))` reads it. -fn parse_integer<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn parse_integer<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = raw.py(); let text: String = raw.str()?.to_str()?.to_owned(); if let Some(value) = parse_integer_text(&text) { @@ -93,7 +87,7 @@ fn integer_failure(py: Python<'_>, error: &PyErr) -> PyResult { } /// `value.astimezone(UTC)` for an aware `datetime`, else `value`. -fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> { if native_field(value, "tzinfo")?.is_none() { return Ok(value.clone()); } @@ -104,7 +98,10 @@ fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> } /// A `DateValue` or `TimestampValue` parsed from warehouse timestamp text. -fn parse_timestamp_text<'py>(raw: &Bound<'py, PyAny>, text: &str) -> PyResult> { +pub(crate) fn parse_timestamp_text<'py>( + raw: &Bound<'py, PyAny>, + text: &str, +) -> PyResult> { let py: Python<'py> = raw.py(); let normalized: String = normalize_timestamp_text(text); let as_date = imported(py, DATETIME_MODULE, "date")? @@ -137,102 +134,12 @@ fn parse_timestamp_text<'py>(raw: &Bound<'py, PyAny>, text: &str) -> PyResult( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = raw.py(); - if cursor_type.eq(INTEGER_CURSOR)? { - return parse_integer(raw); - } - if !cursor_type.eq(TIMESTAMP_CURSOR)? { - return Err(algebra_error( - py, - &format!("unsupported cursor type: {}", cursor_type.str()?), - )); - } - if raw.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)? { - let value: Bound<'py, PyAny> = utc_normalized(raw)?; - return new_record_with(py, RecordId::TimestampValue, &[("value", value)]); - } - if raw.is_instance(&imported(py, DATETIME_MODULE, "date")?)? { - return new_record_with(py, RecordId::DateValue, &[("value", raw.clone())]); - } - let Ok(text) = raw.downcast::() else { - return Err(algebra_error( - py, - &format!("invalid timestamp cursor value: {}", raw.str()?), - )); - }; - parse_timestamp_text(raw, text.to_str()?) -} - -/// `raw` unchanged when it is already a typed value, else parsed. -pub(crate) fn as_scalar<'py>( - session: &CursorSession<'py>, - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult { - if session.scalar_typed(raw)? { - return session.scalar(raw); - } - session.scalar(&parse_value(raw, cursor_type)?) -} - -/// One cursor scalar, preserving date-ness and timestamp offsets. -#[pyfunction(name = "cursor_parse")] -#[pyo3(signature = (*, raw, cursor_type))] -pub(crate) fn parse<'py>( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult> { - parse_value(raw, cursor_type) -} - -/// Discovery values that may be dates or timestamps, or `None` when they do not parse. -#[pyfunction(name = "cursor_try_parse")] -#[pyo3(signature = (*, raw, cursor_type))] -pub(crate) fn try_parse<'py>( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = raw.py(); - match parse_value(raw, cursor_type) { - Ok(value) => Ok(Some(value)), - Err(error) - if error.is_instance_of::(py) - || error.is_instance_of::(py) => - { - Ok(None) - } - Err(error) => Err(error), - } -} - -/// A scalar in the existing cursor output format. -#[pyfunction(name = "cursor_render")] -#[pyo3(signature = (*, value))] -pub(crate) fn render_value(value: &Bound<'_, PyAny>) -> PyResult { - let session: CursorSession<'_> = CursorSession::new(value.py())?; - let scalar: Scalar = session.scalar(value)?; - session.checked(render(&scalar, &session)) -} - -/// A sentinel's token, or a scalar rendered. -#[pyfunction(name = "cursor_sentinel_to_token")] -#[pyo3(signature = (*, sentinel))] -pub(crate) fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { - if sentinel.is_instance(&host_entry(sentinel.py(), ExecutorHost::BoundSentinel)?)? { - return native_field(sentinel, "value")?.extract(); - } - render_value(sentinel) -} - /// The sentinel a SQL token names, if it names one. #[pyfunction(name = "cursor_sentinel_from_token")] #[pyo3(signature = (*, token))] -fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn sentinel_from_token<'py>( + token: &Bound<'py, PyAny>, +) -> PyResult>> { let py: Python<'py> = token.py(); match host_entry(py, ExecutorHost::BoundSentinel)?.call1((token,)) { Ok(sentinel) => Ok(Some(sentinel)), @@ -241,19 +148,6 @@ fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult, right: &Bound<'_, PyAny>) -> PyResult { - let session: CursorSession<'_> = CursorSession::new(left.py())?; - let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); - Ok(match session.checked(compare(&left, &right, &session))? { - Ordering::Less => -1, - Ordering::Equal => 0, - Ordering::Greater => 1, - }) -} - fn extreme_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -280,7 +174,7 @@ fn extreme_bound<'py>( /// The numerically or temporally largest raw bound. #[pyfunction(name = "cursor_max_bound")] #[pyo3(signature = (*, values, cursor_type))] -fn max_bound<'py>( +pub(crate) fn max_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -290,7 +184,7 @@ fn max_bound<'py>( /// The numerically or temporally smallest raw bound. #[pyfunction(name = "cursor_min_bound")] #[pyo3(signature = (*, values, cursor_type))] -fn min_bound<'py>( +pub(crate) fn min_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -298,7 +192,7 @@ fn min_bound<'py>( } /// A scalar stepped by one grain-dependent operation. -fn stepped<'py>( +pub(crate) fn stepped<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, step: impl FnOnce(&Scalar, Option) -> Result, @@ -312,7 +206,7 @@ fn stepped<'py>( /// The aligned boundary at or before a cursor value. #[pyfunction(name = "cursor_floor_to_grain")] #[pyo3(signature = (*, value, grain))] -fn floor_value<'py>( +pub(crate) fn floor_value<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { @@ -324,37 +218,17 @@ fn floor_value<'py>( /// The boundary one grain after an aligned cursor value. #[pyfunction(name = "cursor_next_boundary")] #[pyo3(signature = (*, value, grain))] -fn next_value<'py>( +pub(crate) fn next_value<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { stepped(value, grain, next_boundary) } -/// An operator-declared inclusive bound advanced by one cursor unit. -#[pyfunction(name = "cursor_inclusive_to_exclusive")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn to_exclusive<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - stepped(value, grain, inclusive_to_exclusive) -} - -/// An exclusive bound moved back by one cursor unit. -#[pyfunction(name = "cursor_exclusive_to_inclusive")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn to_inclusive<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - stepped(value, grain, exclusive_to_inclusive) -} - /// The exclusive end of the partition holding an observed maximum. #[pyfunction(name = "cursor_exclusive_end_from_observed_max")] #[pyo3(signature = (*, value, grain))] -fn observed_end<'py>( +pub(crate) fn observed_end<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { @@ -367,20 +241,3 @@ fn observed_end<'py>( ))?; session.python_scalar(&end) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse, module)?)?; - module.add_function(wrap_pyfunction!(try_parse, module)?)?; - module.add_function(wrap_pyfunction!(render_value, module)?)?; - module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; - module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; - module.add_function(wrap_pyfunction!(compare_values, module)?)?; - module.add_function(wrap_pyfunction!(max_bound, module)?)?; - module.add_function(wrap_pyfunction!(min_bound, module)?)?; - module.add_function(wrap_pyfunction!(floor_value, module)?)?; - module.add_function(wrap_pyfunction!(next_value, module)?)?; - module.add_function(wrap_pyfunction!(to_exclusive, module)?)?; - module.add_function(wrap_pyfunction!(to_inclusive, module)?)?; - module.add_function(wrap_pyfunction!(observed_end, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs index de30f9f798..52b8214bb3 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs @@ -1,11 +1,10 @@ //! Python `date`, `datetime` and `timedelta` objects read into and built from cursor values. use crate::cursor::constants::DATETIME_MODULE; -use pyo3::IntoPyObject; +use crate::cursor::main::values::replaced::replaced; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -21,17 +20,6 @@ pub(crate) fn civil_date(value: &Bound<'_, PyAny>) -> PyResult { }) } -/// The wall-clock fields of a `datetime`. -pub(crate) fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { - Ok(LocalDateTime { - date: civil_date(value)?, - hour: native_field(value, "hour")?.extract()?, - minute: native_field(value, "minute")?.extract()?, - second: native_field(value, "second")?.extract()?, - microsecond: native_field(value, "microsecond")?.extract()?, - }) -} - /// A `timedelta` as whole microseconds. pub(crate) fn timedelta_micros(delta: &Bound<'_, PyAny>) -> PyResult { let days: i64 = native_field(delta, "days")?.extract()?; @@ -40,31 +28,6 @@ pub(crate) fn timedelta_micros(delta: &Bound<'_, PyAny>) -> PyResult { Ok((days * SECONDS_PER_DAY + seconds) * MICROS_PER_SECOND + microseconds) } -/// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -pub(crate) fn replaced<'py>( - template: &Bound<'py, PyAny>, - local: &LocalDateTime, - fold: bool, -) -> PyResult> { - let py: Python<'py> = template.py(); - call_with( - &native_field(template, "replace")?, - &[ - ("year", local.date.year.into_pyobject(py)?.into_any()), - ("month", local.date.month.into_pyobject(py)?.into_any()), - ("day", local.date.day.into_pyobject(py)?.into_any()), - ("hour", local.hour.into_pyobject(py)?.into_any()), - ("minute", local.minute.into_pyobject(py)?.into_any()), - ("second", local.second.into_pyobject(py)?.into_any()), - ( - "microsecond", - local.microsecond.into_pyobject(py)?.into_any(), - ), - ("fold", u8::from(fold).into_pyobject(py)?.into_any()), - ], - ) -} - /// A naive `datetime` holding `local`. pub(crate) fn naive_date_time<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs index 67da83f9c4..58efc324ca 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs @@ -1,11 +1,10 @@ //! `CursorAlgebraError(message)`. -use crate::cursor::_helpers::conversion; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; /// `CursorAlgebraError(message)`. -#[inline] pub fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - conversion::algebra_error(py, message) + host_error(py, FacadeHost::CursorAlgebraError, message) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs index 932140f66b..eac5496ad9 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs @@ -1,16 +1,32 @@ //! `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyInt, PyTuple}; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::DurationDigits; /// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -#[inline] +#[pyfunction] +#[pyo3(signature = (value))] pub fn parse_cursor_duration<'py>( py: Python<'py>, value: &str, ) -> PyResult>> { - durations::parse_cursor_duration(py, value) + let Some(digits): Option = parse_duration(value) else { + return Ok(None); + }; + let int = py.get_type::(); + let parts: Vec> = [ + digits.years, + digits.months, + digits.days, + digits.hours, + digits.minutes, + digits.seconds, + ] + .iter() + .map(|text| int.call1((text.as_str(),))) + .collect::>()?; + PyTuple::new(py, parts).map(Some) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs index 77d1ebdb29..516c11fa7b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs @@ -1,12 +1,13 @@ //! Register the durations functions and classes. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::main::durations::parse_cursor_duration::parse_cursor_duration; +use crate::cursor::main::durations::shift_by_cursor_duration::shift_by_cursor_duration; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the durations functions and classes. -#[inline] pub fn register_durations(module: &Bound<'_, PyModule>) -> PyResult<()> { - durations::register(module) + module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; + module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs index 9037da5d56..9065b91a10 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs @@ -1,16 +1,17 @@ //! `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::durations::duration_span; +use crate::cursor::main::durations::shifted_moment::shifted_moment; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, moment, duration, direction))] pub fn shift_by_cursor_duration<'py>( moment: &Bound<'py, PyAny>, duration: &Bound<'py, PyAny>, direction: i64, ) -> PyResult> { - durations::shift_by_cursor_duration(moment, duration, direction) + shifted_moment(moment, &duration_span(duration)?, direction) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs index c92de35127..68f8cfec0d 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs @@ -1,17 +1,45 @@ //! `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_cursor::algebra::models::DurationSpan; +use crate::cursor::_helpers::values::civil_date; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::cursor::main::values::replaced::replaced; +use crate::cursor::models::CursorSession; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; +use sqlbuild_cursor::algebra::models::{DurationSpan, Moment}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -#[inline] pub fn shifted_moment<'py>( moment: &Bound<'py, PyAny>, span: &DurationSpan, direction: i64, ) -> PyResult> { - durations::shifted_moment(moment, span, direction) + let py: Python<'py> = moment.py(); + let session: CursorSession<'py> = CursorSession::new(py)?; + let is_timestamp: bool = moment.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)?; + let typed: Moment = if is_timestamp { + Moment::Timestamp { + local: local_date_time(moment)?, + fold: native_field(moment, "fold")?.extract::()? != 0, + } + } else { + Moment::Date(civil_date(moment)?) + }; + let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; + match shifted { + Moment::Timestamp { local, fold } => replaced(moment, &local, fold), + Moment::Date(date) => call_with( + &native_field(moment, "replace")?, + &[ + ("year", date.year.into_pyobject(py)?.into_any()), + ("month", date.month.into_pyobject(py)?.into_any()), + ("day", date.day.into_pyobject(py)?.into_any()), + ], + ), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs index 549343ce56..f7fbc613cd 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs @@ -1,16 +1,24 @@ //! Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; +use sqlbuild_cursor::algebra::models::Interval; /// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -#[inline] +#[pyfunction(name = "validate_cursor_interval")] +#[pyo3(signature = (*, start, end, grain))] pub fn interval_validation( start: &Bound<'_, PyAny>, end: &Bound<'_, PyAny>, grain: &Bound<'_, PyAny>, ) -> PyResult<()> { - intervals::interval_validation(start, end, grain) + let session: CursorSession<'_> = CursorSession::new(start.py())?; + let interval = Interval { + start: session.scalar(start)?, + end: session.scalar(end)?, + grain: session.grain(grain)?, + }; + session.checked(validate_interval(&interval, &session)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs index 7509cc126b..1c7393d999 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs @@ -1,15 +1,20 @@ //! The canonical partition containing an observed physical value. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; +use sqlbuild_cursor::algebra::models::{Interval, Scalar}; /// The canonical partition containing an observed physical value. -#[inline] +#[pyfunction(name = "cursor_observed_partition")] +#[pyo3(signature = (*, value, grain))] pub fn partition<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - intervals::partition(value, grain) + let session: CursorSession<'py> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + let interval: Interval = session.checked(observed_partition(&scalar, session.grain(grain)?))?; + session.python_interval(&interval) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs index 33e94ed6ad..9e4a224eb8 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs @@ -1,12 +1,25 @@ //! Register the intervals functions and classes. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::intervals::cap_end; +use crate::cursor::_helpers::intervals::cap_start; +use crate::cursor::_helpers::intervals::clamp_intervals; +use crate::cursor::_helpers::intervals::merge_intervals; +use crate::cursor::_helpers::intervals::merge_raw_intervals; +use crate::cursor::_helpers::intervals::split; +use crate::cursor::main::intervals::interval_validation::interval_validation; +use crate::cursor::main::intervals::partition::partition; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the intervals functions and classes. -#[inline] pub fn register_intervals(module: &Bound<'_, PyModule>) -> PyResult<()> { - intervals::register(module) + module.add_function(wrap_pyfunction!(interval_validation, module)?)?; + module.add_function(wrap_pyfunction!(partition, module)?)?; + module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; + module.add_function(wrap_pyfunction!(split, module)?)?; + module.add_function(wrap_pyfunction!(cap_start, module)?)?; + module.add_function(wrap_pyfunction!(cap_end, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs index a7fc01d88c..bc24492456 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs @@ -1,12 +1,34 @@ //! An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -use crate::cursor::_helpers::overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::overrides::DECIMAL_MODULE; +use crate::cursor::_helpers::overrides::TIMESTAMP_AS_INTEGER; +use crate::cursor::_helpers::overrides::instance_of; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyString, PyStringMethods}; +use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -#[inline] pub fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - overrides::cursor_integer(value) + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, RecordId::IntegerValue)? { + return native_field(value, "value"); + } + if instance_of(value, RecordId::DateValue)? || instance_of(value, RecordId::TimestampValue)? { + return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); + } + if let Ok(text) = value.downcast::() + && let Some(parsed) = parse_integer_text(text.to_str()?) + { + return Ok(parsed.into_pyobject(py)?.into_any()); + } + let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; + py.get_type::().call1((decimal,)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs index 193365f411..ab1a9e892a 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs @@ -1,12 +1,30 @@ //! A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -use crate::cursor::_helpers::overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::overrides::INTEGER_AS_TIMESTAMP; +use crate::cursor::_helpers::overrides::instance_of; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -#[inline] pub fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - overrides::cursor_timestamp(value) + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, RecordId::TimestampValue)? { + return native_field(value, "value"); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + if instance_of(value, RecordId::DateValue)? { + let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; + return datetime.call_method1("combine", (native_field(value, "value")?, midnight)); + } + if instance_of(value, RecordId::IntegerValue)? { + return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); + } + datetime.call_method1("fromisoformat", (value,)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs index 56617ed205..987d827623 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs @@ -1,18 +1,18 @@ //! `raw` unchanged when it is already a typed value, else parsed. -use crate::cursor::_helpers::scalars; +use crate::cursor::main::scalars::parse_value::parse_value; use crate::cursor::models::CursorSession; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_cursor::algebra::models::Scalar; /// `raw` unchanged when it is already a typed value, else parsed. -#[inline] pub fn as_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult { - scalars::as_scalar(session, raw, cursor_type) + if session.scalar_typed(raw)? { + return session.scalar(raw); + } + session.scalar(&parse_value(raw, cursor_type)?) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs index d8a1465d67..94c5f12964 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs @@ -1,12 +1,21 @@ //! `-1`, `0` or `1` as `left` orders against `right`. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::models::Scalar; +use std::cmp::Ordering; /// `-1`, `0` or `1` as `left` orders against `right`. -#[inline] +#[pyfunction(name = "cursor_compare")] +#[pyo3(signature = (*, left, right))] pub fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { - scalars::compare_values(left, right) + let session: CursorSession<'_> = CursorSession::new(left.py())?; + let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); + Ok(match session.checked(compare(&left, &right, &session))? { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + }) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs index b9d0b5af14..9c83a032c5 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs @@ -1,15 +1,15 @@ //! One cursor scalar, preserving date-ness and timestamp offsets. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::parse_value::parse_value; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// One cursor scalar, preserving date-ness and timestamp offsets. -#[inline] +#[pyfunction(name = "cursor_parse")] +#[pyo3(signature = (*, raw, cursor_type))] pub fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::parse(raw, cursor_type) + parse_value(raw, cursor_type) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs index 198f84f246..b0f2dbc60b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs @@ -1,15 +1,46 @@ //! One cursor value parsed from its boundary representation, keeping date-ness and offsets. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::INTEGER_CURSOR; +use crate::cursor::_helpers::scalars::TIMESTAMP_CURSOR; +use crate::cursor::_helpers::scalars::parse_integer; +use crate::cursor::_helpers::scalars::parse_timestamp_text; +use crate::cursor::_helpers::scalars::utc_normalized; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// One cursor value parsed from its boundary representation, keeping date-ness and offsets. -#[inline] pub fn parse_value<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::parse_value(raw, cursor_type) + let py: Python<'py> = raw.py(); + if cursor_type.eq(INTEGER_CURSOR)? { + return parse_integer(raw); + } + if !cursor_type.eq(TIMESTAMP_CURSOR)? { + return Err(algebra_error( + py, + &format!("unsupported cursor type: {}", cursor_type.str()?), + )); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)? { + let value: Bound<'py, PyAny> = utc_normalized(raw)?; + return new_record_with(py, RecordId::TimestampValue, &[("value", value)]); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "date")?)? { + return new_record_with(py, RecordId::DateValue, &[("value", raw.clone())]); + } + let Ok(text) = raw.downcast::() else { + return Err(algebra_error( + py, + &format!("invalid timestamp cursor value: {}", raw.str()?), + )); + }; + parse_timestamp_text(raw, text.to_str()?) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs index 5d54f46a04..e4b6eca099 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs @@ -1,12 +1,35 @@ //! Register the scalars functions and classes. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::floor_value; +use crate::cursor::_helpers::scalars::max_bound; +use crate::cursor::_helpers::scalars::min_bound; +use crate::cursor::_helpers::scalars::next_value; +use crate::cursor::_helpers::scalars::observed_end; +use crate::cursor::_helpers::scalars::sentinel_from_token; +use crate::cursor::main::scalars::compare_values::compare_values; +use crate::cursor::main::scalars::parse::parse; +use crate::cursor::main::scalars::render_value::render_value; +use crate::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use crate::cursor::main::scalars::to_exclusive::to_exclusive; +use crate::cursor::main::scalars::to_inclusive::to_inclusive; +use crate::cursor::main::scalars::try_parse::try_parse; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scalars functions and classes. -#[inline] pub fn register_scalars(module: &Bound<'_, PyModule>) -> PyResult<()> { - scalars::register(module) + module.add_function(wrap_pyfunction!(parse, module)?)?; + module.add_function(wrap_pyfunction!(try_parse, module)?)?; + module.add_function(wrap_pyfunction!(render_value, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; + module.add_function(wrap_pyfunction!(compare_values, module)?)?; + module.add_function(wrap_pyfunction!(max_bound, module)?)?; + module.add_function(wrap_pyfunction!(min_bound, module)?)?; + module.add_function(wrap_pyfunction!(floor_value, module)?)?; + module.add_function(wrap_pyfunction!(next_value, module)?)?; + module.add_function(wrap_pyfunction!(to_exclusive, module)?)?; + module.add_function(wrap_pyfunction!(to_inclusive, module)?)?; + module.add_function(wrap_pyfunction!(observed_end, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs index cc7c06fb71..69063d3b68 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs @@ -1,12 +1,16 @@ //! A scalar in the existing cursor output format. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_cursor::algebra::models::Scalar; /// A scalar in the existing cursor output format. -#[inline] +#[pyfunction(name = "cursor_render")] +#[pyo3(signature = (*, value))] pub fn render_value(value: &Bound<'_, PyAny>) -> PyResult { - scalars::render_value(value) + let session: CursorSession<'_> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + session.checked(render(&scalar, &session)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs index fd2abf4730..a6d10c2234 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs @@ -1,12 +1,18 @@ //! A sentinel's token, or a scalar rendered. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::render_value::render_value; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A sentinel's token, or a scalar rendered. -#[inline] +#[pyfunction(name = "cursor_sentinel_to_token")] +#[pyo3(signature = (*, sentinel))] pub fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { - scalars::sentinel_to_token(sentinel) + if sentinel.is_instance(&host_entry(sentinel.py(), ExecutorHost::BoundSentinel)?)? { + return native_field(sentinel, "value")?.extract(); + } + render_value(sentinel) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs index 3f71c47689..45793135f2 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs @@ -1,15 +1,16 @@ //! An operator-declared inclusive bound advanced by one cursor unit. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::stepped; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; /// An operator-declared inclusive bound advanced by one cursor unit. -#[inline] +#[pyfunction(name = "cursor_inclusive_to_exclusive")] +#[pyo3(signature = (*, value, grain))] pub fn to_exclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::to_exclusive(value, grain) + stepped(value, grain, inclusive_to_exclusive) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs index 49da0d7d19..766d74929b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs @@ -1,15 +1,16 @@ //! An exclusive bound moved back by one cursor unit. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::stepped; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::exclusive_to_inclusive::exclusive_to_inclusive; /// An exclusive bound moved back by one cursor unit. -#[inline] +#[pyfunction(name = "cursor_exclusive_to_inclusive")] +#[pyo3(signature = (*, value, grain))] pub fn to_inclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::to_inclusive(value, grain) + stepped(value, grain, exclusive_to_inclusive) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs index aac8b2cfb1..dcf88432f4 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs @@ -1,15 +1,26 @@ //! Discovery values that may be dates or timestamps, or `None` when they do not parse. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::parse_value::parse_value; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; /// Discovery values that may be dates or timestamps, or `None` when they do not parse. -#[inline] +#[pyfunction(name = "cursor_try_parse")] +#[pyo3(signature = (*, raw, cursor_type))] pub fn try_parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult>> { - scalars::try_parse(raw, cursor_type) + let py: Python<'py> = raw.py(); + match parse_value(raw, cursor_type) { + Ok(value) => Ok(Some(value)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + Ok(None) + } + Err(error) => Err(error), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs index 4a4809c85d..c0dde89480 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs @@ -1,13 +1,17 @@ //! The wall-clock fields of a `datetime`. -use crate::cursor::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::values::civil_date; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The wall-clock fields of a `datetime`. -#[inline] pub fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { - values::local_date_time(value) + Ok(LocalDateTime { + date: civil_date(value)?, + hour: native_field(value, "hour")?.extract()?, + minute: native_field(value, "minute")?.extract()?, + second: native_field(value, "second")?.extract()?, + microsecond: native_field(value, "microsecond")?.extract()?, + }) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs index c6d6e8c946..b910208924 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs @@ -1,17 +1,32 @@ //! `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -use crate::cursor::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -#[inline] pub fn replaced<'py>( template: &Bound<'py, PyAny>, local: &LocalDateTime, fold: bool, ) -> PyResult> { - values::replaced(template, local, fold) + let py: Python<'py> = template.py(); + call_with( + &native_field(template, "replace")?, + &[ + ("year", local.date.year.into_pyobject(py)?.into_any()), + ("month", local.date.month.into_pyobject(py)?.into_any()), + ("day", local.date.day.into_pyobject(py)?.into_any()), + ("hour", local.hour.into_pyobject(py)?.into_any()), + ("minute", local.minute.into_pyobject(py)?.into_any()), + ("second", local.second.into_pyobject(py)?.into_any()), + ( + "microsecond", + local.microsecond.into_pyobject(py)?.into_any(), + ), + ("fold", u8::from(fold).into_pyobject(py)?.into_any()), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/cursor/models.rs b/crates/sqlbuild-python-front/src/cursor/models.rs index 7a7a17b513..d93e6b7856 100644 --- a/crates/sqlbuild-python-front/src/cursor/models.rs +++ b/crates/sqlbuild-python-front/src/cursor/models.rs @@ -1,14 +1,14 @@ //! Public records of cursor. -use crate::cursor::_helpers::conversion::algebra_error; use crate::cursor::_helpers::values::civil_date; -use crate::cursor::_helpers::values::local_date_time; use crate::cursor::_helpers::values::naive_date_time; use crate::cursor::_helpers::values::plain_date; -use crate::cursor::_helpers::values::replaced; use crate::cursor::_helpers::values::timedelta_micros; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::constants::GRAIN_NAMES; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::cursor::main::values::replaced::replaced; use pyo3::IntoPyObject; use pyo3::exceptions::PyOverflowError; use pyo3::exceptions::PyTypeError; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs index 7948942caa..1e14af603e 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs @@ -1,61 +1,26 @@ //! Setting guidance shown by diagnostics that depend on project settings. +use crate::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; -use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; -use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyBoolMethods, PyInt, PyString}; use sqlbuild_config::project::main::snippet_help_text::snippet_help_text; -use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; -use sqlbuild_config::project::models::{SettingSite, SettingText}; +use sqlbuild_config::project::models::SettingText; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -const SQL_ANALYSIS_KEY: &str = "sql_analysis"; - -/// The note and help for turning SQL analysis on, or `None` when it is effectively on. -pub(crate) fn sql_analysis_guidance( - discovered_inputs: &Bound<'_, PyAny>, - purpose: &str, -) -> PyResult> { - let local: Bound<'_, PyAny> = native_field(discovered_inputs, "local_config")?; - let local_override: bool = - native_field(&local, "setting_overrides")?.contains(SQL_ANALYSIS_KEY)?; - let owner: Bound<'_, PyAny> = if local_override { - local - } else { - native_field(discovered_inputs, "project_config")? - }; - if native_field(&native_field(&owner, "settings")?, SQL_ANALYSIS_KEY)?.is_truthy()? { - return Ok(None); - } - Ok(Some(analysis_off(local_override, purpose))) -} +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; #[pyfunction] #[pyo3(signature = (*, discovered_inputs, purpose))] -fn sql_analysis_off_guidance( +pub(crate) fn sql_analysis_off_guidance( discovered_inputs: &Bound<'_, PyAny>, purpose: &str, ) -> PyResult> { sql_analysis_guidance(discovered_inputs, purpose) } -#[pyfunction] -pub(crate) fn microbatch_guidance() -> (String, String) { - concurrency_guidance() -} - -#[pyfunction] -#[pyo3(signature = (*, allowed))] -pub(crate) fn explicit_references_help(allowed: &str) -> String { - references_help(allowed) -} - /// A Python `bool | int | str` setting value as help text shows it. -fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(flag) = value.cast::() { return Ok(SettingText::Bool(flag.is_true())); } @@ -70,45 +35,9 @@ fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { )) } -/// Render `, set this in :` followed by the exact TOML section and key. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn setting_help<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "setting_help", - options, - (&["purpose", "file_name", "section", "key", "value"], &[]), - )?; - let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; - let (file_name, section, key): (String, String, String) = - (text("file_name")?, text("section")?, text("key")?); - Ok(setting_help_text( - &text("purpose")?, - SettingSite { - file_name: &file_name, - section: §ion, - key: &key, - }, - &setting_text(&arguments.value("value")?)?, - )) -} - /// Render `, :` followed by the exact lines to write, one per line. #[pyfunction] #[pyo3(signature = (*, purpose, target, lines))] -fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { +pub(crate) fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { snippet_help_text(purpose, target, &lines) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(setting_help, module)?)?; - module.add_function(wrap_pyfunction!(snippet_help, module)?)?; - module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; - module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; - module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs index 51e2e48d0c..7f03682b3f 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs @@ -1,11 +1,9 @@ //! Native discovery failures as the Python exceptions discovery raises for them. use pyo3::exceptions::{PyKeyError, PyRuntimeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::models::{DiscoveryFailure, ReadFailure, StageFailure}; use sqlbuild_discovery::tree::models::ProjectTree; @@ -135,7 +133,7 @@ pub(crate) fn relative_read_error( /// The error `read_error` builds for an operating-system read failure, for the platform oracle. #[pyfunction] #[pyo3(signature = (errno, winerror, message, *, file_path))] -fn _oracle_read_error<'py>( +pub(crate) fn _oracle_read_error<'py>( errno: Option, winerror: Option, message: String, @@ -151,7 +149,3 @@ fn _oracle_read_error<'py>( .into_bound(file_path.py()) .into_any() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs index 4fe5562215..f9c864a4cb 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs @@ -1,10 +1,8 @@ //! Model-header enums and constants, and typed constant file entries, validated natively. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_model_config::declarations::errors::DeclarationFailure; use sqlbuild_model_config::declarations::main::model_constant_declarations::model_constant_declarations; use sqlbuild_model_config::declarations::main::model_enum_declarations::model_enum_declarations; @@ -190,7 +188,3 @@ pub(crate) fn typed_constant_record<'py>( Err(failure) => Err(declaration_error(py, failure)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_header_declarations, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs index d3418c38d6..22bc1ac8d7 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs @@ -1,9 +1,9 @@ //! MODEL-style header values with a bounded parse cache, and authored column locations. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use crate::sqltext::main::header_failure::header_failure; use crate::sqltext::main::model_output_columns::model_output_columns; @@ -30,7 +30,7 @@ fn header_cache(py: Python<'_>) -> Bound<'_, PyDict> { /// Parse every header the cache misses in one native batch. #[pyfunction] -fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = headers.py(); let cache: Bound<'_, PyDict> = header_cache(py); let missing: Bound<'_, PyDict> = PyDict::new(py); @@ -57,7 +57,9 @@ fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { /// `parse_header_values(*, header, file_path, statement_name, header_line, error_class=None)`. #[pyfunction] #[pyo3(signature = (**request))] -fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn parse_header_values<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("parse_header_values", request)?; let py: Python<'py> = request.py(); let header: Bound<'py, PyAny> = request.get("header")?; @@ -131,7 +133,7 @@ fn source_locations<'py>( /// `model_output_column_locations(*, contents, relative_path, extract_implicit_alias_columns=True)`. #[pyfunction] #[pyo3(signature = (**request))] -fn model_output_column_locations<'py>( +pub(crate) fn model_output_column_locations<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("model_output_column_locations", request)?; @@ -143,9 +145,3 @@ fn model_output_column_locations<'py>( let rows: Vec = model_output_columns(&contents, implicit)?; source_locations(rows, &request.get("relative_path")?) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(prepare_model_header_tokens, module)?)?; - module.add_function(wrap_pyfunction!(parse_header_values, module)?)?; - module.add_function(wrap_pyfunction!(model_output_column_locations, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs index 6b775040f7..556f8e94da 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs @@ -1,12 +1,11 @@ //! Model lifecycle hooks as the plain dictionaries manifests and identities record. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::headers::hook_entries::hook_entries; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; const RELATIVE_PATH_FIELD: &str = "relative_path"; const SQL_HOOK_FIELDS: [&str; 6] = [ @@ -18,7 +17,10 @@ const SQL_HOOK_FIELDS: [&str; 6] = [ "description", ]; -fn sql_hook<'py>(entry: &Bound<'py, PyAny>, sql_fields: &[String]) -> PyResult> { +pub(crate) fn sql_hook<'py>( + entry: &Bound<'py, PyAny>, + sql_fields: &[String], +) -> PyResult> { let hook: Bound<'py, PyDict> = PyDict::new(entry.py()); hook.set_item("type", "sql")?; for field in sql_fields { @@ -37,51 +39,12 @@ fn sql_hook<'py>(entry: &Bound<'py, PyAny>, sql_fields: &[String]) -> PyResult( - value: &Bound<'py, PyAny>, - sql_fields: &[String], - python_hook_fields: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let py = value.py(); - let hooks: Bound<'py, PyList> = PyList::empty(py); - if !value.is_instance_of::() && !value.is_instance_of::() { - return Ok(hooks); - } - let sql_class: Bound<'py, PyAny> = record_class(py, RecordId::SqlHookEntry)?; - let python_class: Bound<'py, PyAny> = record_class(py, RecordId::PythonHookEntry)?; - for entry in value.try_iter()? { - let entry: Bound<'py, PyAny> = entry?; - if entry.is_instance(&sql_class)? { - hooks.append(sql_hook(&entry, sql_fields)?)?; - } else if entry.is_instance(&python_class)? { - let hook: Bound<'py, PyDict> = PyDict::new(py); - let name: Bound<'py, PyAny> = native_field(&entry, "name")?; - hook.set_item("type", "python")?; - hook.set_item("name", &name)?; - hook.set_item("kwargs", native_field(&entry, "kwargs")?)?; - if let Some(fields) = python_hook_fields - && let Some(extra) = fields.get_item(&name)? - { - hook.call_method1("update", (extra,))?; - } - hooks.append(hook)?; - } - } - Ok(hooks) -} - #[pyfunction] #[pyo3(signature = (*, value, sql_fields, python_hook_fields = None))] -fn serialize_hook_entries<'py>( +pub(crate) fn serialize_hook_entries<'py>( value: &Bound<'py, PyAny>, sql_fields: Vec, python_hook_fields: Option>, ) -> PyResult> { hook_entries(value, &sql_fields, python_hook_fields.as_ref()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(serialize_hook_entries, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs index a19f1a9e74..bc7b62f748 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs @@ -1,13 +1,12 @@ //! Unknown and unsupported authored keys, rejected with nearest-key suggestions natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::pyfunction; use sqlbuild_discovery::key_checks::main::unknown_mapping_keys::unknown_mapping_keys as unknown_keys; use sqlbuild_discovery::key_checks::main::unsupported_keys_help::unsupported_keys_help as keys_help; /// `(message after the label, help)` when a mapping key is outside `allowed`. #[pyfunction] -fn unknown_mapping_keys( +pub(crate) fn unknown_mapping_keys( keys: Vec<(String, bool)>, allowed: Vec, ) -> Option<(String, String)> { @@ -16,12 +15,7 @@ fn unknown_mapping_keys( /// `unsupported_keys_help` for `keys` outside `supported_keys`. #[pyfunction] -fn unsupported_keys_help(keys: Vec, supported_keys: Vec) -> String { +pub(crate) fn unsupported_keys_help(keys: Vec, supported_keys: Vec) -> String { let keys: Vec<&str> = keys.iter().map(String::as_str).collect(); keys_help(&keys, &supported_keys) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(unknown_mapping_keys, module)?)?; - module.add_function(wrap_pyfunction!(unsupported_keys_help, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs index 5e720d022e..7bd7806e38 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs @@ -1,71 +1,19 @@ //! Project inputs holding only the project and local configuration. +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_host::entries::models::ExecutorHost; -use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; -use crate::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; use crate::compile_driver::models::ProjectConfiguration; -use crate::discovery::_helpers::project::project_config::load_local_config_file; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::project::project_config::warn_deprecated_settings; -use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; -use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -/// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -pub(crate) fn discover_project_configuration<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - configuration_inputs(py, project_dir, |_| Ok(())) -} - -/// The validated configuration without a discovery lifecycle, which reading two files never needs. -pub(crate) fn discover_connection_configuration<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - loaded_configuration(py, project_dir, reject_invalid_connection_configuration) -} - -/// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -pub(crate) fn discover_configured_providers<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, - secret_policy: SecretPolicy, -) -> PyResult> { - let configuration: Bound<'py, PyAny> = loaded_configuration(py, project_dir, |config| { - reject_invalid_project_configuration(config)?; - warn_deprecated_settings(py, project_dir) - })?; - let providers: Bound<'py, PyAny> = - discover_provider_classes(project_dir.clone(), secret_policy.name())?; - new_record_with( - py, - RecordId::DiscoveredProjectInputs, - &[ - ("project_dir", project_dir.clone()), - ( - "project_config", - native_field(&configuration, "project_config")?, - ), - ( - "local_config", - native_field(&configuration, "local_config")?, - ), - ("providers", providers), - ], - ) -} - -fn configuration_inputs<'py>( +pub(crate) fn configuration_inputs<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>, @@ -86,7 +34,7 @@ fn configuration_inputs<'py>( } /// Both configuration files, after `checks` accepts them. -fn loaded_configuration<'py>( +pub(crate) fn loaded_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs index 8998d17785..faa1fea759 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs @@ -1,20 +1,10 @@ //! Authored locations Python consumers read: YAML entry lines and project-relative paths. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// The 1-based line of the outermost `- name: ` entry, if present. -#[pyfunction] -#[pyo3(signature = (*, contents, name))] -pub(crate) fn yaml_entry_line(contents: &str, name: &str) -> Option { - yaml_entry_lines(contents) - .into_iter() - .find_map(|(entry, line)| (entry == name).then_some(line)) -} - /// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] #[pyo3(signature = (*, path, project_dir))] @@ -37,9 +27,3 @@ pub(crate) fn project_relative_path<'py>( path_parts.get_slice(root_parts.len(), path_parts.len()), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; - module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs index 270be60e8e..2aaacb4b95 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs @@ -2,28 +2,15 @@ use std::path::PathBuf; +use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyOSError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; -use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings; -use sqlbuild_config::project::main::load_changes_config::load_changes_config; -use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; -use sqlbuild_config::project::main::load_local_config::load_local_config; -use sqlbuild_config::project::main::load_project_config::load_project_config; -use sqlbuild_config::project::main::load_review_config::load_review_config; -use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; -use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; -use sqlbuild_config::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ReviewConfig, -}; +use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; -use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -41,12 +28,12 @@ const PYTHON_ENUM_MODULES: [(&str, &str); 1] = [("AuditSeverity", "sqlbuild.compiler.auditing.types")]; /// The four configuration file paths of a project directory, as Python and native paths. -struct Files<'py> { - python: Vec>, - native: ConfigPaths, +pub(crate) struct Files<'py> { + pub(crate) python: Vec>, + pub(crate) native: ConfigPaths, } -fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { let python = FILE_NAMES .iter() .map(|name| project_dir.call_method1("__truediv__", (*name,))) @@ -69,7 +56,7 @@ fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { }) } -fn file_index(file: ConfigFile) -> usize { +pub(crate) fn file_index(file: ConfigFile) -> usize { match file { ConfigFile::Project => 0, ConfigFile::LegacyProject => 1, @@ -78,7 +65,11 @@ fn file_index(file: ConfigFile) -> usize { } } -fn config_error(py: Python<'_>, failure: &ConfigFailure, files: &Files<'_>) -> PyResult { +pub(crate) fn config_error( + py: Python<'_>, + failure: &ConfigFailure, + files: &Files<'_>, +) -> PyResult { use sqlbuild_config::project::errors::MessagePiece; let pieces = failure .message @@ -110,7 +101,11 @@ fn os_error(py: Python<'_>, error: &std::io::Error, path: &Bound<'_, PyAny>) -> Ok(PyErr::from_value(value)) } -fn load_error(py: Python<'_>, failure: &LoadFailure, files: &Files<'_>) -> PyResult { +pub(crate) fn load_error( + py: Python<'_>, + failure: &LoadFailure, + files: &Files<'_>, +) -> PyResult { match failure { LoadFailure::Config(failure) => config_error(py, failure, files), LoadFailure::Io(file, error) => os_error(py, error, &files.python[file_index(*file)]), @@ -142,7 +137,7 @@ fn object<'py>( new_record(py, record_named(class)?, (), Some(&keywords)) } -fn built(py: Python<'_>, value: Built) -> PyResult> { +pub(crate) fn built(py: Python<'_>, value: Built) -> PyResult> { match value { Built::Value(value) => config_value_to_python(py, value), Built::Object(class, fields) => Ok(object(py, class, fields)?.unbind()), @@ -168,101 +163,11 @@ fn built(py: Python<'_>, value: Built) -> PyResult> { } } -/// Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -pub(crate) fn warn_deprecated_settings( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult<()> { - let files = files(project_dir)?; - match deprecated_setting_warnings(&files.native) { - Ok(warnings) => warnings - .iter() - .try_for_each(|warning| write_console(py, "stderr", &format!("{warning}\n"))), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// `load_project_config`: the validated `ProjectConfig` of `project_dir`. -#[pyfunction] -pub(crate) fn load_project_config_file( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult> { - let files = files(project_dir)?; - match load_project_config(&files.native) { - Ok(config) => built(py, config), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -pub(crate) fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { - configured(project_dir, load_git_base_ref) -} - -/// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -pub(crate) fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { - configured(project_dir, load_review_config) -} - -/// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -pub(crate) fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { - configured(project_dir, load_changes_config) -} - /// One configuration section of the project at `project_dir`, with load failures raised as Python's. -fn configured( +pub(crate) fn configured( project_dir: &Bound<'_, PyAny>, load: fn(&ConfigPaths) -> Result, ) -> PyResult { let files: Files<'_> = files(project_dir)?; load(&files.native).or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) } - -/// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -#[pyfunction] -pub(crate) fn load_local_config_file( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult> { - let files = files(project_dir)?; - match load_local_config(&files.native) { - Ok(config) => built(py, config), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// The project or local configuration file path, rejecting an unconverted legacy file. -#[pyfunction] -pub(crate) fn resolve_config_file_path<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, - local: bool, -) -> PyResult> { - let files = files(project_dir)?; - match resolve_config_file(&files.native, local) { - Ok(file) => Ok(files.python[file_index(file)].clone()), - Err(failure) => Err(config_error(py, &failure, &files)?), - } -} - -/// `validate_local_sql_analysis_policy` from the project requirement and the local setting. -#[pyfunction] -pub(crate) fn validate_local_sql_analysis( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, - flags: (bool, bool, bool), -) -> Result<(), PyErr> { - let files = files(project_dir)?; - match local_sql_analysis_failure(&files.native, flags) { - None => Ok(()), - Some(failure) => Err(config_error(py, &failure, &files)?), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_project_config_file, module)?)?; - module.add_function(wrap_pyfunction!(load_local_config_file, module)?)?; - module.add_function(wrap_pyfunction!(resolve_config_file_path, module)?)?; - module.add_function(wrap_pyfunction!(validate_local_sql_analysis, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs index 003e2a881e..1fdf94746a 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs @@ -1,26 +1,26 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::discovery::models::ModelFileTexts; -use crate::discovery::models::NativeModelFile; +use crate::discovery::main::project::column_name::column_name; +use crate::discovery::main::project::seed_csv_settings::seed_csv_settings; +use crate::discovery::main::project::stem::stem; use crate::discovery::models::NodeRow; +use crate::discovery::models::{ModelFileTexts, NativeModelFile}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTypeMethods}; use sqlbuild_discovery::project_validation::errors::CsvDialectError; use sqlbuild_discovery::project_validation::models::{ AuditFileFact, CsvSettings, DependencyFact, IntegrationLoaderFact, ModelFact, NamedPath, - NodeFact, NodeKind, ProjectFacts, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, - SeedFileFact, SourceFact, SourceFileFact, TestBlockFact, TestFileFact, + NodeFact, NodeKind, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, SeedFileFact, + SourceFact, SourceFileFact, TestBlockFact, TestFileFact, }; -use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::path_stem::path_stem; const CSV_MODULE: &str = "csv"; /// `DeclarationKind.AUDIT`: a generic audit file. const GENERIC_AUDIT_KIND: &str = "audit"; /// Each Python node kind and the discovered inputs field that lists it, in discovery order. -const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ +pub(crate) const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ ("loader", "loader_functions"), ("task", "task_functions"), ("asset", "asset_functions"), @@ -32,65 +32,6 @@ const PRIVATE_SCOPE: &str = "private"; /// `(SQL resource name, owning node index, decorated name)` of one dependency. pub(crate) type DependencyRow = (Option, Option, Option); -/// The facts of `inputs`, with the Python nodes the facade has described. -pub(crate) fn project_facts( - py: Python<'_>, - inputs: &Bound<'_, PyAny>, - nodes: Vec, -) -> PyResult { - let mut declaration_files: Vec = - each(inputs, "enum_files", declarations_fact)?; - declaration_files.extend(each(inputs, "constant_files", declarations_fact)?); - let mut model_schemas: Vec = Vec::new(); - for schema in native_field(inputs, "model_schema_files")?.try_iter()? { - model_schemas.extend(each(&schema?, "declarations", named)?); - } - Ok(ProjectFacts { - models: each(inputs, "model_files", model_fact)?, - sql_functions: each(inputs, "sql_function_files", stem_named)?, - python_functions: each(inputs, "python_function_files", stem_named)?, - sql_hooks: each(inputs, "sql_hook_files", named)?, - seeds: each(inputs, "seed_files", seed_file_fact)?, - schemas: each(inputs, "schema_files", |schema| schema_fact(py, schema))?, - sources: each(inputs, "source_files", source_file_fact)?, - tests: each(inputs, "test_files", test_file_fact)?, - scenarios: each(inputs, "scenario_files", named)?, - audits: each(inputs, "audit_files", audit_fact)?, - declaration_files, - model_schemas, - materializations: each(inputs, "materialization_files", named)?, - nodes: nodes.into_iter().map(node_fact).collect::>()?, - hook_functions: each(inputs, "hook_functions", named)?, - event_exporters: each(inputs, "event_exporters", named)?, - providers: each(inputs, "providers", named)?, - path_default_keys: native_field(inputs, "project_config")? - .getattr("path_defaults")? - .try_iter()? - .map(|key| key?.extract()) - .collect::>()?, - project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), - csv_field_limit: Some(csv_field_limit(py)?), - orchestration: Vec::new(), - }) -} - -/// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -pub(crate) fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { - let mut rows: Vec = Vec::new(); - for (kind, name) in PYTHON_NODE_FIELDS { - rows.extend(each(inputs, name, |node| { - Ok(NodeRow { - kind: kind.to_owned(), - name: native_field(node, "name")?.extract()?, - path: text(node, "relative_path")?, - decorator_config: false, - depends_on: Vec::new(), - }) - })?); - } - Ok(rows) -} - /// `read` applied to each item of the iterable attribute `name` of `owner`. pub(crate) fn each( owner: &Bound<'_, PyAny>, @@ -104,21 +45,16 @@ pub(crate) fn each( Ok(items) } -fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { +pub(crate) fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { Ok(native_field(owner, name)?.str()?.to_string()) } -fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult> { +pub(crate) fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult> { native_field(owner, name)?.extract() } -/// `str(Path(...).stem)` of the path attribute `name`. -pub(crate) fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { - path_stem(&native_field(owner, name)?) -} - /// `(resource.name, str(resource.relative_path))`. -fn named(resource: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn named(resource: &Bound<'_, PyAny>) -> PyResult { Ok(NamedPath { name: native_field(resource, "name")?.extract()?, path: text(resource, "relative_path")?, @@ -126,7 +62,7 @@ fn named(resource: &Bound<'_, PyAny>) -> PyResult { } /// `(file.file_path.stem, str(file.relative_path))`; a native model file answers from its texts. -fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { if let Ok(model_file) = file.cast::() { let texts: ModelFileTexts = model_file.get().path_texts(file.py())?; return Ok(NamedPath { @@ -152,7 +88,7 @@ fn enum_value(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { native_field(&native_field(owner, name)?, "value")?.extract() } -fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { let mut local_declarations: Vec = each(model, "enum_declarations", named)?; local_declarations.extend(each(model, "constant_declarations", named)?); Ok(ModelFact { @@ -161,30 +97,13 @@ fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { }) } -fn seed_file_fact(seed: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn seed_file_fact(seed: &Bound<'_, PyAny>) -> PyResult { Ok(SeedFileFact { file: stem_named(seed)?, absolute_path: text(seed, "file_path")?, }) } -pub(crate) fn column_name(column: &Bound<'_, PyAny>) -> PyResult { - native_field(column, "name")?.extract() -} - -/// The CSV reader settings a seed declaration authors, without the dialect check. -pub(crate) fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { - let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; - Ok(CsvSettings { - delimiter: optional_text(&settings, "delimiter")?, - quotechar: optional_text(&settings, "quotechar")?, - escapechar: optional_text(&settings, "escapechar")?, - doublequote: native_field(&settings, "doublequote")?.extract()?, - skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, - dialect_error: None, - }) -} - fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { let mut csv: CsvSettings = seed_csv_settings(entry)?; csv.dialect_error = csv_dialect_error(py, &csv)?; @@ -195,7 +114,7 @@ fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult, schema: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn schema_fact(py: Python<'_>, schema: &Bound<'_, PyAny>) -> PyResult { Ok(SchemaFileFact { path: text(schema, "relative_path")?, model_names: each(schema, "model_entries", column_name)?, @@ -220,7 +139,7 @@ fn source_fact(entry: &Bound<'_, PyAny>) -> PyResult { }) } -fn source_file_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn source_file_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(SourceFileFact { path: text(file, "relative_path")?, entries: each(file, "source_entries", source_fact)?, @@ -234,7 +153,7 @@ fn test_block_fact(block: &Bound<'_, PyAny>) -> PyResult { }) } -fn test_file_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn test_file_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(TestFileFact { file: relative_stem_named(file)?, blocks: each(file, "blocks", test_block_fact)?, @@ -245,7 +164,7 @@ fn block_name(block: &Bound<'_, PyAny>) -> PyResult> { optional_text(block, "name") } -fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(AuditFileFact { file: relative_stem_named(file)?, generic: enum_value(file, "declaration_kind")? == GENERIC_AUDIT_KIND, @@ -253,7 +172,7 @@ fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { }) } -fn declarations_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declarations_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(ScopedDeclarationsFact { private: enum_value(file, "scope_kind")? == PRIVATE_SCOPE, declarations: each(file, "declarations", named)?, @@ -282,7 +201,7 @@ fn dependency_fact((sql_name, owner, decorated_name): DependencyRow) -> Dependen } } -fn node_fact(node: NodeRow) -> PyResult { +pub(crate) fn node_fact(node: NodeRow) -> PyResult { Ok(NodeFact { kind: node_kind(&node.kind)?, name: node.name, @@ -321,7 +240,7 @@ fn csv_dialect_error(py: Python<'_>, csv: &CsvSettings) -> PyResult) -> PyResult { +pub(crate) fn csv_field_limit(py: Python<'_>) -> PyResult { py.import(CSV_MODULE)? .call_method0("field_size_limit")? .extract() diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs index c60e8febc9..5d5e9b12ec 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs @@ -1,11 +1,9 @@ //! Native discovery results as the plain data the Python discovery facade materialises. use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, pymethods}; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::tree::models::ProjectTree; @@ -14,12 +12,10 @@ use std::path::Path; use crate::discovery::_helpers::failures::raised::failure_error; use crate::discovery::_helpers::headers::header_declarations::model_header_declarations; use crate::discovery::_helpers::records::declaration_records::source_locations; -use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; -use crate::discovery::_helpers::records::model_file_record::ModelFileParts; -use crate::discovery::_helpers::records::model_file_record::PendingModelFile; -use crate::discovery::_helpers::snapshots::native_classes::ModelTextRequest; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::models::NativeModelFile; +use crate::discovery::_helpers::records::model_file_record::{ + DiscoveredPaths, ModelFileParts, PendingModelFile, +}; +use crate::discovery::_helpers::snapshots::native_classes::{ModelTextRequest, NativeProjectTree}; use crate::sqltext::main::map_to_python::map_to_python; use crate::sqltext::main::optional_authored_value_to_python::optional_authored_value_to_python; use sqlbuild_python_base::types::CompilerDetach; @@ -96,7 +92,7 @@ pub(crate) fn model_record( /// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. #[pyfunction] -fn parse_model_contents( +pub(crate) fn parse_model_contents( py: Python<'_>, request: ModelTextRequest, contents: String, @@ -124,11 +120,3 @@ fn parse_model_contents( .unbind(), )) } - -/// Whether native discovery reproduces the string semantics of this Python and Unicode version. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index 9086c6bdb8..7b412f439f 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -1,27 +1,23 @@ //! Cross-file validation of one discovery's inputs. -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::project::project_facts::project_facts; +use crate::discovery::main::project::validation_error::validation_error; +use crate::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; -use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; +use sqlbuild_discovery::project_validation::models::ProjectFacts; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::project::project_facts::project_facts; -use crate::discovery::_helpers::sources::orchestration_files::orchestration_declarations; use crate::discovery::models::NodeRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const CSV_MODULE: &str = "csv"; -const BUILTINS_MODULE: &str = "builtins"; -const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const CSV_MODULE: &str = "csv"; +pub(crate) const BUILTINS_MODULE: &str = "builtins"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; /// The exceptions a CSV dialect may raise, by class name. -const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; +pub(crate) const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; -fn raised(result: PyResult>) -> PyErr { +pub(crate) fn raised(result: PyResult>) -> PyErr { match result { Ok(error) => PyErr::from_value(error), Err(error) => error, @@ -30,7 +26,7 @@ fn raised(result: PyResult>) -> PyErr { /// Raise the first cross-file conflict across the discovered inputs, as Python raises it. #[pyfunction] -fn validate_discovered_project_inputs( +pub(crate) fn validate_discovered_project_inputs( py: Python<'_>, inputs: Bound<'_, PyAny>, nodes: Vec, @@ -42,43 +38,3 @@ fn validate_discovered_project_inputs( } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } - -/// The Python exception a failed project validation raises. -pub(crate) fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { - match failure { - ValidationFailure::Discovery(failure) => failure_error(py, failure), - ValidationFailure::Unreadable { - absolute_path, - failure, - } => match py - .import(PATHLIB_MODULE) - .and_then(|module| native_field(&module, "Path")) - .and_then(|path| path.call1((absolute_path,))) - { - Ok(file_path) => read_error(failure, &file_path), - Err(error) => error, - }, - ValidationFailure::Csv(message) => raised( - py.import(CSV_MODULE) - .and_then(|module| native_field(&module, "Error")) - .and_then(|class| class.call1((message,))), - ), - ValidationFailure::CsvDialect(error) => { - if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { - return PyKeyError::new_err(error.class.to_string()); - } - raised( - py.import(BUILTINS_MODULE) - .and_then(|module| native_field(&module, error.class.as_str())) - .and_then(|class| class.call1((error.message,))), - ) - } - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - validate_discovered_project_inputs, - module - )?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs index 7e04bf4870..0fd34a7d4c 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::types::{PyTuple, PyTupleMethods}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_host::entries::models::UserCodeHost; @@ -132,30 +132,6 @@ pub(crate) fn is_exception(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) } -/// `{provider.name: provider for provider in providers}`. -pub(crate) fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { - let by_name: Bound<'py, PyDict> = PyDict::new(providers.py()); - for provider in providers.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - by_name.set_item(native_field(&provider, "name")?, provider)?; - } - Ok(by_name) -} - -/// The provider parameters of one loaded user function. -pub(crate) fn provider_usages<'py>( - function: &Bound<'py, PyAny>, - provider_by_name: &Bound<'py, PyDict>, -) -> PyResult> { - call_with( - &shim(function.py(), UserCodeHost::ProviderUsages)?, - &[ - ("function", function.clone()), - ("provider_by_name", provider_by_name.clone().into_any()), - ], - ) -} - /// A tuple of the given items. pub(crate) fn tuple_of<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs index c592f4f592..1fea680c08 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs @@ -1,5 +1,7 @@ //! Provider classes and lifecycle-event sinks from imported providers/ and sinks/ modules. +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; use std::collections::HashSet; use pyo3::exceptions::PyRuntimeError; @@ -9,15 +11,9 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; -use crate::discovery::_helpers::python_files::imported_modules::discovered; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; -use crate::discovery::_helpers::python_files::imported_modules::module_functions; -use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; -use crate::discovery::_helpers::python_files::imported_modules::provider_usages; -use crate::discovery::_helpers::python_files::imported_modules::shim; -use crate::discovery::_helpers::python_files::imported_modules::tuple_of; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, shim, tuple_of, +}; use crate::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -45,7 +41,7 @@ const DEFAULT_ERROR_TYPE: &str = "validation_error"; const DEFAULT_SETTINGS_ERROR: &str = "invalid provider settings"; const RECALL_WRITER_UNINSTALLED: &str = "the native module registered no RecallWriter class"; -static RECALL_WRITER: PyOnceLock> = PyOnceLock::new(); +pub(crate) static RECALL_WRITER: PyOnceLock> = PyOnceLock::new(); fn reject_invalid_identity( name: &Bound<'_, PyAny>, @@ -362,11 +358,6 @@ pub(crate) fn discover_event_exporter_declarations<'py>( tuple_of(py, declarations) } -/// Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -pub(crate) fn install_recall_writer(writer: &Bound<'_, PyType>) { - let _ = RECALL_WRITER.get_or_init(writer.py(), || writer.clone().unbind()); -} - /// Each parameter of `function` with its annotation, and whether that annotation is `RecallWriter`. fn annotated_parameters<'py>( function: &Bound<'py, PyAny>, @@ -388,7 +379,7 @@ fn annotated_parameters<'py>( } /// Whether `declaration`'s function takes a `RecallWriter`, making it a Recall replacement sink. -fn takes_recall_writer(declaration: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn takes_recall_writer(declaration: &Bound<'_, PyAny>) -> PyResult { Ok( annotated_parameters(&native_field(declaration, "function")?)? .iter() @@ -469,31 +460,3 @@ pub(crate) fn bind_event_exporter_declarations<'py>( } tuple_of(py, bound) } - -/// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_event_exporter_functions<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { - path: project_dir.clone(), - }; - if let Some(cached) = project.cached_extensions()? { - return cached.get_item(1); - } - let mut ordinary: Vec> = Vec::new(); - for declaration in discover_event_exporter_declarations(project_dir.clone())?.try_iter()? { - let declaration: Bound<'py, PyAny> = declaration?; - if !takes_recall_writer(&declaration)? { - ordinary.push(declaration); - } - } - bind_event_exporter_declarations( - tuple_of(py, ordinary)?, - providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, - project_dir, - ) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index 68221f5848..c9f981204b 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -1,29 +1,19 @@ //! Decorated Python nodes, audit factories, hooks and materializations from imported modules. -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::constants::PYTHON_NODE_ROOT; use sqlbuild_host::entries::models::UserCodeHost; -use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; -use crate::discovery::_helpers::python_files::imported_modules::definition; -use crate::discovery::_helpers::python_files::imported_modules::discovered; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; -use crate::discovery::_helpers::python_files::imported_modules::is_exception; -use crate::discovery::_helpers::python_files::imported_modules::module_functions; -use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; -use crate::discovery::_helpers::python_files::imported_modules::provider_usages; -use crate::discovery::_helpers::python_files::imported_modules::shim; -use crate::discovery::_helpers::python_files::imported_modules::tuple_of; -use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; -use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; -use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_functions; -use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, definition, discovered, discovery_error, discovery_error_from, is_exception, + module_functions, shim, tuple_of, +}; use crate::discovery::_helpers::python_files::python_hook_files::discover_python_hook_files; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -113,9 +103,9 @@ const NODE_KINDS: [NodeKind; 4] = [ ]; /// Discovered loaders, tasks, assets, checks and audit factories, in discovery order. -struct Buckets<'py> { - nodes: [Vec>; 4], - audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, +pub(crate) struct Buckets<'py> { + pub(crate) nodes: [Vec>; 4], + pub(crate) audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, } /// The module and the file it was imported from. @@ -261,7 +251,7 @@ impl<'py> Buckets<'py> { Ok(()) } - fn discover_nodes( + pub(crate) fn discover_nodes( &mut self, project: &ProjectDir<'py>, providers: &Bound<'py, PyDict>, @@ -406,64 +396,6 @@ fn factory_functions<'py>( Ok(functions) } -/// Decorated Python DAG node functions under python/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_python_node_functions<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { path: project_dir }; - let providers: Bound<'py, PyDict> = match providers { - Some(providers) => provider_by_name(&providers)?, - None => PyDict::new(py), - }; - let mut buckets = Buckets { - nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], - audit_factories: Vec::new(), - }; - let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; - if node_root.call_method0("is_dir")?.is_truthy()? { - let saved: Bound<'py, PyAny> = call_with( - &shim(py, UserCodeHost::InstallPythonRootPackage)?, - &[ - ("node_root", node_root), - ("project_dir", project.path.clone()), - ], - )?; - let discovered_nodes: PyResult<()> = - within_identity_pass(py, || buckets.discover_nodes(&project, &providers)); - call_with( - &shim(py, UserCodeHost::RestorePythonRootModules)?, - &[("saved_modules", saved)], - )?; - discovered_nodes?; - } - let [loaders, tasks, assets, checks] = buckets.nodes; - discovered( - py, - RecordId::DiscoveredPythonNodeFunctions, - &[ - ("loaders", tuple_of(py, loaders)?), - ("tasks", tuple_of(py, tasks)?), - ("assets", tuple_of(py, assets)?), - ("checks", tuple_of(py, checks)?), - ( - "audit_factories", - tuple_of( - py, - buckets - .audit_factories - .into_iter() - .map(|(_, _, record)| record) - .collect(), - )?, - ), - ], - ) -} - /// Decorated model lifecycle hook functions under hooks/python/. #[pyfunction] #[pyo3(signature = (*, project_dir, providers = None))] @@ -524,62 +456,6 @@ pub(crate) fn discover_hook_functions<'py>( tuple_of(py, hooks) } -/// Custom materialization Python files under materializations/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_materialization_files<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { path: project_dir }; - if !project - .child("materializations")? - .call_method0("is_dir")? - .is_truthy()? - { - return tuple_of(py, Vec::new()); - } - let providers: Bound<'py, PyDict> = match providers { - Some(providers) => provider_by_name(&providers)?, - None => PyDict::new(py), - }; - let mut files: Vec> = Vec::new(); - for file_path in project.module_files("materializations", false)? { - let materialize: Bound<'py, PyAny> = - project.load(UserCodeHost::LoadMaterializeFunction, &file_path)?; - let usages: Bound<'py, PyAny> = if materialize.is_none() { - tuple_of(py, Vec::new())? - } else { - provider_usages(&materialize, &providers)? - }; - files.push(discovered( - py, - RecordId::DiscoveredMaterializationFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", project.relative(&file_path)?), - ("name", native_field(&file_path, "stem")?), - ("provider_usages", usages), - ], - )?); - } - tuple_of(py, files) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; - module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; - module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_provider_classes, module)?)?; - module.add_function(wrap_pyfunction!( - discover_event_exporter_declarations, - module - )?)?; - module.add_function(wrap_pyfunction!(bind_event_exporter_declarations, module)?)?; - module.add_function(wrap_pyfunction!(discover_event_exporter_functions, module)?) -} - fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { if value.is_none() { return Ok(value.clone()); diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs index 68aa0b7878..42d3474bf8 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs @@ -1,30 +1,18 @@ //! Python function files discovered natively, with CPython as the authority on syntax and values. use pyo3::exceptions::{PySyntaxError, PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, -}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_discovery::declaration_files::main::python_function_files::python_function_files; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; use sqlbuild_discovery::key_checks::main::unsupported_keys_help::unsupported_keys_help; -use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, ProjectRoot}; +use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_pysource::udf_functions::constants::UDF_KEYS; use sqlbuild_pysource::udf_functions::main::udf_function::udf_function; use sqlbuild_pysource::udf_functions::models::{UdfFunction, UdfShape}; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error}; use crate::discovery::_helpers::project::project_files::path_object; use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -235,7 +223,7 @@ fn parsed_function<'py>( Ok((values, entry_point, warehouse_body(&module, &function)?)) } -fn function_record<'py>( +pub(crate) fn function_record<'py>( project_dir: &Bound<'py, PyAny>, tree: &ProjectTree, file: DiscoveredFile, @@ -268,29 +256,3 @@ fn function_record<'py>( ], ) } - -/// Discover the `@udf(...)` Python function files under `functions/python/`. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_python_function_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - in_snapshot(project_dir, || { - let py: Python<'py> = project_dir.py(); - let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; - let root: ProjectRoot = project_root(project_dir)?; - let inner: &ProjectTree = &tree.get().inner; - let files: Vec> = py - .detach(|| python_function_files(&root, inner)) - .map_err(|failure| stage_error(inner, project_dir, failure))?; - let records: Vec> = files - .into_iter() - .map(|file| function_record(project_dir, inner, file)) - .collect::>()?; - PyTuple::new(py, records) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_function_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs index ded9531391..b7988b6cf0 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs @@ -1,24 +1,23 @@ //! The Python hook files of every hook role, listed natively for the Python module loader. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::python_hook_files::python_hook_files; use sqlbuild_discovery::declaration_files::models::{ CollectionFailure, DiscoverySession, FileScope, }; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{failure_error, stage_error}; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::records::declaration_records::add_scope_fields; -use crate::discovery::_helpers::records::declaration_records::discovery_session; -use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; +use crate::discovery::_helpers::records::declaration_records::{ + add_scope_fields, discovery_session, path_class, +}; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDiscoverySession, NativeProjectTree, +}; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_tree}; /// Each public Python hook file as `file_path`, `relative_path` and its role's scope fields. #[pyfunction] @@ -53,7 +52,3 @@ pub(crate) fn discover_python_hook_files<'py>( PyTuple::new(py, files) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_hook_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs index 2afbf6ce18..c0bb6592c5 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs @@ -1,8 +1,8 @@ //! The project Python module files Python imports, listed natively in Python's sorted order. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::python_module_files::python_module_files; use sqlbuild_discovery::tree::models::ProjectTree; @@ -31,7 +31,3 @@ pub(crate) fn discover_python_module_files<'py>( .collect::>()?; PyTuple::new(py, paths) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_module_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs index 07873c43ed..001e70123c 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs @@ -1,32 +1,4 @@ //! Reject project Python outside documented executable roots. -use std::path::PathBuf; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; - -use crate::discovery::_helpers::failures::raised::discovery_error; - -const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ +pub(crate) const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ python/, or move SQLBuild extensions under their documented extension path."; - -/// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -pub(crate) fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - let root: PathBuf = project_dir.extract()?; - let unsupported: Vec = unsupported_python_paths(&root) - .into_iter() - .map(|parts| parts.join(std::path::MAIN_SEPARATOR_STR)) - .collect(); - if unsupported.is_empty() { - return Ok(()); - } - Err(discovery_error( - project_dir.py(), - "python_path", - format!( - "Unsupported project Python path(s): {}", - unsupported.join(", ") - ), - Some(PYTHON_PATH_HELP.to_owned()), - )) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs index 0e0bb863b9..e1c0c868e3 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs @@ -1,8 +1,8 @@ //! Declaration file collections discovered natively and built as Python discovery records. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ @@ -17,19 +17,16 @@ use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error, stage_error}; use crate::discovery::_helpers::headers::header_declarations::typed_constant_record; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::snapshots::snapshot::snapshot_memo; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDiscoverySession, NativeProjectTree, +}; +use crate::discovery::_helpers::snapshots::snapshot::{ + in_snapshot, project_root, project_tree, snapshot_memo, +}; use crate::model_config::main::headers::schema_columns::schema_columns; use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::declaration_files::main::declaration_file_options::declaration_file_options; @@ -43,7 +40,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; const PATHLIB_MODULE: &str = "pathlib"; -const SESSION_MEMO_KEY: &str = "native_discovery_session"; +pub(crate) const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; /// One declaration file being built: its paths and the scope native discovery listed it in. @@ -391,19 +388,6 @@ pub(crate) fn discovery_session<'py>( Ok(created) } -/// The session this discovery pass read its declaration files with, if any. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn retained_discovery_session<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - Ok(snapshot_memo(project_dir)? - .get_item(SESSION_MEMO_KEY)? - .filter(|session| session.is_instance_of::()) - .unwrap_or_else(|| py.None().into_bound(py))) -} - /// One collection read in the pass's snapshot, with the tree it was read from. fn collection<'py>( project_dir: &Bound<'py, PyAny>, @@ -487,7 +471,7 @@ fn records<'py, T>( record_tuple(py, built) } -fn discovered<'py>( +pub(crate) fn discovered<'py>( faults: &FileFaults<'py>, kind: CollectionKind, isolate_kind: bool, @@ -538,7 +522,7 @@ fn seed_records<'py, T>( record_tuple(py, seeds) } -fn faults<'py>( +pub(crate) fn faults<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> FileFaults<'py> { @@ -547,147 +531,3 @@ fn faults<'py>( on_fault: on_fault.filter(|on_fault| !on_fault.is_none()), } } - -/// Discover global and scoped enum declaration files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] -pub(crate) fn discover_enum_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Enums, - isolate_declaration_kind, - ) -} - -/// Discover global and scoped constant declaration files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] -pub(crate) fn discover_constant_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Constants, - isolate_declaration_kind, - ) -} - -/// Discover project-wide and scoped reusable model schemas. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_model_schema_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::ModelSchemas, - false, - ) -} - -/// Discover SQL function files under functions/sql/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_sql_function_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::SqlFunctions, - false, - ) -} - -/// Discover named SQL lifecycle hooks in their project-wide and scoped roles. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_sql_hook_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::SqlHooks, - false, - ) -} - -/// Discover generic and singular audit SQL files in their project-wide and scoped roles. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_audit_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Audits, - false, - ) -} - -/// Discover global and scoped project macro Python files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] -pub(crate) fn discover_macro_files<'py>( - project_dir: &Bound<'py, PyAny>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, None), - CollectionKind::Macros, - isolate_declaration_kind, - ) -} - -/// Discover seed CSV files under seeds/. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_seed_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - discovered(&faults(project_dir, None), CollectionKind::Seeds, false) -} - -/// Detect a project-level adapter.py without importing it. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_adapter_file<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; - if !file_path.call_method0("is_file")?.is_truthy()? { - return Ok(py.None().into_bound(py)); - } - new_record_with( - py, - RecordId::DiscoveredAdapterFile, - &[ - ("file_path", file_path), - ("relative_path", path_class(py)?.call1(("adapter.py",))?), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; - module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs index 8789e2ac0e..76942ba545 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs @@ -1,16 +1,14 @@ //! Model description resolution for header-only tooling, without importing project Python. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyCFunction; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; -use crate::discovery::_helpers::records::model_records::discovered_model_files; use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; use crate::discovery::models::ModelFileSelection; -use crate::model_config::main::model_nodes::externally_described_models::externally_described_models as described_models; -use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; @@ -59,18 +57,3 @@ pub(crate) fn discover_model_description_inputs<'py>( ) }) } - -/// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -pub(crate) fn externally_described_models<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; - let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; - PyFrozenSet::new(py, described_models(inputs, schemas)?) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_description_inputs, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs index 7c0b3caf9b..213afb1855 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs @@ -205,18 +205,6 @@ fn posix(text: &str) -> String { text.replace(std::path::MAIN_SEPARATOR, "/") } -/// `text` as the Python string `cache` shares with every reader, built on first read. -pub(crate) fn projected_text<'py>( - py: Python<'py>, - cache: &PyOnceLock>, - text: &str, -) -> Bound<'py, PyString> { - cache - .get_or_init(py, || PyString::new(py, text).unbind()) - .bind(py) - .clone() -} - /// A model header's authored values: native entries, with the Python dict built on first read. pub(crate) struct HeaderValues { pub(crate) native: Option>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs index 606e49bb74..5ddaf7edf0 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs @@ -1,46 +1,36 @@ //! SQL model files discovered natively and built as Python discovery records. -use crate::discovery::models::ModelFileSelection; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use std::path::Path; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::project::project_files::model_record; -use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error, stage_error}; +use crate::discovery::_helpers::project::project_files::{model_record, path_object}; use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; /// One discovered model file: its relative path and record, or the error building it raises. -type ModelItem<'py> = (RelativePath<'py>, PyResult>); +pub(crate) type ModelItem<'py> = (RelativePath<'py>, PyResult>); /// A discovered file's relative path: native text, or the Python path already built for it. -enum RelativePath<'py> { +pub(crate) enum RelativePath<'py> { Native(String), Python(Bound<'py, PyAny>), } impl<'py> RelativePath<'py> { - fn object(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn object(&self, py: Python<'py>) -> PyResult> { match self { Self::Native(relative) => Path::new(relative).into_pyobject(py), Self::Python(relative) => Ok(relative.clone()), @@ -48,7 +38,7 @@ impl<'py> RelativePath<'py> { } /// The model's name, its file stem. - fn name(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn name(&self, py: Python<'py>) -> PyResult> { match self { Self::Native(relative) => Ok(PyString::new(py, model_file_stem(relative)).into_any()), Self::Python(relative) => native_field(relative, STEM_ATTRIBUTE), @@ -57,7 +47,7 @@ impl<'py> RelativePath<'py> { } /// Discover every model file natively, each as its record or the error it raises when built. -fn model_items<'py>( +pub(crate) fn model_items<'py>( project_dir: &Bound<'py, PyAny>, options: &ModelFileOptions, ) -> PyResult>> { @@ -106,81 +96,3 @@ fn model_items<'py>( Ok(items) }) } - -/// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -pub(crate) fn discovered_model_files<'py>( - project_dir: &Bound<'py, PyAny>, - extraction: (bool, bool), - selection: ModelFileSelection<'py>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let options: ModelFileOptions = model_file_options( - extraction, - sqlbuild_python_base::compile_context::main::delta::injection::injection() - .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), - ); - let items: Vec> = model_items(project_dir, &options)?; - let faults = FileFaults { - project_dir: project_dir.clone(), - on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), - }; - let selected: Option> = selection - .selected_model_names - .filter(|selected| !selected.is_none()); - let mut records: Vec> = Vec::with_capacity(items.len()); - for (relative_path, built) in items { - if let Some(selected) = &selected - && !selected.contains(relative_path.name(py)?)? - { - continue; - } - match built { - Ok(record) => records.push(record), - Err(error) => { - records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); - } - } - } - record_tuple(py, records) -} - -/// Discover the (selected) model files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn discover_model_files<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "discover_model_files", - options, - ( - &[ - "project_dir", - "extract_implicit_alias_columns", - "extract_output_column_locations", - ], - &["selected_model_names", "on_fault"], - ), - )?; - discovered_model_files( - &arguments.value("project_dir")?, - ( - arguments - .value("extract_implicit_alias_columns")? - .extract()?, - arguments - .value("extract_output_column_locations")? - .extract()?, - ), - ModelFileSelection { - selected_model_names: Some(arguments.value("selected_model_names")?), - on_fault: Some(arguments.value("on_fault")?), - }, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs index 1dd3ed910b..4c0bf51b14 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs @@ -1,34 +1,27 @@ //! SQL unit-test and scenario files discovered natively and built as Python discovery records. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, }; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_discovery::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, StageFailure}; -use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; -use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; -use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; use sqlbuild_discovery::sql_tests::models::{ DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, }; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::sources::sql_test_blocks::sql_test_block_records; -use crate::discovery::_helpers::sources::sql_test_blocks::validate_block_names; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; +use crate::discovery::_helpers::sources::sql_test_blocks::{ + sql_test_block_records, validate_block_names, +}; use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::constants; use sqlbuild_discovery::sql_tests::main::sql_test_file_options::sql_test_file_options; @@ -56,14 +49,14 @@ type Discoverer = fn( type RecordBuilder<'py, T> = fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; -fn options() -> PyResult { +pub(crate) fn options() -> PyResult { Ok(sql_test_file_options( sqlbuild_python_base::compile_context::main::delta::injection::injection() .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), )) } -fn path_class(py: Python<'_>) -> PyResult> { +pub(crate) fn path_class(py: Python<'_>) -> PyResult> { imported(py, PATHLIB_MODULE, "Path") } @@ -73,7 +66,7 @@ fn ownership_root<'py>(py: Python<'py>, root: &str) -> PyResult( +pub(crate) fn discovered_files<'py, T: Send>( project_dir: &Bound<'py, PyAny>, discover: Discoverer, ) -> PyResult<(Bound<'py, NativeProjectTree>, Vec>)> { @@ -93,7 +86,7 @@ fn discovered_files<'py, T: Send>( } /// Each block as a `DiscoveredSqlTestBlock`, then the split failure, then the name check. -fn test_blocks<'py>( +pub(crate) fn test_blocks<'py>( file_path: &Bound<'py, PyAny>, blocks: Vec, failure: Option, @@ -127,7 +120,7 @@ fn test_blocks<'py>( PyTuple::new(py, records) } -fn test_file<'py>( +pub(crate) fn test_file<'py>( paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), file: DiscoveredSqlTestFile, ) -> PyResult> { @@ -153,7 +146,7 @@ fn test_file<'py>( } /// Python's `build_sql_scenario_file`: validate the header and build the record. -fn scenario_file<'py>( +pub(crate) fn scenario_file<'py>( paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), file: DiscoveredScenarioFile, ) -> PyResult> { @@ -229,7 +222,7 @@ fn scenario_file<'py>( } /// Build each file, raising or reporting each failing one. -fn records<'py, T>( +pub(crate) fn records<'py, T>( faults: &FileFaults<'py>, files: Vec<(Bound<'py, PyAny>, FileOutcome)>, build: RecordBuilder<'py, T>, @@ -248,81 +241,6 @@ fn records<'py, T>( record_tuple(py, built) } -/// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] -pub(crate) fn discover_test_files<'py>( - project_dir: &Bound<'py, PyAny>, - selected_paths: Option>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let (tree, files) = discovered_files(project_dir, discover_tests)?; - let tree: &ProjectTree = &tree.get().inner; - let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); - for file in files { - let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); - if let Some(selected) = &selected_paths - && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? - { - continue; - } - kept.push((path_class(py)?.call1((relative,))?, file.outcome)); - } - records( - &FileFaults { - project_dir: project_dir.clone(), - on_fault, - }, - kept, - test_file, - ) -} - -/// Discover scenario files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_scenario_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let (tree, files) = discovered_files(project_dir, discover_scenarios)?; - let tree: &ProjectTree = &tree.get().inner; - let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files - .into_iter() - .map(|file| { - Ok(( - path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, - file.outcome, - )) - }) - .collect::>()?; - records( - &FileFaults { - project_dir: project_dir.clone(), - on_fault, - }, - kept, - scenario_file, - ) -} - -/// Parse in-memory SQL test contents into ordered blocks, as test discovery does. -#[pyfunction] -#[pyo3(signature = (*, contents, file_path))] -pub(crate) fn parse_sql_test_file<'py>( - contents: String, - file_path: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = file_path.py(); - let path: String = file_path.str()?.to_str()?.to_owned(); - let options: SqlTestFileOptions = options()?; - let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); - let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; - test_blocks(file_path, file.blocks, file.failure) -} - /// Parse in-memory scenario contents into the record scenario discovery builds for a file. #[pyfunction] #[pyo3(signature = (*, contents, file_path, relative_path))] @@ -338,10 +256,3 @@ pub(crate) fn parse_sql_scenario_file<'py>( let file: DiscoveredScenarioFile = parsed.map_err(|failure| failure_error(py, failure))?; scenario_file((file_path, relative_path), file) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs index 964e001533..2a70e191a3 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs @@ -1,51 +1,39 @@ //! Source and schema YAML files discovered natively and built as Python discovery records. +use crate::discovery::main::snapshots::escaped::escaped; use pyo3::exceptions::PyUnicodeEncodeError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyList, PyStringMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyList, PyStringMethods}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_config::models::ConfigValue; use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind, ProjectRoot, StageFailure}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; -use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; -use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; -use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use sqlbuild_python_base::compile_context::main::yaml_cache::retain_read_sources::retain_read_sources; use std::path::PathBuf; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::native_classes::YamlDiscoveryRequest; -use crate::discovery::_helpers::snapshots::snapshot::escaped; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; -use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeProjectTree, YamlDiscoveryRequest, +}; +use crate::discovery::_helpers::snapshots::snapshot::{project_root, project_tree}; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::types::RecordId; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; const PATHLIB_MODULE: &str = "pathlib"; const PROJECT_PATH_KIND: &str = "project_path"; -const SCHEMA_KIND: &str = "schema"; +pub(crate) const SCHEMA_KIND: &str = "schema"; /// The YAML files one discovery reads, and the kind their failures raise as. #[derive(Clone, Copy)] -enum YamlKind { +pub(crate) enum YamlKind { Source, Schema, } @@ -59,12 +47,12 @@ impl YamlKind { } } -fn path_class(py: Python<'_>) -> PyResult> { +pub(crate) fn path_class(py: Python<'_>) -> PyResult> { imported(py, PATHLIB_MODULE, "Path") } /// `(contents, loaded)` of a loaded file; a value Python cannot build fails like invalid YAML. -fn loaded<'py>( +pub(crate) fn loaded<'py>( py: Python<'py>, (display_path, kind): (&str, FailureKind), (contents, value): (String, ConfigValue), @@ -101,7 +89,7 @@ fn outcome_loaded<'py>( } /// The pass's tree, or a private one for an undecodable project directory (fails on first read). -fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { match project_dir.str()?.to_str() { Ok(_) => Ok(ListingTree::Shared(project_tree(project_dir)?)), Err(error) if error.is_instance_of::(project_dir.py()) => { @@ -112,13 +100,13 @@ fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult { +pub(crate) enum ListingTree<'py> { Shared(Bound<'py, NativeProjectTree>), Private(ProjectTree), } impl ListingTree<'_> { - fn tree(&self) -> &ProjectTree { + pub(crate) fn tree(&self) -> &ProjectTree { match self { Self::Shared(tree) => &tree.get().inner, Self::Private(tree) => tree, @@ -148,7 +136,7 @@ fn batches(tree: &ProjectTree, paths: Vec, limit: u64) -> Vec( +pub(crate) fn load_files<'py>( project_dir: &Bound<'py, PyAny>, (paths, kind): (Vec, YamlKind), mut on_file: impl FnMut( @@ -233,113 +221,6 @@ fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> )) } -/// Discover source YAML files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_source_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - in_snapshot(project_dir, || { - let listing: ListingTree<'py> = listing_tree(project_dir)?; - let paths: Vec = source_yaml_paths(listing.tree()) - .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; - if paths.is_empty() { - return record_tuple(py, Vec::new()); - } - let faults = FileFaults { - project_dir: project_dir.clone(), - on_fault, - }; - let mut records: Vec> = Vec::new(); - load_files( - project_dir, - (paths, YamlKind::Source), - |(relative_path, file_path), built| { - let record = built.and_then(|(contents, loaded)| { - let shown: String = file_path.str()?.to_str()?.to_owned(); - new_record_with( - py, - RecordId::DiscoveredSourceFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", relative_path.clone()), - ("contents", contents.into_bound_py_any(py)?), - ( - "source_entries", - parse_sources_document(py, loaded, &shown)?.into_any(), - ), - ], - ) - }); - records.extend(faults.kept(&relative_path, record)?); - Ok(()) - }, - )?; - record_tuple(py, records) - }) -} - -/// Discover model schema.yml and seed declaration files natively. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_schema_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - in_snapshot(project_dir, || { - let listing: ListingTree<'py> = listing_tree(project_dir)?; - let paths: Vec = match schema_yaml_paths(listing.tree()) { - Ok(paths) => paths, - Err(YamlPathFailure::Stage(failure)) => { - return Err(stage_error(listing.tree(), project_dir, failure)); - } - Err(YamlPathFailure::YamlSeedDeclaration(path)) => { - let relative: Bound<'py, PyAny> = - path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; - return Err(discovery_error( - py, - SCHEMA_KIND, - format!( - "Seed declaration file {} must use .yml; .yaml is not supported", - relative.str()? - ), - None, - )); - } - }; - if paths.is_empty() { - return record_tuple(py, Vec::new()); - } - let mut records: Vec> = Vec::new(); - load_files( - project_dir, - (paths, YamlKind::Schema), - |(relative_path, file_path), built| { - let (contents, loaded) = built?; - let shown: String = file_path.str()?.to_str()?.to_owned(); - records.push(new_record_with( - py, - RecordId::DiscoveredSchemaFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", relative_path), - ("contents", contents.into_bound_py_any(py)?), - ("model_entries", PyTuple::empty(py).into_any()), - ( - "seed_entries", - parse_schema_document(py, loaded, &shown)?.into_any(), - ), - ], - )?); - Ok(()) - }, - )?; - record_tuple(py, records) - }) -} - fn yaml_kind(kind: &str) -> PyResult { match kind { "source" => Ok(FailureKind::Source), @@ -352,7 +233,7 @@ fn yaml_kind(kind: &str) -> PyResult { /// Read and load YAML files: each file's `(contents, loaded)`, or the error it raises. #[pyfunction] -fn load_yaml_files<'py>( +pub(crate) fn load_yaml_files<'py>( py: Python<'py>, request: YamlDiscoveryRequest, relative_paths: Vec, @@ -382,7 +263,7 @@ fn load_yaml_files<'py>( /// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. #[pyfunction] -fn load_yaml_document<'py>( +pub(crate) fn load_yaml_document<'py>( py: Python<'py>, file_path: &str, text: String, @@ -393,10 +274,3 @@ fn load_yaml_document<'py>( load_yaml_text(file_path, &text, kind).map_err(|failure| failure_error(py, failure))?; Ok(loaded(py, (file_path, kind), (text, value))?.1) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_document, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs index ccffda6237..42e194af63 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs @@ -1,27 +1,23 @@ //! Tolerant discovery of the scope-relevant authored roots, keeping each independent file fault. +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::records::discover_audit_files::discover_audit_files; +use crate::discovery::main::records::discover_constant_files::discover_constant_files; +use crate::discovery::main::records::discover_enum_files::discover_enum_files; +use crate::discovery::main::records::discover_macro_files::discover_macro_files; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discover_scenario_files::discover_scenario_files; +use crate::discovery::main::records::discover_source_files::discover_source_files; +use crate::discovery::main::records::discover_sql_function_files::discover_sql_function_files; +use crate::discovery::main::records::discover_sql_hook_files::discover_sql_hook_files; +use crate::discovery::main::records::discover_test_files::discover_test_files; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::discovery::_helpers::project::project_config::load_local_config_file; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::records::declaration_records::discover_audit_files; -use crate::discovery::_helpers::records::declaration_records::discover_constant_files; -use crate::discovery::_helpers::records::declaration_records::discover_enum_files; -use crate::discovery::_helpers::records::declaration_records::discover_macro_files; -use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; -use crate::discovery::_helpers::records::declaration_records::discover_sql_function_files; -use crate::discovery::_helpers::records::declaration_records::discover_sql_hook_files; use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::records::model_records::discovered_model_files; -use crate::discovery::_helpers::records::sql_test_records::discover_scenario_files; -use crate::discovery::_helpers::records::sql_test_records::discover_test_files; -use crate::discovery::_helpers::records::yaml_records::discover_source_files; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; use crate::discovery::models::ModelFileSelection; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -177,7 +173,9 @@ fn chained<'py>(py: Python<'py>, lists: &[&Bound<'py, PyList>]) -> PyResult(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn tolerant_discovery<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project_dir.py(); let faults = Faults { project_dir: project_dir.clone(), @@ -263,16 +261,3 @@ fn tolerant_discovery<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - in_snapshot(project_dir, || tolerant_discovery(project_dir)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_scope_snapshot, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs index a8d137b27b..5cd57480ef 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs @@ -1,19 +1,19 @@ //! The directory snapshot one discovery pass shares, and the native tree it memoises. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, -}; +use crate::discovery::main::snapshots::escaped::escaped; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRefMut, PyResult, Python}; +use pyo3::pymethods; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::models::ProjectRoot; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_python_base::boundary::main::context_variable::context_variable; use std::path::PathBuf; use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDirectorySnapshot, NativeProjectTree, +}; use crate::discovery::models::NativeDirectorySnapshotScope; const SNAPSHOT_VARIABLE: &str = "sqlbuild_directory_snapshot"; @@ -23,8 +23,8 @@ const PROJECT_PATH_KIND: &str = "project_path"; const SURROGATE_ESCAPE_BASE: u32 = 0xDC00; const ESCAPED_BYTE_FIRST: u16 = 0xDC80; const ESCAPED_BYTE_LAST: u16 = 0xDCFF; -const UTF16_ENCODING: &str = "utf-16-le"; -const SURROGATE_PASS: &str = "surrogatepass"; +pub(crate) const UTF16_ENCODING: &str = "utf-16-le"; +pub(crate) const SURROGATE_PASS: &str = "surrogatepass"; static ACTIVE_SNAPSHOT: PyOnceLock> = PyOnceLock::new(); @@ -140,25 +140,6 @@ impl NativeDirectorySnapshotScope { } } -/// Share one directory snapshot across every discovery call for `project_dir` inside the scope. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn directory_snapshot_scope( - project_dir: Bound<'_, PyAny>, -) -> NativeDirectorySnapshotScope { - NativeDirectorySnapshotScope { - project_dir: project_dir.unbind(), - token: None, - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(directory_snapshot_scope, module)?)?; - Ok(()) -} - /// `str(project_dir / "_")` without its probe: the text printed before a relative path. pub(crate) fn display_prefix(project_dir: &Bound<'_, PyAny>) -> PyResult { let probed: String = project_dir.div(DISPLAY_PROBE)?.str()?.to_str()?.to_owned(); @@ -213,25 +194,7 @@ pub(crate) fn project_tree<'py>( Ok(created) } -/// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -pub(crate) fn escaped(text: &Bound<'_, PyString>) -> PyResult { - let encoded: Vec = text - .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? - .extract()?; - let units = encoded - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)); - Ok(char::decode_utf16(units) - .map(|decoded| match decoded { - Ok(character) => character.to_string(), - Err(error) => escaped_unit(error.unpaired_surrogate()), - }) - .collect()) -} - -fn escaped_unit(unit: u16) -> String { +pub(crate) fn escaped_unit(unit: u16) -> String { if cfg!(unix) && (ESCAPED_BYTE_FIRST..=ESCAPED_BYTE_LAST).contains(&unit) { return format!("\\x{:02x}", u32::from(unit) - SURROGATE_ESCAPE_BASE); } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs index 029e093f42..0b91dea1bc 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs @@ -1,8 +1,8 @@ //! In-memory declaration file parsing, for the oracle comparing native and Python parsing. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_discovery::declaration_files::main::parse_declaration_text::parse_declaration_text; use sqlbuild_discovery::declaration_files::models::{ CollectionKind, DeclarationFileOptions, ParsedDeclarationText, @@ -10,7 +10,6 @@ use sqlbuild_discovery::declaration_files::models::{ use sqlbuild_discovery::models::FileOutcome; use crate::discovery::_helpers::snapshots::native_classes::DeclarationTextRequest; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; use sqlbuild_python_base::types::CompilerDetach; const PARSED_TAG: &str = "ok"; @@ -19,7 +18,7 @@ const UNREADABLE_TAG: &str = "read"; /// Whether a `kind` declaration file with these contents parses (`ok`), fails or is unreadable. #[pyfunction] -fn parse_declaration_contents( +pub(crate) fn parse_declaration_contents( py: Python<'_>, request: DeclarationTextRequest, names: (String, String), @@ -54,9 +53,3 @@ fn parse_declaration_contents( ))), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_declaration_contents, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs index 43e1733cdd..086922ecf7 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs @@ -1,35 +1,14 @@ //! Orchestration declarations read natively, raised as the Python exceptions discovery raises. -use std::path::PathBuf; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; -use sqlbuild_discovery::orchestration_files::models::{ - OrchestrationFailure, OrchestrationInventory, -}; +use pyo3::prelude::{Bound, PyAny, PyErr}; +use sqlbuild_discovery::orchestration_files::models::OrchestrationFailure; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::relative_read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; - -/// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -pub(crate) fn orchestration_declarations( - project_dir: &Bound<'_, PyAny>, -) -> PyResult { - let py: Python<'_> = project_dir.py(); - let root: PathBuf = project_dir.extract()?; - let tree = ProjectTree::new(&root); - let version = py.version_info(); - let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); - let inventory = - inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; - check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; - Ok(inventory) -} +use crate::discovery::_helpers::failures::raised::{ + failure_error, relative_read_error, stage_error, +}; -fn orchestration_error( +pub(crate) fn orchestration_error( tree: &ProjectTree, project_dir: &Bound<'_, PyAny>, failure: OrchestrationFailure, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs index 2b83ef870b..334ad93016 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs @@ -10,30 +10,17 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::ErrorId; /// Discovered collections of sinks that run with every command. -const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; +pub(crate) const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; thread_local! { /// Whether the invocation running on this thread passed `--allow-missing-secrets`. pub(crate) static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; } -/// Whether the running invocation passed `--allow-missing-secrets`. -pub(crate) fn invocation_allows_missing_secrets() -> bool { - INVOCATION_ALLOWS_MISSING_SECRETS.get() -} - -/// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -pub(crate) fn provider_uses( - usages: &Bound<'_, PyAny>, +pub(crate) fn usage_pair( + usage: &Bound<'_, PyAny>, consumer: Option<&str>, -) -> PyResult> { - usages - .try_iter()? - .map(|usage| usage_pair(&usage?, consumer)) - .collect() -} - -fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(String, String)> { +) -> PyResult<(String, String)> { let provider: String = field(usage, "provider_name")?.extract()?; let name: String = match consumer { Some(name) => name.to_owned(), @@ -42,31 +29,6 @@ fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(Str Ok((provider, name)) } -/// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -pub(crate) fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { - let mut uses: Vec<(String, String)> = Vec::new(); - for consumer in consumers.try_iter()? { - let consumer: Bound<'_, PyAny> = consumer?; - let name: String = field(&consumer, "name")?.extract()?; - uses.extend(provider_uses( - &field(&consumer, "provider_usages")?, - Some(&name), - )?); - } - Ok(uses) -} - -/// The providers the project's lifecycle and command-output sinks use, which run with every command. -pub(crate) fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { - let mut uses: Vec<(String, String)> = Vec::new(); - for collection in SINK_COLLECTIONS { - if let Some(sinks) = discovered.getattr_opt(collection)? { - uses.extend(consumer_uses(&sinks)?); - } - } - Ok(uses) -} - /// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. pub(crate) fn require_used_secrets( discovered: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs index 74a8d6921f..cdfa999a41 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs @@ -1,10 +1,8 @@ //! Loaded schema and source YAML documents parsed natively into the Python contract objects. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyfunction}; use sqlbuild_discovery::yaml_entries::main::parse_schema_document::parse_schema_document as parse_schema; use sqlbuild_discovery::yaml_entries::main::parse_sources_document::parse_sources_document as parse_sources; use sqlbuild_discovery::yaml_entries::models::{ @@ -383,8 +381,3 @@ fn loaded_sources(sources: &[ParsedSourceEntry>]) -> Vec { }) .collect() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; - module.add_function(wrap_pyfunction!(parse_sources_document, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs index 9fbbcfeb55..7921f405d0 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs @@ -1,9 +1,11 @@ //! Explicit references help. -use crate::discovery::_helpers::failures::guidance; +use pyo3::pyfunction; +use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; /// Explicit references help. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, allowed))] pub fn explicit_references_help(allowed: &str) -> String { - guidance::explicit_references_help(allowed) + references_help(allowed) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs index 6b6a9694f2..f46db01825 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs @@ -1,9 +1,10 @@ //! Microbatch guidance. -use crate::discovery::_helpers::failures::guidance; +use pyo3::pyfunction; +use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; /// Microbatch guidance. -#[inline] +#[pyfunction] pub fn microbatch_guidance() -> (String, String) { - guidance::microbatch_guidance() + concurrency_guidance() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs index 1de4ee6af6..451a179aa5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs @@ -1,12 +1,19 @@ //! Register the guidance functions and classes. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::guidance::snippet_help; +use crate::discovery::_helpers::failures::guidance::sql_analysis_off_guidance; +use crate::discovery::main::failures::explicit_references_help::explicit_references_help; +use crate::discovery::main::failures::microbatch_guidance::microbatch_guidance; +use crate::discovery::main::failures::setting_help::setting_help; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the guidance functions and classes. -#[inline] pub fn register_guidance(module: &Bound<'_, PyModule>) -> PyResult<()> { - guidance::register(module) + module.add_function(wrap_pyfunction!(setting_help, module)?)?; + module.add_function(wrap_pyfunction!(snippet_help, module)?)?; + module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; + module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; + module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs index 923c3390b6..86bc484252 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs @@ -1,12 +1,10 @@ //! Register the raised functions and classes. -use crate::discovery::_helpers::failures::raised; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::_oracle_read_error; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the raised functions and classes. -#[inline] pub fn register_raised(module: &Bound<'_, PyModule>) -> PyResult<()> { - raised::register(module) + module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs index e63a8de597..d52e8dfd6d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs @@ -1,13 +1,33 @@ //! Render `, set this in :` followed by the exact TOML section and key. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::failures::guidance::setting_text; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use sqlbuild_config::project::models::SettingSite; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Render `, set this in :` followed by the exact TOML section and key. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { - guidance::setting_help(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "setting_help", + options, + (&["purpose", "file_name", "section", "key", "value"], &[]), + )?; + let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; + let (file_name, section, key): (String, String, String) = + (text("file_name")?, text("section")?, text("key")?); + Ok(setting_help_text( + &text("purpose")?, + SettingSite { + file_name: &file_name, + section: §ion, + key: &key, + }, + &setting_text(&arguments.value("value")?)?, + )) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs index 4073b59569..003f8d47b9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs @@ -1,15 +1,25 @@ //! The note and help for turning SQL analysis on, or `None` when it is effectively on. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::guidance::SQL_ANALYSIS_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The note and help for turning SQL analysis on, or `None` when it is effectively on. -#[inline] pub fn sql_analysis_guidance( discovered_inputs: &Bound<'_, PyAny>, purpose: &str, ) -> PyResult> { - guidance::sql_analysis_guidance(discovered_inputs, purpose) + let local: Bound<'_, PyAny> = native_field(discovered_inputs, "local_config")?; + let local_override: bool = + native_field(&local, "setting_overrides")?.contains(SQL_ANALYSIS_KEY)?; + let owner: Bound<'_, PyAny> = if local_override { + local + } else { + native_field(discovered_inputs, "project_config")? + }; + if native_field(&native_field(&owner, "settings")?, SQL_ANALYSIS_KEY)?.is_truthy()? { + return Ok(None); + } + Ok(Some(analysis_off(local_override, purpose))) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs index 5467e5d106..6c12c31e92 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs @@ -1,18 +1,42 @@ //! Serialize SQL and Python hook entries; other values serialize to no hooks. -use crate::discovery::_helpers::headers::hook_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyList; +use crate::discovery::_helpers::headers::hook_entries::sql_hook; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Serialize SQL and Python hook entries; other values serialize to no hooks. -#[inline] pub fn hook_entries<'py>( value: &Bound<'py, PyAny>, sql_fields: &[String], python_hook_fields: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - hook_entries::hook_entries(value, sql_fields, python_hook_fields) + let py = value.py(); + let hooks: Bound<'py, PyList> = PyList::empty(py); + if !value.is_instance_of::() && !value.is_instance_of::() { + return Ok(hooks); + } + let sql_class: Bound<'py, PyAny> = record_class(py, RecordId::SqlHookEntry)?; + let python_class: Bound<'py, PyAny> = record_class(py, RecordId::PythonHookEntry)?; + for entry in value.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if entry.is_instance(&sql_class)? { + hooks.append(sql_hook(&entry, sql_fields)?)?; + } else if entry.is_instance(&python_class)? { + let hook: Bound<'py, PyDict> = PyDict::new(py); + let name: Bound<'py, PyAny> = native_field(&entry, "name")?; + hook.set_item("type", "python")?; + hook.set_item("name", &name)?; + hook.set_item("kwargs", native_field(&entry, "kwargs")?)?; + if let Some(fields) = python_hook_fields + && let Some(extra) = fields.get_item(&name)? + { + hook.call_method1("update", (extra,))?; + } + hooks.append(hook)?; + } + } + Ok(hooks) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs index fda14402b4..086a5a861b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs @@ -1,12 +1,10 @@ //! Register the header declarations functions and classes. -use crate::discovery::_helpers::headers::header_declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::header_declarations::model_header_declarations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header declarations functions and classes. -#[inline] pub fn register_header_declarations(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_declarations::register(module) + module.add_function(wrap_pyfunction!(model_header_declarations, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs index ae668cb5c5..48cc7c0303 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs @@ -1,12 +1,14 @@ //! Register the header values functions and classes. -use crate::discovery::_helpers::headers::header_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::header_values::model_output_column_locations; +use crate::discovery::_helpers::headers::header_values::parse_header_values; +use crate::discovery::_helpers::headers::header_values::prepare_model_header_tokens; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header values functions and classes. -#[inline] pub fn register_header_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_values::register(module) + module.add_function(wrap_pyfunction!(prepare_model_header_tokens, module)?)?; + module.add_function(wrap_pyfunction!(parse_header_values, module)?)?; + module.add_function(wrap_pyfunction!(model_output_column_locations, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs index cd409be58a..b763d2f151 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs @@ -1,12 +1,11 @@ //! Register the hook entries functions and classes. -use crate::discovery::_helpers::headers::hook_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::hook_entries::serialize_hook_entries; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the hook entries functions and classes. -#[inline] pub fn register_hook_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { - hook_entries::register(module) + module.add_function(wrap_pyfunction!(serialize_hook_entries, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs index c09f4fc734..bc5dea63d9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs @@ -1,12 +1,12 @@ //! Register the key checks functions and classes. -use crate::discovery::_helpers::headers::key_checks; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::key_checks::unknown_mapping_keys; +use crate::discovery::_helpers::headers::key_checks::unsupported_keys_help; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the key checks functions and classes. -#[inline] pub fn register_key_checks(module: &Bound<'_, PyModule>) -> PyResult<()> { - key_checks::register(module) + module.add_function(wrap_pyfunction!(unknown_mapping_keys, module)?)?; + module.add_function(wrap_pyfunction!(unsupported_keys_help, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs index 6c6a3da67e..9a678b15df 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs @@ -1,12 +1,9 @@ //! Column name. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Column name. -#[inline] pub fn column_name(column: &Bound<'_, PyAny>) -> PyResult { - project_facts::column_name(column) + native_field(column, "name")?.extract() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs index c647110437..a0628da829 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs @@ -1,13 +1,11 @@ //! `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_changes_config::load_changes_config; use sqlbuild_config::project::models::ChangesConfig; /// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -#[inline] pub fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { - project_config::configured_changes(project_dir) + configured(project_dir, load_changes_config) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs index 453f1ecffe..cc3ee70351 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs @@ -1,12 +1,10 @@ //! `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; /// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -#[inline] pub fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { - project_config::configured_git_base_ref(project_dir) + configured(project_dir, load_git_base_ref) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs index 37b4f3abc4..e683ec6c8a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs @@ -1,13 +1,11 @@ //! `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_review_config::load_review_config; use sqlbuild_config::project::models::ReviewConfig; /// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -#[inline] pub fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { - project_config::configured_review(project_dir) + configured(project_dir, load_review_config) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs index 5a7dc7607d..6e4457a04a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs @@ -1,18 +1,41 @@ //! The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -use crate::discovery::_helpers::project::configuration; +use crate::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; +use crate::discovery::_helpers::project::configuration::loaded_configuration; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; use crate::discovery::models::SecretPolicy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -#[inline] pub fn discover_configured_providers<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, secret_policy: SecretPolicy, ) -> PyResult> { - configuration::discover_configured_providers(py, project_dir, secret_policy) + let configuration: Bound<'py, PyAny> = loaded_configuration(py, project_dir, |config| { + reject_invalid_project_configuration(config)?; + warn_deprecated_settings(py, project_dir) + })?; + let providers: Bound<'py, PyAny> = + discover_provider_classes(project_dir.clone(), secret_policy.name())?; + new_record_with( + py, + RecordId::DiscoveredProjectInputs, + &[ + ("project_dir", project_dir.clone()), + ( + "project_config", + native_field(&configuration, "project_config")?, + ), + ( + "local_config", + native_field(&configuration, "local_config")?, + ), + ("providers", providers), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs index 72edfe0943..caae7dab5d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs @@ -1,16 +1,13 @@ //! The validated configuration without a discovery lifecycle, which reading two files never needs. -use crate::discovery::_helpers::project::configuration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; +use crate::discovery::_helpers::project::configuration::loaded_configuration; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// The validated configuration without a discovery lifecycle, which reading two files never needs. -#[inline] pub fn discover_connection_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - configuration::discover_connection_configuration(py, project_dir) + loaded_configuration(py, project_dir, reject_invalid_connection_configuration) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs index cec76516c8..1e1144fe5a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs @@ -1,16 +1,12 @@ //! `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -use crate::discovery::_helpers::project::configuration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::configuration::configuration_inputs; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -#[inline] pub fn discover_project_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - configuration::discover_project_configuration(py, project_dir) + configuration_inputs(py, project_dir, |_| Ok(())) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/each.rs b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs index 55055191f5..3eae8b954e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/each.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs @@ -1,9 +1,7 @@ //! `read` applied to each item of the iterable attribute `name` of `owner`. use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `read` applied to each item of the iterable attribute `name` of `owner`. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs index cf145a6ad4..689f5f745d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs @@ -1,17 +1,21 @@ //! `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::built; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::load_local_config::load_local_config; /// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -#[inline] +#[pyfunction] pub fn load_local_config_file( py: Python<'_>, project_dir: &Bound<'_, PyAny>, ) -> PyResult> { - project_config::load_local_config_file(py, project_dir) + let files = files(project_dir)?; + match load_local_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs index e72087f8ec..8d3ce64549 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs @@ -1,17 +1,21 @@ //! `load_project_config`: the validated `ProjectConfig` of `project_dir`. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::built; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::load_project_config::load_project_config; /// `load_project_config`: the validated `ProjectConfig` of `project_dir`. -#[inline] +#[pyfunction] pub fn load_project_config_file( py: Python<'_>, project_dir: &Bound<'_, PyAny>, ) -> PyResult> { - project_config::load_project_config_file(py, project_dir) + let files = files(project_dir)?; + match load_project_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs index c7291aa22e..37581a4ac6 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs @@ -1,13 +1,25 @@ //! The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -use crate::discovery::_helpers::project::project_facts; +use crate::discovery::_helpers::project::project_facts::PYTHON_NODE_FIELDS; +use crate::discovery::_helpers::project::project_facts::each; +use crate::discovery::_helpers::project::project_facts::text; use crate::discovery::models::NodeRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -#[inline] pub fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { - project_facts::named_node_rows(inputs) + let mut rows: Vec = Vec::new(); + for (kind, name) in PYTHON_NODE_FIELDS { + rows.extend(each(inputs, name, |node| { + Ok(NodeRow { + kind: kind.to_owned(), + name: native_field(node, "name")?.extract()?, + path: text(node, "relative_path")?, + decorator_config: false, + depends_on: Vec::new(), + }) + })?); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs index 4cd226e6e7..a3bd94c780 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs @@ -1,19 +1,63 @@ //! The facts of `inputs`, with the Python nodes the facade has described. -use crate::discovery::_helpers::project::project_facts; +use crate::discovery::_helpers::project::project_facts::audit_fact; +use crate::discovery::_helpers::project::project_facts::csv_field_limit; +use crate::discovery::_helpers::project::project_facts::declarations_fact; +use crate::discovery::_helpers::project::project_facts::each; +use crate::discovery::_helpers::project::project_facts::model_fact; +use crate::discovery::_helpers::project::project_facts::named; +use crate::discovery::_helpers::project::project_facts::node_fact; +use crate::discovery::_helpers::project::project_facts::schema_fact; +use crate::discovery::_helpers::project::project_facts::seed_file_fact; +use crate::discovery::_helpers::project::project_facts::source_file_fact; +use crate::discovery::_helpers::project::project_facts::stem_named; +use crate::discovery::_helpers::project::project_facts::test_file_fact; use crate::discovery::models::NodeRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_discovery::project_validation::models::ProjectFacts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::project_validation::models::{ + NamedPath, ProjectFacts, ScopedDeclarationsFact, +}; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The facts of `inputs`, with the Python nodes the facade has described. -#[inline] pub fn project_facts( py: Python<'_>, inputs: &Bound<'_, PyAny>, nodes: Vec, ) -> PyResult { - project_facts::project_facts(py, inputs, nodes) + let mut declaration_files: Vec = + each(inputs, "enum_files", declarations_fact)?; + declaration_files.extend(each(inputs, "constant_files", declarations_fact)?); + let mut model_schemas: Vec = Vec::new(); + for schema in native_field(inputs, "model_schema_files")?.try_iter()? { + model_schemas.extend(each(&schema?, "declarations", named)?); + } + Ok(ProjectFacts { + models: each(inputs, "model_files", model_fact)?, + sql_functions: each(inputs, "sql_function_files", stem_named)?, + python_functions: each(inputs, "python_function_files", stem_named)?, + sql_hooks: each(inputs, "sql_hook_files", named)?, + seeds: each(inputs, "seed_files", seed_file_fact)?, + schemas: each(inputs, "schema_files", |schema| schema_fact(py, schema))?, + sources: each(inputs, "source_files", source_file_fact)?, + tests: each(inputs, "test_files", test_file_fact)?, + scenarios: each(inputs, "scenario_files", named)?, + audits: each(inputs, "audit_files", audit_fact)?, + declaration_files, + model_schemas, + materializations: each(inputs, "materialization_files", named)?, + nodes: nodes.into_iter().map(node_fact).collect::>()?, + hook_functions: each(inputs, "hook_functions", named)?, + event_exporters: each(inputs, "event_exporters", named)?, + providers: each(inputs, "providers", named)?, + path_default_keys: native_field(inputs, "project_config")? + .getattr("path_defaults")? + .try_iter()? + .map(|key| key?.extract()) + .collect::>()?, + project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), + csv_field_limit: Some(csv_field_limit(py)?), + orchestration: Vec::new(), + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs index 8765670175..0bc99fcf13 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs @@ -1,12 +1,13 @@ //! Register the locations functions and classes. -use crate::discovery::_helpers::project::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::locations::project_relative_path; +use crate::discovery::main::project::yaml_entry_line::yaml_entry_line; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the locations functions and classes. -#[inline] pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { - locations::register(module) + module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; + module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs index 065654f6f6..1717f22fe1 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs @@ -1,12 +1,16 @@ //! Register the project config functions and classes. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::project::resolve_config_file_path::resolve_config_file_path; +use crate::discovery::main::project::validate_local_sql_analysis::validate_local_sql_analysis; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project config functions and classes. -#[inline] pub fn register_project_config(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_config::register(module) + module.add_function(wrap_pyfunction!(load_project_config_file, module)?)?; + module.add_function(wrap_pyfunction!(load_local_config_file, module)?)?; + module.add_function(wrap_pyfunction!(resolve_config_file_path, module)?)?; + module.add_function(wrap_pyfunction!(validate_local_sql_analysis, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs index 117895dbe5..009817cbf6 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs @@ -1,12 +1,15 @@ //! Whether native discovery reproduces the string semantics of this Python and Unicode version. -use crate::discovery::_helpers::project::project_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_files::parse_model_contents; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::models::NativeModelFile; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Whether native discovery reproduces the string semantics of this Python and Unicode version. -#[inline] pub fn register_project_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_files::register(module) + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs index b2f59cb128..b29729cdea 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs @@ -1,12 +1,13 @@ //! Register the project validation functions and classes. -use crate::discovery::_helpers::project::project_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_validation::validate_discovered_project_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project validation functions and classes. -#[inline] pub fn register_project_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_validation::register(module) + module.add_function(wrap_pyfunction!( + validate_discovered_project_inputs, + module + )?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs index 72d5f0368a..b241d7adc3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs @@ -1,17 +1,22 @@ //! The project or local configuration file path, rejecting an unconverted legacy file. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::config_error; +use crate::discovery::_helpers::project::project_config::file_index; +use crate::discovery::_helpers::project::project_config::files; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; /// The project or local configuration file path, rejecting an unconverted legacy file. -#[inline] +#[pyfunction] pub fn resolve_config_file_path<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, local: bool, ) -> PyResult> { - project_config::resolve_config_file_path(py, project_dir, local) + let files = files(project_dir)?; + match resolve_config_file(&files.native, local) { + Ok(file) => Ok(files.python[file_index(file)].clone()), + Err(failure) => Err(config_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs index c8f03c42c9..c541f47b6b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs @@ -1,13 +1,19 @@ //! The CSV reader settings a seed declaration authors, without the dialect check. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_facts::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_discovery::project_validation::models::CsvSettings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The CSV reader settings a seed declaration authors, without the dialect check. -#[inline] pub fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { - project_facts::seed_csv_settings(entry) + let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; + Ok(CsvSettings { + delimiter: optional_text(&settings, "delimiter")?, + quotechar: optional_text(&settings, "quotechar")?, + escapechar: optional_text(&settings, "escapechar")?, + doublequote: native_field(&settings, "doublequote")?.extract()?, + skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, + dialect_error: None, + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs index aadcc56de9..77509601f3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs @@ -1,12 +1,10 @@ //! `str(Path(...).stem)` of the path attribute `name`. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::path_stem::path_stem; /// `str(Path(...).stem)` of the path attribute `name`. -#[inline] pub fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { - project_facts::stem(owner, name) + path_stem(&native_field(owner, name)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs index 9b0eb51bbd..b002613ca3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs @@ -1,17 +1,21 @@ //! `validate_local_sql_analysis_policy` from the project requirement and the local setting. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::config_error; +use crate::discovery::_helpers::project::project_config::files; +use pyo3::prelude::{Bound, PyAny, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; /// `validate_local_sql_analysis_policy` from the project requirement and the local setting. -#[inline] +#[pyfunction] pub fn validate_local_sql_analysis( py: Python<'_>, project_dir: &Bound<'_, PyAny>, flags: (bool, bool, bool), ) -> Result<(), PyErr> { - project_config::validate_local_sql_analysis(py, project_dir, flags) + let files = files(project_dir)?; + match local_sql_analysis_failure(&files.native, flags) { + None => Ok(()), + Some(failure) => Err(config_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs index f8785d84ce..8e420b2c35 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs @@ -1,12 +1,45 @@ //! The Python exception a failed project validation raises. -use crate::discovery::_helpers::project::project_validation; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error}; +use crate::discovery::_helpers::project::project_validation::BUILTINS_MODULE; +use crate::discovery::_helpers::project::project_validation::CSV_DIALECT_ERRORS; +use crate::discovery::_helpers::project::project_validation::CSV_MODULE; +use crate::discovery::_helpers::project::project_validation::PATHLIB_MODULE; +use crate::discovery::_helpers::project::project_validation::raised; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; use sqlbuild_discovery::project_validation::models::ValidationFailure; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The Python exception a failed project validation raises. -#[inline] pub fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { - project_validation::validation_error(py, failure) + match failure { + ValidationFailure::Discovery(failure) => failure_error(py, failure), + ValidationFailure::Unreadable { + absolute_path, + failure, + } => match py + .import(PATHLIB_MODULE) + .and_then(|module| native_field(&module, "Path")) + .and_then(|path| path.call1((absolute_path,))) + { + Ok(file_path) => read_error(failure, &file_path), + Err(error) => error, + }, + ValidationFailure::Csv(message) => raised( + py.import(CSV_MODULE) + .and_then(|module| native_field(&module, "Error")) + .and_then(|class| class.call1((message,))), + ), + ValidationFailure::CsvDialect(error) => { + if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { + return PyKeyError::new_err(error.class.to_string()); + } + raised( + py.import(BUILTINS_MODULE) + .and_then(|module| native_field(&module, error.class.as_str())) + .and_then(|class| class.call1((error.message,))), + ) + } + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs index d3b3a3bf12..b97eb0c921 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs @@ -1,13 +1,18 @@ //! Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings; +use sqlbuild_python_base::native_host::main::write_console::write_console; /// Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -#[inline] pub fn warn_deprecated_settings(py: Python<'_>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - project_config::warn_deprecated_settings(py, project_dir) + let files = files(project_dir)?; + match deprecated_setting_warnings(&files.native) { + Ok(warnings) => warnings + .iter() + .try_for_each(|warning| write_console(py, "stderr", &format!("{warning}\n"))), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs index b524063a91..65d6a7bf7c 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs @@ -1,9 +1,13 @@ //! The 1-based line of the outermost `- name: ` entry, if present. -use crate::discovery::_helpers::project::locations; +use pyo3::pyfunction; +use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; /// The 1-based line of the outermost `- name: ` entry, if present. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, contents, name))] pub fn yaml_entry_line(contents: &str, name: &str) -> Option { - locations::yaml_entry_line(contents, name) + yaml_entry_lines(contents) + .into_iter() + .find_map(|(entry, line)| (entry == name).then_some(line)) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs index acb768a560..bf94e79267 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs @@ -1,15 +1,36 @@ //! Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -use crate::discovery::_helpers::python_files::imported_providers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ProjectDir, tuple_of}; +use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::takes_recall_writer; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; /// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_event_exporter_functions<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_providers::discover_event_exporter_functions(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { + path: project_dir.clone(), + }; + if let Some(cached) = project.cached_extensions()? { + return cached.get_item(1); + } + let mut ordinary: Vec> = Vec::new(); + for declaration in discover_event_exporter_declarations(project_dir.clone())?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + if !takes_recall_writer(&declaration)? { + ordinary.push(declaration); + } + } + bind_event_exporter_declarations( + tuple_of(py, ordinary)?, + providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, + project_dir, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs index f046e4db10..ff31f4be65 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs @@ -1,9 +1,7 @@ //! Decorated model lifecycle hook functions under hooks/python/. use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Decorated model lifecycle hook functions under hooks/python/. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs index 2b665bfcc1..e370b7e2c0 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs @@ -1,15 +1,56 @@ //! Custom materialization Python files under materializations/. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, tuple_of, +}; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Custom materialization Python files under materializations/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_materialization_files<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_python_nodes::discover_materialization_files(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + if !project + .child("materializations")? + .call_method0("is_dir")? + .is_truthy()? + { + return tuple_of(py, Vec::new()); + } + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut files: Vec> = Vec::new(); + for file_path in project.module_files("materializations", false)? { + let materialize: Bound<'py, PyAny> = + project.load(UserCodeHost::LoadMaterializeFunction, &file_path)?; + let usages: Bound<'py, PyAny> = if materialize.is_none() { + tuple_of(py, Vec::new())? + } else { + provider_usages(&materialize, &providers)? + }; + files.push(discovered( + py, + RecordId::DiscoveredMaterializationFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", project.relative(&file_path)?), + ("name", native_field(&file_path, "stem")?), + ("provider_usages", usages), + ], + )?); + } + tuple_of(py, files) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs index c261a096ca..7c4625cbfc 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs @@ -1,9 +1,7 @@ //! Provider classes under providers/. use crate::discovery::_helpers::python_files::imported_providers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Provider classes under providers/. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs index 89093f5de4..2cb0f9887c 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs @@ -1,15 +1,34 @@ //! Discover the `@udf(...)` Python function files under `functions/python/`. -use crate::discovery::_helpers::python_files::python_function_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::python_files::python_function_files::function_record; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::main::python_function_files::python_function_files; +use sqlbuild_discovery::models::{DiscoveredFile, ProjectRoot}; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover the `@udf(...)` Python function files under `functions/python/`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_python_function_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - python_function_files::discover_python_function_files(project_dir) + in_snapshot(project_dir, || { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root: ProjectRoot = project_root(project_dir)?; + let inner: &ProjectTree = &tree.get().inner; + let files: Vec> = py + .detach(|| python_function_files(&root, inner)) + .map_err(|failure| stage_error(inner, project_dir, failure))?; + let records: Vec> = files + .into_iter() + .map(|file| function_record(project_dir, inner, file)) + .collect::>()?; + PyTuple::new(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs index ab6f9875ce..446f0a8e07 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs @@ -1,15 +1,73 @@ //! Decorated Python DAG node functions under python/. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, shim, tuple_of, +}; +use crate::discovery::_helpers::python_files::imported_python_nodes::Buckets; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_core::constants::PYTHON_NODE_ROOT; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use sqlbuild_python_base::records::types::RecordId; /// Decorated Python DAG node functions under python/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_python_node_functions<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_python_nodes::discover_python_node_functions(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut buckets = Buckets { + nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], + audit_factories: Vec::new(), + }; + let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; + if node_root.call_method0("is_dir")?.is_truthy()? { + let saved: Bound<'py, PyAny> = call_with( + &shim(py, UserCodeHost::InstallPythonRootPackage)?, + &[ + ("node_root", node_root), + ("project_dir", project.path.clone()), + ], + )?; + let discovered_nodes: PyResult<()> = + within_identity_pass(py, || buckets.discover_nodes(&project, &providers)); + call_with( + &shim(py, UserCodeHost::RestorePythonRootModules)?, + &[("saved_modules", saved)], + )?; + discovered_nodes?; + } + let [loaders, tasks, assets, checks] = buckets.nodes; + discovered( + py, + RecordId::DiscoveredPythonNodeFunctions, + &[ + ("loaders", tuple_of(py, loaders)?), + ("tasks", tuple_of(py, tasks)?), + ("assets", tuple_of(py, assets)?), + ("checks", tuple_of(py, checks)?), + ( + "audit_factories", + tuple_of( + py, + buckets + .audit_factories + .into_iter() + .map(|(_, _, record)| record) + .collect(), + )?, + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs index 7cfe250281..afc6216ac9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs @@ -1,11 +1,10 @@ //! Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -use crate::discovery::_helpers::python_files::imported_providers; +use crate::discovery::_helpers::python_files::imported_providers::RECALL_WRITER; use pyo3::prelude::Bound; use pyo3::types::PyType; /// Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -#[inline] pub fn install_recall_writer(writer: &Bound<'_, PyType>) { - imported_providers::install_recall_writer(writer) + let _ = RECALL_WRITER.get_or_init(writer.py(), || writer.clone().unbind()); } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs index 49f66190e2..5a49c753d8 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs @@ -1,10 +1,7 @@ //! The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs index 24098feb21..b38ff8467d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs @@ -1,13 +1,15 @@ //! `{provider.name: provider for provider in providers}`. -use crate::discovery::_helpers::python_files::imported_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `{provider.name: provider for provider in providers}`. -#[inline] pub fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { - imported_modules::provider_by_name(providers) + let by_name: Bound<'py, PyDict> = PyDict::new(providers.py()); + for provider in providers.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + by_name.set_item(native_field(&provider, "name")?, provider)?; + } + Ok(by_name) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs index 5685f5b30d..2b39ba1923 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs @@ -1,16 +1,21 @@ //! The provider parameters of one loaded user function. -use crate::discovery::_helpers::python_files::imported_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::shim; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; /// The provider parameters of one loaded user function. -#[inline] pub fn provider_usages<'py>( function: &Bound<'py, PyAny>, provider_by_name: &Bound<'py, PyDict>, ) -> PyResult> { - imported_modules::provider_usages(function, provider_by_name) + call_with( + &shim(function.py(), UserCodeHost::ProviderUsages)?, + &[ + ("function", function.clone()), + ("provider_by_name", provider_by_name.clone().into_any()), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs index ea02864cab..7ba126ee98 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs @@ -1,12 +1,25 @@ //! Register the imported python nodes functions and classes. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::_helpers::python_files::imported_python_nodes::discover_hook_functions; +use crate::discovery::main::python_files::discover_event_exporter_functions::discover_event_exporter_functions; +use crate::discovery::main::python_files::discover_materialization_files::discover_materialization_files; +use crate::discovery::main::python_files::discover_python_node_functions::discover_python_node_functions; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the imported python nodes functions and classes. -#[inline] pub fn register_imported_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { - imported_python_nodes::register(module) + module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_provider_classes, module)?)?; + module.add_function(wrap_pyfunction!( + discover_event_exporter_declarations, + module + )?)?; + module.add_function(wrap_pyfunction!(bind_event_exporter_declarations, module)?)?; + module.add_function(wrap_pyfunction!(discover_event_exporter_functions, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs index 8913e46dae..ef78cd7906 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs @@ -1,12 +1,10 @@ //! Register the python function files functions and classes. -use crate::discovery::_helpers::python_files::python_function_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::python_files::discover_python_function_files::discover_python_function_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python function files functions and classes. -#[inline] pub fn register_python_function_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_function_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_function_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs index 3c7f62435a..01dd4787f3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs @@ -1,12 +1,10 @@ //! Register the python hook files functions and classes. -use crate::discovery::_helpers::python_files::python_hook_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::python_hook_files::discover_python_hook_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python hook files functions and classes. -#[inline] pub fn register_python_hook_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_hook_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_hook_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs index 8939ee1aec..35f50d5eeb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs @@ -1,12 +1,10 @@ //! Register the python module files functions and classes. -use crate::discovery::_helpers::python_files::python_module_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::python_module_files::discover_python_module_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python module files functions and classes. -#[inline] pub fn register_python_module_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_module_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_module_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs index af1736586b..b176075c6d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs @@ -1,12 +1,28 @@ //! Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -use crate::discovery::_helpers::python_files::python_paths; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::discovery_error; +use crate::discovery::_helpers::python_files::python_paths::PYTHON_PATH_HELP; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; +use std::path::PathBuf; /// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -#[inline] pub fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - python_paths::reject_unsupported_python_paths(project_dir) + let root: PathBuf = project_dir.extract()?; + let unsupported: Vec = unsupported_python_paths(&root) + .into_iter() + .map(|parts| parts.join(std::path::MAIN_SEPARATOR_STR)) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + Err(discovery_error( + project_dir.py(), + "python_path", + format!( + "Unsupported project Python path(s): {}", + unsupported.join(", ") + ), + Some(PYTHON_PATH_HELP.to_owned()), + )) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs index e4962b97a9..4d5a007a95 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs @@ -1,12 +1,26 @@ //! Detect a project-level adapter.py without importing it. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::path_class; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Detect a project-level adapter.py without importing it. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_adapter_file<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - declaration_records::discover_adapter_file(project_dir) + let py: Python<'py> = project_dir.py(); + let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; + if !file_path.call_method0("is_file")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + new_record_with( + py, + RecordId::DiscoveredAdapterFile, + &[ + ("file_path", file_path), + ("relative_path", path_class(py)?.call1(("adapter.py",))?), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs index 19af629ec1..e68dd482e7 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs @@ -1,16 +1,22 @@ //! Discover generic and singular audit SQL files in their project-wide and scoped roles. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover generic and singular audit SQL files in their project-wide and scoped roles. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_audit_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_audit_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Audits, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs index 05ec7b9656..828c180472 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs @@ -1,17 +1,23 @@ //! Discover global and scoped constant declaration files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped constant declaration files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] pub fn discover_constant_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_constant_files(project_dir, on_fault, isolate_declaration_kind) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Constants, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs index d9eacb8305..cf228ee104 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs @@ -1,17 +1,23 @@ //! Discover global and scoped enum declaration files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped enum declaration files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] pub fn discover_enum_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_enum_files(project_dir, on_fault, isolate_declaration_kind) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Enums, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs index fbd7a0a5da..55bac8b128 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs @@ -1,16 +1,22 @@ //! Discover global and scoped project macro Python files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped project macro Python files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] pub fn discover_macro_files<'py>( project_dir: &Bound<'py, PyAny>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_macro_files(project_dir, isolate_declaration_kind) + discovered( + &faults(project_dir, None), + CollectionKind::Macros, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs index 2ac5e14538..d2b189514b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs @@ -1,17 +1,45 @@ //! Discover the (selected) model files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::model_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; +use crate::discovery::models::ModelFileSelection; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Discover the (selected) model files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn discover_model_files<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - model_records::discover_model_files(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_model_files", + options, + ( + &[ + "project_dir", + "extract_implicit_alias_columns", + "extract_output_column_locations", + ], + &["selected_model_names", "on_fault"], + ), + )?; + discovered_model_files( + &arguments.value("project_dir")?, + ( + arguments + .value("extract_implicit_alias_columns")? + .extract()?, + arguments + .value("extract_output_column_locations")? + .extract()?, + ), + ModelFileSelection { + selected_model_names: Some(arguments.value("selected_model_names")?), + on_fault: Some(arguments.value("on_fault")?), + }, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs index 9c886cd419..be85d1fae1 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs @@ -1,16 +1,22 @@ //! Discover project-wide and scoped reusable model schemas. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover project-wide and scoped reusable model schemas. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_model_schema_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_model_schema_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::ModelSchemas, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs index 1dd47a45dc..517373e7a2 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs @@ -1,16 +1,44 @@ //! Discover scenario files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::sql_test_records::discovered_files; +use crate::discovery::_helpers::records::sql_test_records::path_class; +use crate::discovery::_helpers::records::sql_test_records::records; +use crate::discovery::_helpers::records::sql_test_records::scenario_file; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::models::FileOutcome; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::models::DiscoveredScenarioFile; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover scenario files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_scenario_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - sql_test_records::discover_scenario_files(project_dir, on_fault) + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_scenarios)?; + let tree: &ProjectTree = &tree.get().inner; + let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files + .into_iter() + .map(|file| { + Ok(( + path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, + file.outcome, + )) + }) + .collect::>()?; + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + scenario_file, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs index 3481d9f5bd..b1abfbd1c7 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs @@ -1,15 +1,79 @@ //! Discover model schema.yml and seed declaration files natively. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::{discovery_error, stage_error}; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::yaml_records::ListingTree; +use crate::discovery::_helpers::records::yaml_records::SCHEMA_KIND; +use crate::discovery::_helpers::records::yaml_records::YamlKind; +use crate::discovery::_helpers::records::yaml_records::listing_tree; +use crate::discovery::_helpers::records::yaml_records::load_files; +use crate::discovery::_helpers::records::yaml_records::path_class; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use sqlbuild_discovery::yaml_files::models::YamlPathFailure; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Discover model schema.yml and seed declaration files natively. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_schema_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - yaml_records::discover_schema_files(project_dir) + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = match schema_yaml_paths(listing.tree()) { + Ok(paths) => paths, + Err(YamlPathFailure::Stage(failure)) => { + return Err(stage_error(listing.tree(), project_dir, failure)); + } + Err(YamlPathFailure::YamlSeedDeclaration(path)) => { + let relative: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; + return Err(discovery_error( + py, + SCHEMA_KIND, + format!( + "Seed declaration file {} must use .yml; .yaml is not supported", + relative.str()? + ), + None, + )); + } + }; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Schema), + |(relative_path, file_path), built| { + let (contents, loaded) = built?; + let shown: String = file_path.str()?.to_str()?.to_owned(); + records.push(new_record_with( + py, + RecordId::DiscoveredSchemaFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("contents", contents.into_bound_py_any(py)?), + ("model_entries", PyTuple::empty(py).into_any()), + ( + "seed_entries", + parse_schema_document(py, loaded, &shown)?.into_any(), + ), + ], + )?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs index 89c3cc5a60..06db4657df 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs @@ -1,13 +1,15 @@ //! Discover seed CSV files under seeds/. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover seed CSV files under seeds/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_seed_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - declaration_records::discover_seed_files(project_dir) + discovered(&faults(project_dir, None), CollectionKind::Seeds, false) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs index bd3e3d36ef..cbdb40e600 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs @@ -1,16 +1,64 @@ //! Discover source YAML files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::records::yaml_records::ListingTree; +use crate::discovery::_helpers::records::yaml_records::YamlKind; +use crate::discovery::_helpers::records::yaml_records::listing_tree; +use crate::discovery::_helpers::records::yaml_records::load_files; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Discover source YAML files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_source_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - yaml_records::discover_source_files(project_dir, on_fault) + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = source_yaml_paths(listing.tree()) + .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault, + }; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Source), + |(relative_path, file_path), built| { + let record = built.and_then(|(contents, loaded)| { + let shown: String = file_path.str()?.to_str()?.to_owned(); + new_record_with( + py, + RecordId::DiscoveredSourceFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", contents.into_bound_py_any(py)?), + ( + "source_entries", + parse_sources_document(py, loaded, &shown)?.into_any(), + ), + ], + ) + }); + records.extend(faults.kept(&relative_path, record)?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs index 2cb6438195..49665dc655 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs @@ -1,16 +1,22 @@ //! Discover SQL function files under functions/sql/. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover SQL function files under functions/sql/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_sql_function_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_sql_function_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlFunctions, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs index 6d970cbb61..94416431aa 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs @@ -1,16 +1,22 @@ //! Discover named SQL lifecycle hooks in their project-wide and scoped roles. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover named SQL lifecycle hooks in their project-wide and scoped roles. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_sql_hook_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_sql_hook_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlHooks, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs index 3f73ba9034..69ef1a190f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs @@ -1,17 +1,46 @@ //! Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::sql_test_records::discovered_files; +use crate::discovery::_helpers::records::sql_test_records::path_class; +use crate::discovery::_helpers::records::sql_test_records::records; +use crate::discovery::_helpers::records::sql_test_records::test_file; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::models::FileOutcome; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::models::DiscoveredSqlTestFile; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] pub fn discover_test_files<'py>( project_dir: &Bound<'py, PyAny>, selected_paths: Option>, on_fault: Option>, ) -> PyResult> { - sql_test_records::discover_test_files(project_dir, selected_paths, on_fault) + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_tests)?; + let tree: &ProjectTree = &tree.get().inner; + let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); + for file in files { + let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); + if let Some(selected) = &selected_paths + && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? + { + continue; + } + kept.push((path_class(py)?.call1((relative,))?, file.outcome)); + } + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + test_file, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs index 9df1381eaf..8e9439d82b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs @@ -1,18 +1,47 @@ //! The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -use crate::discovery::_helpers::records::model_records; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::records::model_records::ModelItem; +use crate::discovery::_helpers::records::model_records::model_items; use crate::discovery::models::ModelFileSelection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; +use sqlbuild_discovery::model_files::models::ModelFileOptions; /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -#[inline] pub fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, extraction: (bool, bool), selection: ModelFileSelection<'py>, ) -> PyResult> { - model_records::discovered_model_files(project_dir, extraction, selection) + let py: Python<'py> = project_dir.py(); + let options: ModelFileOptions = model_file_options( + extraction, + sqlbuild_python_base::compile_context::main::delta::injection::injection() + .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), + ); + let items: Vec> = model_items(project_dir, &options)?; + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), + }; + let selected: Option> = selection + .selected_model_names + .filter(|selected| !selected.is_none()); + let mut records: Vec> = Vec::with_capacity(items.len()); + for (relative_path, built) in items { + if let Some(selected) = &selected + && !selected.contains(relative_path.name(py)?)? + { + continue; + } + match built { + Ok(record) => records.push(record), + Err(error) => { + records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); + } + } + } + record_tuple(py, records) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs index fd6cd14803..c222efacc5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs @@ -1,15 +1,17 @@ //! Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -use crate::discovery::_helpers::records::description_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::discovery::_helpers::records::description_inputs::discover_model_description_inputs; +use crate::model_config::main::model_nodes::externally_described_models::externally_described_models as described_models; +use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; /// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -#[inline] pub fn externally_described_models<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - description_inputs::externally_described_models(project_dir) + let py: Python<'py> = project_dir.py(); + let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; + let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; + PyFrozenSet::new(py, described_models(inputs, schemas)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs index c10bd475da..26e80582f5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs @@ -1,16 +1,25 @@ //! Parse in-memory SQL test contents into ordered blocks, as test discovery does. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::records::sql_test_records::options; +use crate::discovery::_helpers::records::sql_test_records::test_blocks; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyStringMethods, PyTuple}; +use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; +use sqlbuild_discovery::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; /// Parse in-memory SQL test contents into ordered blocks, as test discovery does. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, contents, file_path))] pub fn parse_sql_test_file<'py>( contents: String, file_path: &Bound<'py, PyAny>, ) -> PyResult> { - sql_test_records::parse_sql_test_file(contents, file_path) + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options()?; + let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); + let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; + test_blocks(file_path, file.blocks, file.failure) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs index 2c1eadeda9..d420097c19 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs @@ -1,18 +1,17 @@ //! `text` as the Python string `cache` shares with every reader, built on first read. -use crate::discovery::_helpers::records::model_file_record; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, Py, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyString; /// `text` as the Python string `cache` shares with every reader, built on first read. -#[inline] pub fn projected_text<'py>( py: Python<'py>, cache: &PyOnceLock>, text: &str, ) -> Bound<'py, PyString> { - model_file_record::projected_text(py, cache, text) + cache + .get_or_init(py, || PyString::new(py, text).unbind()) + .bind(py) + .clone() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs index 19486f7f7c..a18ff416eb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs @@ -1,12 +1,29 @@ //! Register the declaration records functions and classes. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::records::discover_adapter_file::discover_adapter_file; +use crate::discovery::main::records::discover_audit_files::discover_audit_files; +use crate::discovery::main::records::discover_constant_files::discover_constant_files; +use crate::discovery::main::records::discover_enum_files::discover_enum_files; +use crate::discovery::main::records::discover_macro_files::discover_macro_files; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discover_seed_files::discover_seed_files; +use crate::discovery::main::records::discover_sql_function_files::discover_sql_function_files; +use crate::discovery::main::records::discover_sql_hook_files::discover_sql_hook_files; +use crate::discovery::main::records::retained_discovery_session::retained_discovery_session; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration records functions and classes. -#[inline] pub fn register_declaration_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_records::register(module) + module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; + module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs index f633072057..a0e2c138eb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs @@ -1,12 +1,11 @@ //! Register the description inputs functions and classes. -use crate::discovery::_helpers::records::description_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::description_inputs::discover_model_description_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the description inputs functions and classes. -#[inline] pub fn register_description_inputs(module: &Bound<'_, PyModule>) -> PyResult<()> { - description_inputs::register(module) + module.add_function(wrap_pyfunction!(discover_model_description_inputs, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs index 1ca1846266..1daae8294a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs @@ -1,12 +1,10 @@ //! Register the model records functions and classes. -use crate::discovery::_helpers::records::model_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::records::discover_model_files::discover_model_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model records functions and classes. -#[inline] pub fn register_model_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_records::register(module) + module.add_function(wrap_pyfunction!(discover_model_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs index ab0033786d..ce02a5a2f9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs @@ -1,12 +1,16 @@ //! Register the sql test records functions and classes. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::sql_test_records::parse_sql_scenario_file; +use crate::discovery::main::records::discover_scenario_files::discover_scenario_files; +use crate::discovery::main::records::discover_test_files::discover_test_files; +use crate::discovery::main::records::parse_sql_test_file::parse_sql_test_file; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql test records functions and classes. -#[inline] pub fn register_sql_test_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_test_records::register(module) + module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs index 9f117e1b44..ce8112b7c9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs @@ -1,12 +1,16 @@ //! Register the yaml records functions and classes. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::yaml_records::load_yaml_document; +use crate::discovery::_helpers::records::yaml_records::load_yaml_files; +use crate::discovery::main::records::discover_schema_files::discover_schema_files; +use crate::discovery::main::records::discover_source_files::discover_source_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the yaml records functions and classes. -#[inline] pub fn register_yaml_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - yaml_records::register(module) + module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_document, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs index 75b0fd3daf..5c6ea9667e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs @@ -1,14 +1,21 @@ //! The session this discovery pass read its declaration files with, if any. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::SESSION_MEMO_KEY; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::snapshots::snapshot::snapshot_memo; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDictMethods; /// The session this discovery pass read its declaration files with, if any. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn retained_discovery_session<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - declaration_records::retained_discovery_session(project_dir) + let py: Python<'py> = project_dir.py(); + Ok(snapshot_memo(project_dir)? + .get_item(SESSION_MEMO_KEY)? + .filter(|session| session.is_instance_of::()) + .unwrap_or_else(|| py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs index 611b9a2b24..54342c1938 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs @@ -1,12 +1,15 @@ //! Share one directory snapshot across every discovery call for `project_dir` inside the scope. -use crate::discovery::_helpers::snapshots::snapshot; use crate::discovery::models::NativeDirectorySnapshotScope; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; +use pyo3::pyfunction; /// Share one directory snapshot across every discovery call for `project_dir` inside the scope. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn directory_snapshot_scope(project_dir: Bound<'_, PyAny>) -> NativeDirectorySnapshotScope { - snapshot::directory_snapshot_scope(project_dir) + NativeDirectorySnapshotScope { + project_dir: project_dir.unbind(), + token: None, + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs index bbd45acb50..41052eb99f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs @@ -1,14 +1,15 @@ //! Parse the scope-relevant authored roots while retaining independent file faults. -use crate::discovery::_helpers::snapshots::scope_snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::scope_snapshot::tolerant_discovery; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Parse the scope-relevant authored roots while retaining independent file faults. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_scope_snapshot<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - scope_snapshot::discover_scope_snapshot(project_dir) + in_snapshot(project_dir, || tolerant_discovery(project_dir)) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs index 860c66ec23..7dbe88f293 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs @@ -1,12 +1,25 @@ //! Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::snapshot::SURROGATE_PASS; +use crate::discovery::_helpers::snapshots::snapshot::UTF16_ENCODING; +use crate::discovery::_helpers::snapshots::snapshot::escaped_unit; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; use pyo3::types::PyString; /// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -#[inline] pub fn escaped(text: &Bound<'_, PyString>) -> PyResult { - snapshot::escaped(text) + let encoded: Vec = text + .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? + .extract()?; + let units = encoded + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)); + Ok(char::decode_utf16(units) + .map(|decoded| match decoded { + Ok(character) => character.to_string(), + Err(error) => escaped_unit(error.unpaired_surrogate()), + }) + .collect()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs index c21854391c..6205158bad 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs @@ -1,9 +1,7 @@ //! Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs index de088e3cb1..230e013c51 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs @@ -1,12 +1,10 @@ //! Register the scope snapshot functions and classes. -use crate::discovery::_helpers::snapshots::scope_snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::snapshots::discover_scope_snapshot::discover_scope_snapshot; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope snapshot functions and classes. -#[inline] pub fn register_scope_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_snapshot::register(module) + module.add_function(wrap_pyfunction!(discover_scope_snapshot, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs index e29fc29827..6976e44a1f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs @@ -1,12 +1,15 @@ //! Register the snapshot functions and classes. -use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; +use crate::discovery::main::snapshots::directory_snapshot_scope::directory_snapshot_scope; +use crate::discovery::models::NativeDirectorySnapshotScope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the snapshot functions and classes. -#[inline] pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { - snapshot::register(module) + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(directory_snapshot_scope, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs index bf17b61bf6..b84aaf6c0b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs @@ -1,12 +1,19 @@ //! `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::main::sources::provider_uses::provider_uses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field; /// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -#[inline] pub fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { - provider_secrets::consumer_uses(consumers) + let mut uses: Vec<(String, String)> = Vec::new(); + for consumer in consumers.try_iter()? { + let consumer: Bound<'_, PyAny> = consumer?; + let name: String = field(&consumer, "name")?.extract()?; + uses.extend(provider_uses( + &field(&consumer, "provider_usages")?, + Some(&name), + )?); + } + Ok(uses) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs index 95d0bc0f90..f9ee98d687 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs @@ -1,9 +1,8 @@ //! Whether the running invocation passed `--allow-missing-secrets`. -use crate::discovery::_helpers::sources::provider_secrets; +use crate::discovery::_helpers::sources::provider_secrets::INVOCATION_ALLOWS_MISSING_SECRETS; /// Whether the running invocation passed `--allow-missing-secrets`. -#[inline] pub fn invocation_allows_missing_secrets() -> bool { - provider_secrets::invocation_allows_missing_secrets() + INVOCATION_ALLOWS_MISSING_SECRETS.get() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs index 915c41e145..3f64b315b8 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -1,15 +1,25 @@ //! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -use crate::discovery::_helpers::sources::orchestration_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::sources::orchestration_files::orchestration_error; +use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_discovery::tree::models::ProjectTree; +use std::path::PathBuf; /// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -#[inline] pub fn orchestration_declarations( project_dir: &Bound<'_, PyAny>, ) -> PyResult { - orchestration_files::orchestration_declarations(project_dir) + let py: Python<'_> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let tree = ProjectTree::new(&root); + let version = py.version_info(); + let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); + let inventory = + inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + Ok(inventory) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs index 872118c52d..70ce058b68 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs @@ -1,15 +1,15 @@ //! `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::provider_secrets::usage_pair; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -#[inline] pub fn provider_uses( usages: &Bound<'_, PyAny>, consumer: Option<&str>, ) -> PyResult> { - provider_secrets::provider_uses(usages, consumer) + usages + .try_iter()? + .map(|usage| usage_pair(&usage?, consumer)) + .collect() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs index 407e5eb323..ea23bafb31 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs @@ -1,12 +1,13 @@ //! Register the declaration files functions and classes. -use crate::discovery::_helpers::sources::declaration_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::sources::declaration_files::parse_declaration_contents; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration files functions and classes. -#[inline] pub fn register_declaration_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_files::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_declaration_contents, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs index 37f5ed9420..325ad3623f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs @@ -1,12 +1,12 @@ //! Register the yaml entries functions and classes. -use crate::discovery::_helpers::sources::yaml_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; +use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the yaml entries functions and classes. -#[inline] pub fn register_yaml_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { - yaml_entries::register(module) + module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; + module.add_function(wrap_pyfunction!(parse_sources_document, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs index 7342258da8..652eafdb1e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs @@ -1,9 +1,7 @@ //! Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs index 5d4fc617cf..295e5b35ef 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs @@ -1,12 +1,16 @@ //! The providers the project's lifecycle and command-output sinks use, which run with every command. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::provider_secrets::SINK_COLLECTIONS; +use crate::discovery::main::sources::consumer_uses::consumer_uses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The providers the project's lifecycle and command-output sinks use, which run with every command. -#[inline] pub fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { - provider_secrets::sink_uses(discovered) + let mut uses: Vec<(String, String)> = Vec::new(); + for collection in SINK_COLLECTIONS { + if let Some(sinks) = discovered.getattr_opt(collection)? { + uses.extend(consumer_uses(&sinks)?); + } + } + Ok(uses) } diff --git a/crates/sqlbuild-python-front/src/discovery/models.rs b/crates/sqlbuild-python-front/src/discovery/models.rs index df8f05bc3b..bbe037f371 100644 --- a/crates/sqlbuild-python-front/src/discovery/models.rs +++ b/crates/sqlbuild-python-front/src/discovery/models.rs @@ -6,8 +6,8 @@ use crate::discovery::_helpers::records::model_file_record::HeaderValues; use crate::discovery::_helpers::records::model_file_record::Locations; use crate::discovery::_helpers::records::model_file_record::ModelFileFields; use crate::discovery::_helpers::records::model_file_record::ModelFilePaths; -use crate::discovery::_helpers::records::model_file_record::projected_text; use crate::discovery::_helpers::sources::provider_secrets::INVOCATION_ALLOWS_MISSING_SECRETS; +use crate::discovery::main::records::projected_text::projected_text; use pyo3::FromPyObject; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyTypeError; diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs index 11e0d1c083..c80df95845 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs @@ -2,24 +2,23 @@ use std::sync::{Mutex, MutexGuard, PoisonError}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::pyfunction; use sqlbuild_python_base::compile_context::main::delta::injection::injection; /// Whether a compile claimed the first-compile slot, and whether a delta compile only borrowed it. #[derive(Debug)] -struct FirstCompile { +pub(crate) struct FirstCompile { claimed: bool, - lent_to_delta: bool, + pub(crate) lent_to_delta: bool, } -static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { +pub(crate) static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { claimed: false, lent_to_delta: false, }); -fn locked(slot: &Mutex) -> MutexGuard<'_, T> { +pub(crate) fn locked(slot: &Mutex) -> MutexGuard<'_, T> { slot.lock().unwrap_or_else(PoisonError::into_inner) } @@ -41,12 +40,3 @@ pub(crate) fn claim_first_compile() -> bool { first.claimed = true; granted } - -/// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -pub(crate) fn keep_delta_claim() { - locked(&FIRST_COMPILE).lent_to_delta = false; -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(claim_first_compile, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs index 45cc26a97a..a593eabe83 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs @@ -1,16 +1,16 @@ //! The module file digests a macro call store records and checks before reuse. +use crate::macro_bridge::main::text::text; use std::path::PathBuf; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::module_digests::main::module_digests_metadata::module_digests_metadata as metadata_text; use sqlbuild_core::json::models::JsonValue; -use crate::macro_bridge::_helpers::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::main::digest_files::digest_files; use sqlbuild_python_base::frontier::main::emit_error::emit_error; @@ -48,7 +48,7 @@ pub(crate) fn digest_module_files<'py>( /// Encode module file digests for the store's metadata. #[pyfunction] -fn module_digests_metadata<'py>( +pub(crate) fn module_digests_metadata<'py>( py: Python<'py>, digests: &Bound<'py, PyAny>, ) -> PyResult> { @@ -83,7 +83,7 @@ fn digest_row_check(row: &Bound<'_, PyAny>) -> PyResult { /// Return the stored module digests when every module file still has them, else None. #[pyfunction] #[pyo3(signature = (*, metadata, known))] -fn unchanged_module_digests<'py>( +pub(crate) fn unchanged_module_digests<'py>( py: Python<'py>, metadata: &Bound<'py, PyAny>, known: &Bound<'py, PyAny>, @@ -125,9 +125,3 @@ fn unchanged_module_digests<'py>( None }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(digest_module_files, module)?)?; - module.add_function(wrap_pyfunction!(module_digests_metadata, module)?)?; - module.add_function(wrap_pyfunction!(unchanged_module_digests, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs index 21573a9fe5..49b76e2977 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs @@ -1,24 +1,18 @@ //! Value-based text of everything a stored macro call result depends on besides its call text. +use crate::macro_bridge::main::text::text; +use pyo3::create_exception; use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{ PyBool, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyStringMethods, PyTuple, PyTypeMethods, }; -use pyo3::{create_exception, pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::boundary::main::utf16_units::utf16_units; use sqlbuild_cache::macro_store_keys::main::canonical_sort_text::canonical_sort_text; -use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; -use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -33,7 +27,7 @@ const CANONICAL_STACK_GROWTH: usize = 2 * 1024 * 1024; create_exception!(_native, UncacheableValue, pyo3::exceptions::PyException); /// Whether `error` is one that makes a value unkeyable rather than a failure to report. -fn unstable(py: Python<'_>, error: &PyErr) -> bool { +pub(crate) fn unstable(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) || error.is_instance_of::(py) || error.is_instance_of::(py) @@ -48,21 +42,13 @@ fn type_name(kind: &Bound<'_, PyAny>) -> PyResult { )) } -/// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -pub(crate) fn text(value: &Bound<'_, PyString>) -> PyResult { - match value.to_str() { - Ok(text) => Ok(JsonValue::String(text.to_owned())), - Err(_) => utf16_units(value).map(JsonValue::Utf16), - } -} - /// `[tag, value]`. fn tagged(tag: String, value: JsonValue) -> JsonValue { JsonValue::Array(vec![JsonValue::String(tag), value]) } /// The Python types `_canonical` distinguishes. -struct CanonicalTypes<'py> { +pub(crate) struct CanonicalTypes<'py> { enumeration: Bound<'py, PyAny>, decimal: Bound<'py, PyAny>, dates: Bound<'py, PyTuple>, @@ -75,7 +61,7 @@ struct CanonicalTypes<'py> { } impl<'py> CanonicalTypes<'py> { - fn load(py: Python<'py>) -> PyResult { + pub(crate) fn load(py: Python<'py>) -> PyResult { Ok(Self { enumeration: imported(py, "enum", "Enum")?, decimal: imported(py, "decimal", "Decimal")?, @@ -96,7 +82,7 @@ impl<'py> CanonicalTypes<'py> { } /// `_canonical(value)`, on a stack that grows so nesting up to `MAX_DEPTH` cannot overflow it. - fn canonical(&self, value: &Bound<'py, PyAny>, depth: usize) -> PyResult { + pub(crate) fn canonical(&self, value: &Bound<'py, PyAny>, depth: usize) -> PyResult { stacker::maybe_grow(CANONICAL_RED_ZONE, CANONICAL_STACK_GROWTH, || { self.canonical_level(value, depth) }) @@ -218,85 +204,8 @@ impl<'py> CanonicalTypes<'py> { } } -/// Name a macro by its name, file, identity and exact source, or None when it has no key. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn macro_store_token<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "macro_store_token", - options, - (&["loaded", "identity"], &[]), - )?; - let loaded: Bound<'py, PyAny> = arguments.value("loaded")?; - let keyed: PyResult> = (|| { - let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; - let name: JsonValue = text(native_field(&loaded, "name")?.downcast::()?)?; - let path: JsonValue = text( - native_field(&loaded, "relative_path")? - .call_method0("as_posix")? - .downcast::()?, - )?; - let identity: JsonValue = types.canonical(&arguments.value("identity")?, 0)?; - let raw_source: String = native_field(&loaded, "raw_source")?.extract()?; - macro_key((name, path), identity, &raw_source).map_err(emit_error) - })(); - match keyed { - Err(error) if unstable(py, &error) => Ok(None), - other => other, - } -} - -/// Digest the whole context a macro may read, or None when a value has no stable text. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn context_store_token<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "context_store_token", - options, - (&["macro_context", "declarations"], &[]), - )?; - let keyed: PyResult> = (|| { - let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; - let context: JsonValue = canonical_context(&types, &arguments.value("macro_context")?)?; - let declarations: Bound<'py, PyAny> = arguments.value("declarations")?; - let declared: JsonValue = if declarations.is_none() { - JsonValue::Null - } else { - let sorted = imported(py, "builtins", "sorted")?; - let mut parts: Vec = vec![ - types.canonical(&native_field(&declarations, "constants")?, 0)?, - types.canonical(&native_field(&declarations, "enums")?, 0)?, - ]; - for name in ["inaccessible_constants", "inaccessible_enums"] { - let mut items: Vec = Vec::new(); - for item in sorted - .call1((native_field(&declarations, name)?,))? - .try_iter()? - { - items.push(plain_json(&item?)?); - } - parts.push(JsonValue::Array(items)); - } - JsonValue::Array(parts) - }; - context_key(&JsonValue::Array(vec![context, declared])).map_err(emit_error) - })(); - match keyed { - Err(error) if unstable(py, &error) => Ok(None), - other => other, - } -} - /// `_canonical_context`: the context's type and each field, the value renderer by its type. -fn canonical_context<'py>( +pub(crate) fn canonical_context<'py>( types: &CanonicalTypes<'py>, macro_context: &Bound<'py, PyAny>, ) -> PyResult { @@ -322,7 +231,7 @@ fn canonical_context<'py>( } /// An inaccessible declaration name, which is always a string, as JSON. -fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { match value.downcast::() { Ok(string) => text(string), Err(_) => Err(pyo3::exceptions::PyTypeError::new_err(format!( @@ -331,8 +240,3 @@ fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { ))), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(macro_store_token, module)?)?; - module.add_function(wrap_pyfunction!(context_store_token, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs index be5b01f271..6224727df0 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs @@ -1,16 +1,60 @@ //! Digest the whole context a macro may read, or None when a value has no stable text. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::macro_bridge::_helpers::store_keys::CanonicalTypes; +use crate::macro_bridge::_helpers::store_keys::canonical_context; +use crate::macro_bridge::_helpers::store_keys::plain_json; +use crate::macro_bridge::_helpers::store_keys::unstable; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Digest the whole context a macro may read, or None when a value has no stable text. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn context_store_token<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - store_keys::context_store_token(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "context_store_token", + options, + (&["macro_context", "declarations"], &[]), + )?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let context: JsonValue = canonical_context(&types, &arguments.value("macro_context")?)?; + let declarations: Bound<'py, PyAny> = arguments.value("declarations")?; + let declared: JsonValue = if declarations.is_none() { + JsonValue::Null + } else { + let sorted = imported(py, "builtins", "sorted")?; + let mut parts: Vec = vec![ + types.canonical(&native_field(&declarations, "constants")?, 0)?, + types.canonical(&native_field(&declarations, "enums")?, 0)?, + ]; + for name in ["inaccessible_constants", "inaccessible_enums"] { + let mut items: Vec = Vec::new(); + for item in sorted + .call1((native_field(&declarations, name)?,))? + .try_iter()? + { + items.push(plain_json(&item?)?); + } + parts.push(JsonValue::Array(items)); + } + JsonValue::Array(parts) + }; + context_key(&JsonValue::Array(vec![context, declared])).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs index f2e88dec30..ffb7bc548a 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs @@ -1,9 +1,9 @@ //! Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -use crate::macro_bridge::_helpers::first_compile; +use crate::macro_bridge::_helpers::first_compile::FIRST_COMPILE; +use crate::macro_bridge::_helpers::first_compile::locked; /// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -#[inline] pub fn keep_delta_claim() { - first_compile::keep_delta_claim() + locked(&FIRST_COMPILE).lent_to_delta = false; } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs index 1219b76282..211f76335b 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs @@ -1,16 +1,45 @@ //! Name a macro by its name, file, identity and exact source, or None when it has no key. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::macro_bridge::_helpers::store_keys::CanonicalTypes; +use crate::macro_bridge::_helpers::store_keys::unstable; +use crate::macro_bridge::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Name a macro by its name, file, identity and exact source, or None when it has no key. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn macro_store_token<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - store_keys::macro_store_token(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "macro_store_token", + options, + (&["loaded", "identity"], &[]), + )?; + let loaded: Bound<'py, PyAny> = arguments.value("loaded")?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let name: JsonValue = text(native_field(&loaded, "name")?.downcast::()?)?; + let path: JsonValue = text( + native_field(&loaded, "relative_path")? + .call_method0("as_posix")? + .downcast::()?, + )?; + let identity: JsonValue = types.canonical(&arguments.value("identity")?, 0)?; + let raw_source: String = native_field(&loaded, "raw_source")?.extract()?; + macro_key((name, path), identity, &raw_source).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs index d255b60486..5f490e2e9e 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs @@ -1,12 +1,10 @@ //! Register the first compile functions and classes. -use crate::macro_bridge::_helpers::first_compile; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::_helpers::first_compile::claim_first_compile; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the first compile functions and classes. -#[inline] pub fn register_first_compile(module: &Bound<'_, PyModule>) -> PyResult<()> { - first_compile::register(module) + module.add_function(wrap_pyfunction!(claim_first_compile, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs index 5c4403757a..174cb2e3ee 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs @@ -1,12 +1,14 @@ //! Register the module digests functions and classes. -use crate::macro_bridge::_helpers::module_digests; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::_helpers::module_digests::digest_module_files; +use crate::macro_bridge::_helpers::module_digests::module_digests_metadata; +use crate::macro_bridge::_helpers::module_digests::unchanged_module_digests; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the module digests functions and classes. -#[inline] pub fn register_module_digests(module: &Bound<'_, PyModule>) -> PyResult<()> { - module_digests::register(module) + module.add_function(wrap_pyfunction!(digest_module_files, module)?)?; + module.add_function(wrap_pyfunction!(module_digests_metadata, module)?)?; + module.add_function(wrap_pyfunction!(unchanged_module_digests, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs index 8ec80a3394..216dd80429 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs @@ -1,12 +1,12 @@ //! Register the store keys functions and classes. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::main::context_store_token::context_store_token; +use crate::macro_bridge::main::macro_store_token::macro_store_token; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the store keys functions and classes. -#[inline] pub fn register_store_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { - store_keys::register(module) + module.add_function(wrap_pyfunction!(macro_store_token, module)?)?; + module.add_function(wrap_pyfunction!(context_store_token, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs index 4a1a649ee5..82ac2875ce 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs @@ -1,13 +1,14 @@ //! A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyString; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyString, PyStringMethods}; use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::main::utf16_units::utf16_units; /// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -#[inline] pub fn text(value: &Bound<'_, PyString>) -> PyResult { - store_keys::text(value) + match value.to_str() { + Ok(text) => Ok(JsonValue::String(text.to_owned())), + Err(_) => utf16_units(value).map(JsonValue::Utf16), + } } diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs index 94efd6949a..6d4d27fd1b 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs @@ -1,19 +1,17 @@ //! The Python classes a model's `CompileModelConfig` is built from, and building one from a native build. +use crate::model_config::main::config::config_fields::config_fields; +use crate::model_config::main::config::view::view; use crate::model_config::models::ConfigClasses; -use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use crate::model_config::_helpers::config::typed_config::retain_values; use crate::model_config::models::BuiltConfig; -use crate::model_config::types::BuiltConfigValues; const MATERIALIZED_KEY: &str = "materialized"; -const VALUES_FIELD: &str = "values"; -const PERMANENT_TABLE_TYPE: &str = "permanent"; -const TRANSIENT_TABLE_TYPE: &str = "transient"; +pub(crate) const VALUES_FIELD: &str = "values"; +pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; +pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; /// The model's `CompileModelConfig`, once its template reads are recorded. pub(crate) fn built_config<'py>( @@ -25,61 +23,8 @@ pub(crate) fn built_config<'py>( view(classes, &built.values, &fields, built.nodes.as_ref()) } -/// Every `CompileModelConfig` field but `values`, recording the build's template reads first. -pub(crate) fn config_fields<'py>( - classes: &ConfigClasses<'py>, - built: &BuiltConfig<'py>, - matched: Option<&str>, -) -> PyResult> { - let py: Python<'py> = built.values.py(); - if !built.reads.is_empty() { - classes - .record_template_reads - .call1((PyTuple::new(py, &built.reads)?,))?; - } - let none: Bound<'py, PyAny> = py.None().into_bound(py); - let (inherited_retention, inherited_table_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = - classes - .inherited_storage - .as_any() - .get_item(storage_key(classes, &built.values)?)? - .extract()?; - let retention: Bound<'py, PyAny> = match built.retention { - RetentionOverride::Inherit => inherited_retention, - RetentionOverride::Unmanaged => resolved_retention(classes, none.clone(), true)?, - RetentionOverride::Days(days) => { - resolved_retention(classes, days.into_pyobject(py)?.into_any(), false)? - } - }; - let table_type: Bound<'py, PyAny> = match built.table_type { - TableTypeOverride::Inherit => inherited_table_type, - TableTypeOverride::Permanent => resolved_table_type(classes, PERMANENT_TABLE_TYPE)?, - TableTypeOverride::Transient => resolved_table_type(classes, TRANSIENT_TABLE_TYPE)?, - }; - let logical_schema: Bound<'py, PyAny> = built.logical_schema.clone().unwrap_or(none.clone()); - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("model_header_keys", PyTuple::new(py, &built.header_keys)?)?; - fields.set_item("matched_path_default", matched)?; - fields.set_item("logical_schema", &logical_schema)?; - fields.set_item( - "layer_schema", - if built.layer_schema_configured { - logical_schema - } else { - none.clone() - }, - )?; - fields.set_item( - "logical_database", - built.logical_database.clone().unwrap_or(none), - )?; - fields.set_item("time_travel_retention", retention)?; - fields.set_item("table_type", table_type)?; - Ok(fields) -} - /// The materialization whose inherited storage policy applies, or `None` when not table-backed. -fn storage_key<'py>( +pub(crate) fn storage_key<'py>( classes: &ConfigClasses<'py>, values: &Bound<'py, PyDict>, ) -> PyResult> { @@ -116,17 +61,3 @@ pub(crate) fn resolved_table_type<'py>( fields.set_item("declared", true)?; classes.resolved_table_type.call((), Some(&fields)) } - -/// A `CompileModelConfig` of `values` and the other `fields`. -pub(crate) fn view<'py>( - classes: &ConfigClasses<'py>, - values: &Bound<'py, PyDict>, - fields: &Bound<'py, PyDict>, - nodes: Option<&BuiltConfigValues>, -) -> PyResult> { - let arguments: Bound<'py, PyDict> = fields.copy()?; - arguments.set_item(VALUES_FIELD, values)?; - let config = classes.compile_model_config.call((), Some(&arguments))?; - retain_values(&config, nodes)?; - Ok(config) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs index 1ff777cbd3..77a9eb366d 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs @@ -1,9 +1,8 @@ //! Native config errors handed to Python, which raises them as the Python stages would. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{Py, PyErr, pymethods}; -use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use crate::model_config::_helpers::config::config_values::config_value_type_error; use crate::model_config::errors::NativeConfigError; @@ -12,33 +11,10 @@ use sqlbuild_python_base::records::types::ErrorId; const EXPECTED_STRING: &str = "a string"; -const COMPILE_INPUT_CLASS: &str = "compile_input"; -const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; -const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; -const DISCOVERY_CONFLICT_CLASS: &str = "discovery_conflict"; - -/// Wrap `error` for Python. -pub(crate) fn native_config_error( - py: Python<'_>, - error: ConfigError, -) -> PyResult> { - let class_name = match error.class { - ErrorClass::CompileInput => COMPILE_INPUT_CLASS, - ErrorClass::ConfigValueType => CONFIG_VALUE_TYPE_CLASS, - ErrorClass::ResourceIdentity => RESOURCE_IDENTITY_CLASS, - ErrorClass::DiscoveryConflict => DISCOVERY_CONFLICT_CLASS, - }; - Py::new( - py, - NativeConfigError { - class_name, - message: error.message, - code: error.code, - help: error.help, - key: error.key, - }, - ) -} +pub(crate) const COMPILE_INPUT_CLASS: &str = "compile_input"; +pub(crate) const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; +pub(crate) const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; +pub(crate) const DISCOVERY_CONFLICT_CLASS: &str = "discovery_conflict"; /// The Python exception a native config error raises as, given the values it names. pub(crate) fn config_exception<'py>( @@ -49,17 +25,6 @@ pub(crate) fn config_exception<'py>( error.get().exception(error.py(), values) } -/// `config_exception` as the raised error, or the failure building it. -pub(crate) fn raised_config_exception( - error: &Bound<'_, PyAny>, - values: Option<&Bound<'_, PyAny>>, -) -> PyErr { - match config_exception(error, values) { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - #[pymethods] impl NativeConfigError { /// The exception this error raises as: its class, message, code and help. @@ -91,8 +56,3 @@ impl NativeConfigError { error_class(py, class)?.call((&self.message,), Some(&keywords)) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs index 33b0181197..b3f1843564 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs @@ -1,10 +1,11 @@ //! Python config values converted once into `ConfigNode`s, and projected back into Python objects. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; use std::sync::Arc; -use pyo3::IntoPyObject; use pyo3::exceptions::{PyOverflowError, PyUnicodeEncodeError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ @@ -40,7 +41,10 @@ fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { } /// An opaque value with the cheap facts `PyNode` reads of it, and its converted items. -fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResult { +pub(crate) fn host( + value: &Bound<'_, PyAny>, + children: HostChildren, +) -> PyResult { let node: PyNode<'_> = PyNode(value.clone()); let strict_text: bool = match value.downcast::() { Ok(text) => utf8_text(text)?, @@ -109,7 +113,7 @@ fn container( }) } -fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { +pub(crate) fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { if value.is_none() { return Ok(Some(ConfigNode::Null)); } @@ -147,15 +151,6 @@ fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult) -> PyResult { - if is_exact_container(value) { - host(value, HostChildren::None) - } else { - config_node(value) - } -} - /// Whether `value` is an exact `list`, `tuple` or `dict`. pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { value.is_exact_instance_of::() @@ -163,14 +158,6 @@ pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { || value.is_exact_instance_of::() } -/// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -pub(crate) fn config_node(value: &Bound<'_, PyAny>) -> PyResult { - match convert(value, &[])? { - Some(node) => Ok(node), - None => host(value, HostChildren::None), - } -} - /// A Python mapping's entries as config entries. pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult { mapping @@ -179,38 +166,6 @@ pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult( - py: Python<'py>, - node: &PyConfigNode, -) -> PyResult> { - Ok(match node { - ConfigNode::Null => py.None().into_bound(py), - ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), - ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), - ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), - ConfigNode::Str(text) => PyString::new(py, text).into_any(), - ConfigNode::List(items) => PyList::new( - py, - items - .iter() - .map(|item| config_object(py, item)) - .collect::>>()?, - )? - .into_any(), - ConfigNode::Tuple(items) => PyTuple::new( - py, - items - .iter() - .map(|item| config_object(py, item)) - .collect::>>()?, - )? - .into_any(), - ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), - ConfigNode::Host(host) => host.handle.0.bind(py).clone(), - }) -} - /// A new Python `dict` of config entries. pub(crate) fn config_dict<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs index 759e89e53f..f22d92a421 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs @@ -1,26 +1,21 @@ //! `${...}` templates in authored Python config values and effective vars, expanded natively. +use crate::model_config::main::config::expanded::expanded; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::config::template_error::template_error; use crate::model_config::models::PythonHost; use crate::model_config::models::Stop; use crate::model_config::models::TemplateSources; use std::cell::RefCell; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, - PyTupleMethods, -}; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::templates::errors::TemplateError; -use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; -use sqlbuild_model_config::templates::main::template_error_message::template_error_message; -use sqlbuild_model_config::templates::models::{StringExpansion, TemplateOptions}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; +use sqlbuild_model_config::templates::models::TemplateOptions; use sqlbuild_model_config::templates::types::TemplateHost; -use crate::model_config::_helpers::config::config_errors::native_config_error; use crate::model_config::errors::NativeConfigError; -const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const ENVIRONMENT_READ: &str = "env"; pub(crate) const CONTEXT_READ: &str = "ctx"; pub(crate) const EFFECTIVE_VARS_LABEL: &str = "effective vars"; @@ -38,11 +33,6 @@ pub(crate) struct EffectiveVars<'py> { pub(crate) resolving: RefCell>, } -/// The `CompileInputError` Python raises for a template error in the context `label`. -pub(crate) fn template_error(error: &TemplateError, label: &str) -> ConfigError { - ConfigError::compile(template_error_message(error, label)) -} - /// The expanded value, or the native error expansion stopped at, and the reads in order. pub(crate) type TemplateOutcome<'py> = ( Result, Py>, @@ -160,52 +150,3 @@ fn outcome<'py>( }; Ok((value, host.into_reads())) } - -/// Expand every template in `value` and the containers inside it. -pub(crate) fn expanded<'py>( - host: &PythonHost<'py>, - value: &Bound<'py, PyAny>, - options: TemplateOptions, -) -> Result, Stop> { - let py = host.py; - if let Ok(text) = value.downcast::() { - let text = text.to_str().map_err(Stop::Python)?; - if !text.contains(TEMPLATE_OPEN_TOKEN) { - return Ok(value.clone()); - } - return match expand_template_string(host, text, options) { - Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), - Ok(StringExpansion::Value(result)) => Ok(result), - Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), - Err(failure) => Err(host.stop(failure)), - }; - } - if let Ok(mapping) = value.downcast::() { - let result = PyDict::new(py); - for (key, item) in mapping.iter() { - result - .set_item(key, expanded(host, &item, options)?) - .map_err(Stop::Python)?; - } - return Ok(result.into_any()); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyList::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyTuple::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - Ok(value.clone()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs index 8cf5b4efed..bdeb7dc758 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs @@ -1,16 +1,15 @@ //! Typed values read from raw authored configuration mappings, rejecting the wrong types. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use crate::model_config::main::config::config_node::config_node; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; -use sqlbuild_model_config::config_values::main::checked_text::checked_text; use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::CursorBound; -use crate::model_config::_helpers::config::config_nodes::config_node; use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -47,7 +46,7 @@ pub(crate) fn config_value_type_error<'py>( } /// `values.get(key)` as a config node. -fn authored_node<'py>( +pub(crate) fn authored_node<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult<(PyConfigNode, Bound<'py, PyAny>)> { @@ -55,19 +54,6 @@ fn authored_node<'py>( Ok((config_node(&value)?, value)) } -/// An authored string, preserving a present empty string. -#[pyfunction(name = "get_config_str")] -#[pyo3(signature = (*, values, key))] -pub(crate) fn config_str<'py>( - values: &Bound<'py, PyAny>, - key: &str, -) -> PyResult>> { - let (node, value) = authored_node(values, key)?; - let checked = - checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; - Ok(checked.map(|_| value)) -} - /// An authored boolean, or `None` when the key is absent or `None`. #[pyfunction(name = "get_config_bool")] #[pyo3(signature = (*, values, key))] @@ -127,12 +113,3 @@ pub(crate) fn config_cursor_bound<'py>( Some(CursorBound::Stringified(_)) => Some(value.str()?.into_any()), }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(config_str, module)?)?; - module.add_function(wrap_pyfunction!(config_bool, module)?)?; - module.add_function(wrap_pyfunction!(config_string_tuple, module)?)?; - module.add_function(wrap_pyfunction!(config_int, module)?)?; - module.add_function(wrap_pyfunction!(config_cursor_bound, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs index 5a8c4bb375..4e4eae0527 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs @@ -1,55 +1,33 @@ //! Template expansion for compile inputs and effective vars, with the reads it makes recorded. -use std::time::{SystemTime, UNIX_EPOCH}; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; +use crate::model_config::main::config::expanded_template_data::expanded_template_data; +use crate::model_config::main::config::record_reads::record_reads; +use crate::model_config::main::config::run_id::run_id; +use crate::model_config::main::model_nodes::unicode_environment::unicode_environment; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::model_config::_helpers::config::config_errors::config_exception; use crate::model_config::_helpers::config::config_templates::TemplateOutcome; -use crate::model_config::_helpers::config::config_templates::expand_config_templates; -use crate::model_config::_helpers::config::config_templates::lone_surrogate_site; -use crate::model_config::_helpers::config::config_templates::resolve_effective_vars; -use crate::model_config::_helpers::model_nodes::unicode_environment::unicode_environment; -use crate::model_config::models::TemplateSources; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::compile_context::main::input_reads::context_read::context_read; -use sqlbuild_python_base::compile_context::main::input_reads::environment_read::environment_read; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; - -const UUID_MODULE: &str = "uuid"; -const ENVIRONMENT_READ: &str = "env"; -const RUN_ID_KEY: &str = "run.id"; -const RUN_TARGET_KEY: &str = "run.target"; -const MODEL_CONFIG_LABEL: &str = "model config"; -const RUN_ID_SUFFIX_LENGTH: usize = 12; -const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; -/// Record native template environment and context reads for project reuse, in order. -pub(crate) fn record_reads<'a>( - py: Python<'_>, - reads: impl IntoIterator, -) -> PyResult<()> { - for (kind, name) in reads { - if kind == ENVIRONMENT_READ { - environment_read(py, name)?; - } else { - context_read(py, name)?; - } - } - Ok(()) -} +pub(crate) const UUID_MODULE: &str = "uuid"; +pub(crate) const ENVIRONMENT_READ: &str = "env"; +pub(crate) const RUN_ID_KEY: &str = "run.id"; +pub(crate) const RUN_TARGET_KEY: &str = "run.target"; +pub(crate) const MODEL_CONFIG_LABEL: &str = "model config"; +pub(crate) const RUN_ID_SUFFIX_LENGTH: usize = 12; +pub(crate) const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; /// Record `outcome`'s reads, then return its value or raise its error as Python's stage does. -fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult> { +pub(crate) fn applied<'py>( + py: Python<'py>, + outcome: TemplateOutcome<'py>, +) -> PyResult> { let (value, reads) = outcome; record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; match value { @@ -62,106 +40,16 @@ fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult) -> PyResult> { +pub(crate) fn environment(py: Python<'_>) -> PyResult> { unicode_environment(py) } -/// `value` with every `${...}` template expanded, recording the reads it made. -pub(crate) fn expanded_template_data<'py>( - value: &Bound<'py, PyAny>, - (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), - (options, label): (TemplateOptions, &str), -) -> PyResult> { - let py: Python<'py> = value.py(); - let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); - applied( - py, - expand_config_templates(py, value, sources, (options, label))?, - ) -} - -/// Merged effective vars with recursive `${name}` expansion. -pub(crate) fn expanded_effective_vars<'py>( - raw_values: &Bound<'py, PyDict>, -) -> PyResult> { - let py: Python<'py> = raw_values.py(); - if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { - let error: Bound<'py, PyAny> = error_class(py, ErrorId::CompileInput)?.call( - (format!( - "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ - {byte_offset}, which is not valid Unicode text" - ),), - Some(&keywords( - py, - &[( - "help", - PyString::new( - py, - &format!( - "Set '{name}' to valid Unicode text in --vars or the [vars] \ - table; the value is not shown because it may be a secret" - ), - ) - .into_any(), - )], - )?), - )?; - return Err(PyErr::from_value(error)); - } - applied( - py, - resolve_effective_vars(py, raw_values, environment(py)?)?, - ) -} - -/// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -pub(crate) fn early_model_templates<'py>( - values: &Bound<'py, PyAny>, - effective_vars: &Bound<'py, PyAny>, - (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = values.py(); - let context_values: Bound<'py, PyDict> = PyDict::new(py); - context_values.set_item(RUN_ID_KEY, run_id)?; - context_values.set_item(RUN_TARGET_KEY, target_name)?; - expanded_template_data( - values, - (effective_vars.downcast()?, &context_values), - ( - TemplateOptions { - allow_context: true, - preserve_context_tokens: false, - preserve_unknown_context: true, - }, - MODEL_CONFIG_LABEL, - ), - ) -} - -/// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -pub(crate) fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = selected_run_id.py(); - if !selected_run_id.is_none() { - return Ok(selected_run_id.clone()); - } - let epoch_seconds: i64 = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) - }); - let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); - let suffix: String = imported(py, UUID_MODULE, "uuid4")? - .call0()? - .getattr("hex")? - .extract()?; - let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); - Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) -} - /// Recursively expand template strings inside supported Python container values. #[pyfunction] #[pyo3(signature = (**request))] -fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn expand_template_data<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request: Request<'_, 'py> = Request::new("expand_template_data", request)?; let flag = |name: &str| -> PyResult { request.get(name)?.is_truthy() }; expanded_template_data( @@ -183,7 +71,9 @@ fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(raw_values: &Bound<'py, PyDict>) -> PyResult> { +pub(crate) fn expand_effective_vars<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult> { expanded_effective_vars(raw_values) } @@ -201,21 +91,8 @@ pub(crate) fn record_template_reads(py: Python<'_>, reads: Vec<(String, String)> /// Resolve a stable compile invocation id. #[pyfunction] #[pyo3(signature = (*, selected_run_id))] -fn resolve_run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn resolve_run_id<'py>( + selected_run_id: &Bound<'py, PyAny>, +) -> PyResult> { run_id(selected_run_id) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; - module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; - module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; - module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; - Ok(()) -} - -/// `record_template_reads` as a Python function. -pub(crate) fn record_template_reads_function( - py: pyo3::Python<'_>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(record_template_reads, py) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs index 5d628ab405..3b27a96718 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs @@ -1,6 +1,7 @@ //! Built model config trees retained natively, with Python-authored values converted on first read. -use crate::model_config::models::TypedConfigValues; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::BuiltConfigValues; use crate::model_config::types::ConfigValues; use std::collections::HashMap; @@ -8,35 +9,23 @@ use std::sync::{Arc, Mutex}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyString, PyTuple}; +use pyo3::types::PyString; use sqlbuild_model_config::config_values::errors::ConfigTypeError; -use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; -use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; -use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; -use sqlbuild_model_config::config_values::main::checked_text::checked_text; -use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::{ConfigNode, CursorBound, HostChildren}; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::config_nodes::{PyConfigEntries, is_exact_container}; use crate::model_config::_helpers::config::config_values::config_value_type_error; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::compile_driver::constants::TYPED_CONFIG_READS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::cached_value::cached_value; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::main::fields::store_cached::store_cached; -use sqlbuild_python_base::records::models::NativeRecord; -const CONFIG_FIELD: &str = "config"; -const VALUES_FIELD: &str = "values"; +pub(crate) const CONFIG_FIELD: &str = "config"; +pub(crate) const VALUES_FIELD: &str = "values"; const ISOFORMAT_METHOD: &str = "isoformat"; /// The config record's cache slot holding its typed values; no Python attribute reads it. -const TYPED_VALUES_SLOT: &str = "__typed_values__"; +pub(crate) const TYPED_VALUES_SLOT: &str = "__typed_values__"; const POISONED: &str = "typed config values cache is poisoned"; /// Keep the builder's tree without interpreting custom Python mapping keys natively. @@ -51,28 +40,8 @@ pub(crate) fn built_values(entries: PyConfigEntries) -> Option, - built: Option<&BuiltConfigValues>, -) -> PyResult<()> { - if let Some(built) = built - && let Ok(record) = config.cast::() - { - let typed = Bound::new( - config.py(), - TypedConfigValues { - built: Some(Arc::clone(built)), - read: Mutex::new(Vec::new()), - }, - )?; - store_cached(record, TYPED_VALUES_SLOT, &typed.into_any().unbind())?; - } - Ok(()) -} - /// `values.get(key)` as a typed node, `None` when absent; each key is converted once. -fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { +pub(crate) fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { let (mapping, typed) = values; let py: Python<'_> = mapping.py(); let built = typed.get().built.as_ref().map(|values| values.get(key)); @@ -111,43 +80,6 @@ fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { Ok(node) } -/// A config record's values, their typed nodes kept on the record so each key converts once. -pub(crate) fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = config.py(); - let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; - let fresh = || { - Bound::new( - py, - TypedConfigValues { - built: None, - read: Mutex::new(Vec::new()), - }, - ) - }; - let Ok(record) = config.cast::() else { - let typed: Bound<'py, TypedConfigValues> = fresh()?; - return Ok((values, typed)); - }; - let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { - Some(typed) => typed.into_bound(py).cast_into()?, - None => { - let typed: Bound<'py, TypedConfigValues> = fresh()?; - store_cached( - record, - TYPED_VALUES_SLOT, - &typed.clone().into_any().unbind(), - )?; - typed - } - }; - Ok((values, typed)) -} - -/// A model's config values as typed nodes, read key by key. -pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - typed_values(&native_field(model, CONFIG_FIELD)?) -} - /// The `ConfigValueTypeError` a typed read raises, naming the value's Python type. pub(crate) fn type_error(py: Python<'_>, error: &ConfigTypeError<'_, HostHandle>) -> PyErr { let raised = || -> PyResult { @@ -172,51 +104,6 @@ pub(crate) fn node_text(py: Python<'_>, node: &PyConfigNode) -> PyResult } } -/// `key`'s string value, or `None` when absent or `None`. -pub(crate) fn config_text( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let node: Arc = node(values, key)?; - checked_text(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| node_text(py, value)) - .transpose() -} - -/// `key`'s string value as its Python object, or `None` when absent or `None`. -pub(crate) fn config_text_object<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_text(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| config_object(py, value)) - .transpose() -} - -/// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -pub(crate) fn config_text_is( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, - expected: &str, -) -> PyResult { - let node: Arc = node(values, key)?; - let text: Option<&PyConfigNode> = - checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; - Ok(match text { - Some(ConfigNode::Str(text)) => text == expected, - Some(ConfigNode::Host(host)) if host.facts.strict_text => { - host.facts.text.as_deref() == Some(expected) - } - _ => false, - }) -} - /// A cursor bound's text object: a string's or integer's `str()`, a date's `isoformat()`. pub(crate) fn cursor_bound_object<'py>( py: Python<'py>, @@ -231,72 +118,3 @@ pub(crate) fn cursor_bound_object<'py>( CursorBound::Stringified(value) => Ok(config_object(py, value)?.str()?.into_any()), } } - -/// `key`'s cursor bound as text, or `None` when absent or `None`. -pub(crate) fn config_cursor_bound<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_cursor_bound(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|bound| cursor_bound_object(py, bound)) - .transpose() -} - -/// `values.get(key)` as its Python object, `None` when absent. -pub(crate) fn config_value<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let value: Arc = node(values, key)?; - config_object(py, &value) -} - -/// `values.get(key)` as its typed node, `None` (null) when absent. -pub(crate) fn config_node_of(values: &ConfigValues<'_>, key: &str) -> PyResult> { - node(values, key) -} - -/// `key`'s boolean value, or `None` when absent or `None`. -pub(crate) fn config_flag( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let node: Arc = node(values, key)?; - checked_flag(key, Some(&node)).map_err(|error| type_error(py, &error)) -} - -/// `key`'s integer value as its Python object, or `None` when absent or `None`. -pub(crate) fn config_integer_object<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_integer(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| config_object(py, value)) - .transpose() -} - -/// `key`'s strings as a tuple, or `None` when absent or `None`. -pub(crate) fn config_text_tuple<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - let Some(items) = checked_texts(key, Some(&node)).map_err(|error| type_error(py, &error))? - else { - return Ok(None); - }; - let objects: Vec> = items - .into_iter() - .map(|item| config_object(py, item)) - .collect::>()?; - PyTuple::new(py, objects).map(Some) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs index 1811576f97..1ffb1169b1 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs @@ -1,5 +1,7 @@ //! Retained audit contracts; Python-authored factory results remain opaque values. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use pyo3::prelude::{PyResult, Python}; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_discovery::model_files::main::config_value::config_value; @@ -8,8 +10,6 @@ use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::types::HeaderValue; use crate::model_config::types::PyConfigNode; use crate::sqltext::main::authored_value_to_python::authored_value_to_python; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs index 86f53dccfa..29d7ed5ff6 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs @@ -1,20 +1,13 @@ //! Authored MODEL and SCHEMA columns parsed natively into contract columns. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::headers::schema_columns::schema_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; -use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_schema_columns as parse_columns; -use sqlbuild_model_config::header_metadata::models::ColumnRequest; use crate::model_config::_helpers::config::config_errors::config_exception; -use crate::model_config::_helpers::config::config_errors::native_config_error; -use crate::model_config::_helpers::headers::header_metadata::column_tuple; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; -use crate::model_config::models::ContractClasses; -use crate::model_config::models::PyNode; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; @@ -24,31 +17,9 @@ pub(crate) type ColumnSite<'a> = (&'a str, &'a str); /// `(require_columns, allow_migrate_from, declaration_errors)`. pub(crate) type ColumnOptions = (bool, bool, bool); -/// The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -pub(crate) fn schema_columns<'py>( - raw_columns: &Bound<'py, PyAny>, - site: ColumnSite<'_>, - options: ColumnOptions, - column_locations: &Bound<'py, PyDict>, -) -> PyResult> { - let py: Python<'py> = raw_columns.py(); - let (path, label) = site; - let (require_columns, allow_migrate_from, declaration_errors) = options; - let request = ColumnRequest { - path, - label, - require_columns, - allow_migrate_from, - }; - let columns = parse_columns(&PyNode(raw_columns.clone()), request) - .map_err(|error| raised(py, error, declaration_errors))?; - let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; - column_tuple(py, &classes, &columns, column_locations) -} - /// Parse one MODEL or SCHEMA `columns` mapping into contract columns. #[pyfunction] -fn parse_schema_columns<'py>( +pub(crate) fn parse_schema_columns<'py>( raw_columns: Bound<'py, PyAny>, site: (String, String), options: ColumnOptions, @@ -58,7 +29,7 @@ fn parse_schema_columns<'py>( } /// A plain parse error as `DeclarationParseError` when asked; every other error as Python raises it. -fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr { +pub(crate) fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr { let built = (|| { if declaration_errors && error.class == ErrorClass::CompileInput { return error_class(py, ErrorId::DeclarationParse)?.call1((error.message,)); @@ -71,7 +42,3 @@ fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr Err(failure) => failure, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_schema_columns, module)?) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs deleted file mode 100644 index 144dc5045a..0000000000 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Native dynamic-column contracts with a shared, lazy public projection. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; - -use sqlbuild_python_base::records::main::fields::field::field; - -/// A Python-authored family contract crosses into native ownership once. -pub(crate) fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { - families - .try_iter()? - .map(|family| { - let family = family?; - Ok(DynamicFamily { - name: field(&family, "name")?.extract()?, - pivot_column: field(&family, "pivot_column")?.extract()?, - value_column: field(&family, "value_column")?.extract()?, - aggregate: field(&family, "aggregate")?.extract()?, - data_type: field(&family, "type")?.extract()?, - name_pattern: field(&family, "name_pattern")?.extract()?, - }) - }) - .collect() -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs index 068e2e8701..363dd967cc 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs @@ -1,7 +1,10 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::headers::native_schema_column::native_schema_column; +use crate::model_config::main::headers::thresholds::thresholds; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use crate::model_config::models::ContractClasses; -use crate::model_config::models::HeaderColumn; use crate::model_config::models::NativeColumnSpec; use crate::model_config::types::HeaderValue; use std::sync::OnceLock; @@ -12,20 +15,14 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; use sqlbuild_discovery::models::LineColumnSpan; -use sqlbuild_model_config::header_metadata::models::{ - ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, -}; +use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn, ThresholdBound}; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::_helpers::config::typed_config::node_text; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; -use crate::model_config::models::HeaderAudits; use crate::model_config::models::PyNode; use crate::model_config::types::PyConfigNode; use crate::sqltext::main::authored_value_to_python::authored_value_to_python; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::all_specs::all_specs; use sqlbuild_python_base::records::types::RecordId; @@ -111,22 +108,6 @@ pub(crate) enum ColumnFields { Projected, } -/// The `SchemaColumn` a native column projects to when its location names `location`'s path. -pub(crate) fn native_column_record<'py>( - py: Python<'py>, - column: &NativeColumn, - location: Option<(Bound<'py, PyAny>, LineColumnSpan)>, -) -> PyResult> { - let location: Bound<'py, PyAny> = match location { - Some((path, span)) => location_record(py, path, span)?, - None => py.None().into_bound(py), - }; - contract_object( - &record_class(py, RecordId::SchemaColumn)?, - column_values(py, column, location)?, - ) -} - /// A native column's record fields, in declaration order. pub(crate) fn native_values<'py>( py: Python<'py>, @@ -136,7 +117,7 @@ pub(crate) fn native_values<'py>( } /// A native column's record fields at `location`, in declaration order. -fn column_values<'py>( +pub(crate) fn column_values<'py>( py: Python<'py>, column: &NativeColumn, location: Bound<'py, PyAny>, @@ -187,7 +168,7 @@ pub(crate) fn native_location<'py>( } /// `SourceLocation(path, line, column, end_line, end_column)`. -fn location_record<'py>( +pub(crate) fn location_record<'py>( py: Python<'py>, path: Bound<'py, PyAny>, span: LineColumnSpan, @@ -281,14 +262,6 @@ pub(crate) fn schema_columns<'py>( Ok(columns) } -pub(crate) fn audit_tuple<'py>( - py: Python<'py>, - classes: &ContractClasses<'py>, - parsed: &[ParsedAudit>], -) -> PyResult> { - PyTuple::new(py, audit_instances(py, classes, parsed)?) -} - /// The parsed audits as `SchemaAuditInstance` objects without a location, in order. pub(crate) fn audit_instances<'py, N: HeaderValue>( py: Python<'py>, @@ -303,19 +276,6 @@ pub(crate) fn audit_instances<'py, N: HeaderValue>( Ok(audits) } -/// `audit.name or audit.definition_name` of the instance `audit` builds. -pub(crate) fn parsed_audit_identity( - py: Python<'_>, - audit: &ParsedAudit, -) -> PyResult { - if let Some(name) = &audit.name - && name.object(py)?.is_truthy()? - { - return name.exact_text(py); - } - audit.definition_name.exact_text(py) -} - /// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. pub(crate) fn contract_object<'py, const N: usize>( class: &Bound<'py, PyAny>, @@ -358,7 +318,7 @@ fn mismatched_record() -> Option<&'static str> { None } -fn optional<'py, N: HeaderValue>( +pub(crate) fn optional<'py, N: HeaderValue>( py: Python<'py>, value: Option<&N>, ) -> PyResult> { @@ -377,43 +337,6 @@ pub(crate) fn schema_column<'py>( native_schema_column(py, classes, column, locations)?.project(py) } -pub(crate) fn native_schema_column<'py, N: HeaderValue>( - py: Python<'py>, - classes: &ContractClasses<'py>, - column: &ParsedColumn, - locations: &Bound<'py, PyDict>, -) -> PyResult { - let location = locations - .get_item(column.name.object(py)?)? - .unwrap_or_else(|| py.None().into_bound(py)); - let audits = HeaderAudits::new(py, classes, &column.audits, &location)?; - let projected_audits = if classes - .schema_column - .is(&record_class(py, RecordId::SchemaColumn)?) - { - PyTuple::empty(py) - } else { - audits.project(py)? - }; - let mut result = HeaderColumn::new( - &classes.schema_column, - [ - column.name.object(py)?, - optional(py, column.column_type.as_ref())?, - optional(py, column.nullable.as_ref())?, - optional(py, column.description.as_ref())?, - PyDict::new(py).into_any(), - projected_audits.into_any(), - location, - optional(py, column.migrate_from.as_ref())?, - ], - )?; - if !matches!(result.fields, ColumnFields::Projected) { - result.audits = Some(audits); - } - Ok(result) -} - pub(crate) fn audit_instance<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, @@ -454,25 +377,7 @@ pub(crate) fn audit_instance<'py, N: HeaderValue>( ) } -/// `MeasurementThresholds` of `parsed`, or `None`. -pub(crate) fn thresholds<'py>( - py: Python<'py>, - classes: &ContractClasses<'py>, - parsed: Option<&ParsedThresholds>, -) -> PyResult> { - let Some(parsed) = parsed else { - return Ok(py.None().into_bound(py)); - }; - contract_object( - &classes.measurement_thresholds, - [ - threshold_bound(py, classes, parsed.warn)?, - threshold_bound(py, classes, parsed.error)?, - ], - ) -} - -fn threshold_bound<'py>( +pub(crate) fn threshold_bound<'py>( py: Python<'py>, classes: &ContractClasses<'py>, bound: Option, diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs index cf1bbd1e20..cf36afabaf 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod header_audits; pub(crate) mod header_columns; -pub(crate) mod header_families; pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs index 41a2ea95ec..4c0fa7f06a 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs @@ -1,36 +1,4 @@ //! Column renames validated natively, raised by Python as the attachment raises them. -use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::model_validation::main::validate_column_migrations::validate_column_migrations; -use sqlbuild_model_config::model_validation::models::ColumnMigration; - -const MATERIALIZED_KEY: &str = "materialized"; -const MIGRATE_FROM_KEY: &str = "migrate_from"; - -/// The first column-rename problem for a model with these config values, if any. -pub(crate) fn column_migration_error( - model_name: &str, - values: &Bound<'_, PyDict>, - columns: &[ColumnMigration], -) -> PyResult> { - if columns.iter().all(|column| column.migrate_from.is_none()) { - return Ok(None); - } - let materialized: Option = match values.get_item(MATERIALIZED_KEY)? { - None => None, - Some(value) if value.is_none() => None, - Some(value) => match value.downcast::() { - Ok(text) => Some(text.to_str()?.to_owned()), - Err(_) => return Ok(Some(ConfigError::config_value_type(MATERIALIZED_KEY))), - }, - }; - let model_migrates: bool = values - .get_item(MIGRATE_FROM_KEY)? - .is_some_and(|value| !value.is_none()); - Ok( - validate_column_migrations(model_name, materialized.as_deref(), model_migrates, columns) - .err(), - ) -} +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const MIGRATE_FROM_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs index 8666da7ad7..6a4c93a030 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs @@ -1,5 +1,6 @@ //! The Python classes native header and YAML parsing build contract objects as. +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; @@ -54,24 +55,10 @@ const INTEGRATION_ENTRY_CLASSES: [(&str, ThirdPartyHost); 3] = [ ("dlt_resource_config", ThirdPartyHost::DltResourceConfig), ]; -static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); -static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +pub(crate) static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +pub(crate) static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); -/// The contract classes native header parsing builds columns and audits as. -pub(crate) fn contract_classes(py: Python<'_>) -> PyResult> { - CONTRACT_CLASS_TABLE - .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - -/// The classes parsed seed and source declarations become. -pub(crate) fn yaml_entry_classes(py: Python<'_>) -> PyResult> { - YAML_ENTRY_CLASS_TABLE - .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - -fn built_contract_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_contract_classes(py: Python<'_>) -> PyResult> { let classes: Bound<'_, PyDict> = PyDict::new(py); for (key, class) in CONTRACT_CLASSES { classes.set_item(key, record_class(py, class)?)?; @@ -87,7 +74,7 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { Ok(classes) } -fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { let classes: Bound<'_, PyDict> = PyDict::new(py); classes.set_item("contracts", contract_classes(py)?)?; for (key, class) in YAML_ENTRY_CLASSES { diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs index 4a1a1ef509..696a59cb1a 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs @@ -1,5 +1,8 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::model_nodes::raised_config_error::raised_config_error; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,12 +14,9 @@ use sqlbuild_model_config::path_defaults::main::select_path_default::select_path use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; use sqlbuild_model_config::types::AuthoredNode; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::config_entries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::is_text_node; -use crate::model_config::_helpers::model_nodes::model_config_builder::raised_config_error; +use crate::model_config::_helpers::config::config_nodes::{ + PyConfigEntries, config_entries, is_text_node, +}; use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs index bbf87cae21..00be9fcd29 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs @@ -1,12 +1,14 @@ //! The native model config build handed Python inputs once, and its results projected for Python. +use crate::model_config::main::config::carried_node::carried_node; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::models::BuiltConfig; -use crate::model_config::models::NativeModelConfigBuilder; use std::collections::HashMap; use pyo3::PyErr; use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_discovery::model_files::main::config_value::config_value; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; @@ -20,16 +22,10 @@ use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::authored_values::main::project_var_text::project_var_text; -use crate::model_config::_helpers::config::config_errors::config_exception; -use crate::model_config::_helpers::config::config_errors::native_config_error; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::carried_node; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::hook_entry; -use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::config_nodes::{ + PyConfigEntries, hook_entry, is_exact_container, +}; use crate::model_config::_helpers::config::typed_config::built_values; -use crate::model_config::errors::NativeConfigError; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; @@ -256,20 +252,3 @@ fn projected<'py>( ) -> PyResult>> { value.map(|value| config_object(py, &value)).transpose() } - -/// The Python exception a native config error raises as. -pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { - let raised = || -> PyResult { - let error: Py = native_config_error(py, error)?; - Ok(PyErr::from_value(config_exception( - error.bind(py).as_any(), - None, - )?)) - }; - raised().unwrap_or_else(|error| error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs index d90bce6055..f82f3a1eff 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs @@ -1,18 +1,17 @@ //! The Python model validators, run natively up to the first error they raise. -use crate::model_config::models::NativeModelValidator; +use crate::model_config::main::config::native_config_error::native_config_error; use sqlbuild_python_base::positional_record; use std::collections::HashSet; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_model_config::model_validation::models::ValidationStop; use crate::discovery::main::project::column_name::column_name; use crate::discovery::main::project::each::each; use crate::discovery::main::project::stem::stem; -use crate::model_config::_helpers::config::config_errors::native_config_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -100,8 +99,3 @@ pub(crate) fn stop_object(py: Python<'_>, stop: ValidationStop) -> PyResult Ok(native_config_error(py, error)?.into_any()), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs index 76060f9163..df70b779e4 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs @@ -1,8 +1,6 @@ //! The process environment as compile text reads it, rejecting values that are not UTF-8. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyString}; use pyo3::{pyclass, pymethods}; @@ -68,12 +66,3 @@ impl UnicodeEnvironment { environ(py)?.len() } } - -/// A new view of the process environment as compile text reads it. -pub(crate) fn unicode_environment(py: Python<'_>) -> PyResult> { - Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs index 4c32af982e..e30ee2aa49 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs @@ -1,13 +1,17 @@ //! Carry an uninspected value: exact containers keep their own objects without opening items. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::host; +use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::main::config::config_node::config_node; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_model_config::config_values::models::HostChildren; /// Carry an uninspected value: exact containers keep their own objects without opening items. -#[inline] pub fn carried_node(value: &Bound<'_, PyAny>) -> PyResult { - config_nodes::carried_node(value) + if is_exact_container(value) { + host(value, HostChildren::None) + } else { + config_node(value) + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs index d018ca76ee..fe5b28bad1 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs @@ -1,18 +1,23 @@ //! `key`'s cursor bound as text, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::cursor_bound_object; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; +use std::sync::Arc; /// `key`'s cursor bound as text, or `None` when absent or `None`. -#[inline] pub fn config_cursor_bound<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_cursor_bound(py, values, key) + let node: Arc = node(values, key)?; + checked_cursor_bound(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|bound| cursor_bound_object(py, bound)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs index 385beff58a..7a127f9563 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs @@ -1,18 +1,66 @@ //! Every `CompileModelConfig` field but `values`, recording the build's template reads first. -use crate::model_config::_helpers::config::config_classes; +use crate::model_config::_helpers::config::config_classes::PERMANENT_TABLE_TYPE; +use crate::model_config::_helpers::config::config_classes::TRANSIENT_TABLE_TYPE; +use crate::model_config::_helpers::config::config_classes::resolved_retention; +use crate::model_config::_helpers::config::config_classes::resolved_table_type; +use crate::model_config::_helpers::config::config_classes::storage_key; use crate::model_config::models::BuiltConfig; use crate::model_config::models::ConfigClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; /// Every `CompileModelConfig` field but `values`, recording the build's template reads first. -#[inline] pub fn config_fields<'py>( classes: &ConfigClasses<'py>, built: &BuiltConfig<'py>, matched: Option<&str>, ) -> PyResult> { - config_classes::config_fields(classes, built, matched) + let py: Python<'py> = built.values.py(); + if !built.reads.is_empty() { + classes + .record_template_reads + .call1((PyTuple::new(py, &built.reads)?,))?; + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let (inherited_retention, inherited_table_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + classes + .inherited_storage + .as_any() + .get_item(storage_key(classes, &built.values)?)? + .extract()?; + let retention: Bound<'py, PyAny> = match built.retention { + RetentionOverride::Inherit => inherited_retention, + RetentionOverride::Unmanaged => resolved_retention(classes, none.clone(), true)?, + RetentionOverride::Days(days) => { + resolved_retention(classes, days.into_pyobject(py)?.into_any(), false)? + } + }; + let table_type: Bound<'py, PyAny> = match built.table_type { + TableTypeOverride::Inherit => inherited_table_type, + TableTypeOverride::Permanent => resolved_table_type(classes, PERMANENT_TABLE_TYPE)?, + TableTypeOverride::Transient => resolved_table_type(classes, TRANSIENT_TABLE_TYPE)?, + }; + let logical_schema: Bound<'py, PyAny> = built.logical_schema.clone().unwrap_or(none.clone()); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("model_header_keys", PyTuple::new(py, &built.header_keys)?)?; + fields.set_item("matched_path_default", matched)?; + fields.set_item("logical_schema", &logical_schema)?; + fields.set_item( + "layer_schema", + if built.layer_schema_configured { + logical_schema + } else { + none.clone() + }, + )?; + fields.set_item( + "logical_database", + built.logical_database.clone().unwrap_or(none), + )?; + fields.set_item("time_travel_retention", retention)?; + fields.set_item("table_type", table_type)?; + Ok(fields) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs index 6ae5cde364..5f091b71fd 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs @@ -1,12 +1,15 @@ //! `key`'s boolean value, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; +use std::sync::Arc; /// `key`'s boolean value, or `None` when absent or `None`. -#[inline] pub fn config_flag(py: Python<'_>, values: &ConfigValues<'_>, key: &str) -> PyResult> { - typed_config::config_flag(py, values, key) + let node: Arc = node(values, key)?; + checked_flag(key, Some(&node)).map_err(|error| type_error(py, &error)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs index a6ad072d82..27b8e6d8a1 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs @@ -1,18 +1,23 @@ //! `key`'s integer value as its Python object, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; +use std::sync::Arc; /// `key`'s integer value as its Python object, or `None` when absent or `None`. -#[inline] pub fn config_integer_object<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_integer_object(py, values, key) + let node: Arc = node(values, key)?; + checked_integer(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs index cfa8c0e2fd..7a1d0af107 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs @@ -1,13 +1,15 @@ //! One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::convert; +use crate::model_config::_helpers::config::config_nodes::host; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_model_config::config_values::models::HostChildren; /// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -#[inline] pub fn config_node(value: &Bound<'_, PyAny>) -> PyResult { - config_nodes::config_node(value) + match convert(value, &[])? { + Some(node) => Ok(node), + None => host(value, HostChildren::None), + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs index 8fa7b8d8f3..b43f99f24e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs @@ -1,13 +1,12 @@ //! `values.get(key)` as its typed node, `None` (null) when absent. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; use crate::model_config::types::ConfigValues; use crate::model_config::types::PyConfigNode; use pyo3::prelude::PyResult; use std::sync::Arc; /// `values.get(key)` as its typed node, `None` (null) when absent. -#[inline] pub fn config_node_of(values: &ConfigValues<'_>, key: &str) -> PyResult> { - typed_config::config_node_of(values, key) + node(values, key) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs index dbe0dcfb9c..72c987410e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs @@ -1,14 +1,37 @@ //! The Python object of one config value: new builtins, and opaque values as themselves. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::config_dict; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyBool, PyFloat, PyList, PyString, PyTuple}; +use sqlbuild_model_config::config_values::models::ConfigNode; /// The Python object of one config value: new builtins, and opaque values as themselves. -#[inline] pub fn config_object<'py>(py: Python<'py>, node: &PyConfigNode) -> PyResult> { - config_nodes::config_object(py, node) + Ok(match node { + ConfigNode::Null => py.None().into_bound(py), + ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), + ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), + ConfigNode::Str(text) => PyString::new(py, text).into_any(), + ConfigNode::List(items) => PyList::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Tuple(items) => PyTuple::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), + ConfigNode::Host(host) => host.handle.0.bind(py).clone(), + }) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs index 2e2bf43ad3..9b6d05b42f 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs @@ -1,15 +1,20 @@ //! An authored string, preserving a present empty string. -use crate::model_config::_helpers::config::config_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_values::authored_node; +use crate::model_config::_helpers::config::typed_config::type_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; /// An authored string, preserving a present empty string. -#[inline] +#[pyfunction(name = "get_config_str")] +#[pyo3(signature = (*, values, key))] pub fn config_str<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - config_values::config_str(values, key) + let (node, value) = authored_node(values, key)?; + let checked = + checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs index 50d70bbe63..e2222f3321 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs @@ -1,16 +1,23 @@ //! `key`'s string value, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::node_text; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use std::sync::Arc; /// `key`'s string value, or `None` when absent or `None`. -#[inline] pub fn config_text( py: Python<'_>, values: &ConfigValues<'_>, key: &str, ) -> PyResult> { - typed_config::config_text(py, values, key) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| node_text(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs index 44ca9d53fe..2f8f7d5366 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs @@ -1,17 +1,29 @@ //! Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::models::ConfigNode; +use std::sync::Arc; /// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -#[inline] pub fn config_text_is( py: Python<'_>, values: &ConfigValues<'_>, key: &str, expected: &str, ) -> PyResult { - typed_config::config_text_is(py, values, key, expected) + let node: Arc = node(values, key)?; + let text: Option<&PyConfigNode> = + checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; + Ok(match text { + Some(ConfigNode::Str(text)) => text == expected, + Some(ConfigNode::Host(host)) if host.facts.strict_text => { + host.facts.text.as_deref() == Some(expected) + } + _ => false, + }) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs index 21b9eb3c03..4ed04a64c5 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs @@ -1,18 +1,23 @@ //! `key`'s string value as its Python object, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use std::sync::Arc; /// `key`'s string value as its Python object, or `None` when absent or `None`. -#[inline] pub fn config_text_object<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_text_object(py, values, key) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs index 97dfd82467..73bf247ace 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs @@ -1,18 +1,29 @@ //! `key`'s strings as a tuple, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; +use std::sync::Arc; /// `key`'s strings as a tuple, or `None` when absent or `None`. -#[inline] pub fn config_text_tuple<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_text_tuple(py, values, key) + let node: Arc = node(values, key)?; + let Some(items) = checked_texts(key, Some(&node)).map_err(|error| type_error(py, &error))? + else { + return Ok(None); + }; + let objects: Vec> = items + .into_iter() + .map(|item| config_object(py, item)) + .collect::>()?; + PyTuple::new(py, objects).map(Some) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs index dac00dad84..1d2df8bd45 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs @@ -1,18 +1,18 @@ //! `values.get(key)` as its Python object, `None` when absent. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use std::sync::Arc; /// `values.get(key)` as its Python object, `None` when absent. -#[inline] pub fn config_value<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult> { - typed_config::config_value(py, values, key) + let value: Arc = node(values, key)?; + config_object(py, &value) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs index aa4ac4f1f3..9c976302b0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs @@ -1,16 +1,33 @@ //! Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::MODEL_CONFIG_LABEL; +use crate::model_config::_helpers::config::template_data::RUN_ID_KEY; +use crate::model_config::_helpers::config::template_data::RUN_TARGET_KEY; +use crate::model_config::main::config::expanded_template_data::expanded_template_data; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_model_config::templates::models::TemplateOptions; /// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -#[inline] pub fn early_model_templates<'py>( values: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - template_data::early_model_templates(values, effective_vars, (target_name, run_id)) + let py: Python<'py> = values.py(); + let context_values: Bound<'py, PyDict> = PyDict::new(py); + context_values.set_item(RUN_ID_KEY, run_id)?; + context_values.set_item(RUN_TARGET_KEY, target_name)?; + expanded_template_data( + values, + (effective_vars.downcast()?, &context_values), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + MODEL_CONFIG_LABEL, + ), + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs index 77bd879839..d1e35728eb 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs @@ -1,18 +1,61 @@ //! Expand every template in `value` and the containers inside it. -use crate::model_config::_helpers::config::config_templates; +use crate::model_config::_helpers::config::config_templates::TEMPLATE_OPEN_TOKEN; use crate::model_config::models::PythonHost; use crate::model_config::models::Stop; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use sqlbuild_model_config::templates::models::TemplateOptions; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, +}; +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::models::{StringExpansion, TemplateOptions}; /// Expand every template in `value` and the containers inside it. -#[inline] pub fn expanded<'py>( host: &PythonHost<'py>, value: &Bound<'py, PyAny>, options: TemplateOptions, ) -> Result, Stop> { - config_templates::expanded(host, value, options) + let py = host.py; + if let Ok(text) = value.downcast::() { + let text = text.to_str().map_err(Stop::Python)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + return match expand_template_string(host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), + Ok(StringExpansion::Value(result)) => Ok(result), + Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), + Err(failure) => Err(host.stop(failure)), + }; + } + if let Ok(mapping) = value.downcast::() { + let result = PyDict::new(py); + for (key, item) in mapping.iter() { + result + .set_item(key, expanded(host, &item, options)?) + .map_err(Stop::Python)?; + } + return Ok(result.into_any()); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyList::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyTuple::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + Ok(value.clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs index 4ad1859ae5..7f20b1a239 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs @@ -1,15 +1,46 @@ //! Merged effective vars with recursive `${name}` expansion. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::model_config::_helpers::config::config_templates::{ + lone_surrogate_site, resolve_effective_vars, +}; +use crate::model_config::_helpers::config::template_data::applied; +use crate::model_config::_helpers::config::template_data::environment; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// Merged effective vars with recursive `${name}` expansion. -#[inline] pub fn expanded_effective_vars<'py>( raw_values: &Bound<'py, PyDict>, ) -> PyResult> { - template_data::expanded_effective_vars(raw_values) + let py: Python<'py> = raw_values.py(); + if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { + let error: Bound<'py, PyAny> = error_class(py, ErrorId::CompileInput)?.call( + (format!( + "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ + {byte_offset}, which is not valid Unicode text" + ),), + Some(&keywords( + py, + &[( + "help", + PyString::new( + py, + &format!( + "Set '{name}' to valid Unicode text in --vars or the [vars] \ + table; the value is not shown because it may be a secret" + ), + ) + .into_any(), + )], + )?), + )?; + return Err(PyErr::from_value(error)); + } + applied( + py, + resolve_effective_vars(py, raw_values, environment(py)?)?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs index d4d335fff6..fdfbe56e1b 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs @@ -1,18 +1,23 @@ //! `value` with every `${...}` template expanded, recording the reads it made. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_templates::expand_config_templates; +use crate::model_config::_helpers::config::template_data::applied; +use crate::model_config::_helpers::config::template_data::environment; +use crate::model_config::models::TemplateSources; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; /// `value` with every `${...}` template expanded, recording the reads it made. -#[inline] pub fn expanded_template_data<'py>( value: &Bound<'py, PyAny>, (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), (options, label): (TemplateOptions, &str), ) -> PyResult> { - template_data::expanded_template_data(value, (variables, context_values), (options, label)) + let py: Python<'py> = value.py(); + let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); + applied( + py, + expand_config_templates(py, value, sources, (options, label))?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs index f8a5513f46..0562f0ecc6 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs @@ -1,13 +1,12 @@ //! A model's config values as typed nodes, read key by key. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::CONFIG_FIELD; +use crate::model_config::main::config::typed_values::typed_values; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A model's config values as typed nodes, read key by key. -#[inline] pub fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - typed_config::model_values(model) + typed_values(&native_field(model, CONFIG_FIELD)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs index ecd8bf24c2..3552e4a3e7 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs @@ -1,14 +1,30 @@ //! Wrap `error` for Python. -use crate::model_config::_helpers::config::config_errors; +use crate::model_config::_helpers::config::config_errors::COMPILE_INPUT_CLASS; +use crate::model_config::_helpers::config::config_errors::CONFIG_VALUE_TYPE_CLASS; +use crate::model_config::_helpers::config::config_errors::DISCOVERY_CONFLICT_CLASS; +use crate::model_config::_helpers::config::config_errors::RESOURCE_IDENTITY_CLASS; use crate::model_config::errors::NativeConfigError; use pyo3::Py; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_model_config::errors::ConfigError; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; /// Wrap `error` for Python. -#[inline] pub fn native_config_error(py: Python<'_>, error: ConfigError) -> PyResult> { - config_errors::native_config_error(py, error) + let class_name = match error.class { + ErrorClass::CompileInput => COMPILE_INPUT_CLASS, + ErrorClass::ConfigValueType => CONFIG_VALUE_TYPE_CLASS, + ErrorClass::ResourceIdentity => RESOURCE_IDENTITY_CLASS, + ErrorClass::DiscoveryConflict => DISCOVERY_CONFLICT_CLASS, + }; + Py::new( + py, + NativeConfigError { + class_name, + message: error.message, + code: error.code, + help: error.help, + key: error.key, + }, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs index a9c52c7bc0..9a590b7d43 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs @@ -1,15 +1,16 @@ //! `config_exception` as the raised error, or the failure building it. -use crate::model_config::_helpers::config::config_errors; +use crate::model_config::_helpers::config::config_errors::config_exception; use pyo3::PyErr; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; /// `config_exception` as the raised error, or the failure building it. -#[inline] pub fn raised_config_exception( error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyAny>>, ) -> PyErr { - config_errors::raised_config_exception(error, values) + match config_exception(error, values) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs index b45e88ec17..ef6515c5da 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs @@ -1,14 +1,21 @@ //! Record native template environment and context reads for project reuse, in order. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::config::template_data::ENVIRONMENT_READ; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_python_base::compile_context::main::input_reads::context_read::context_read; +use sqlbuild_python_base::compile_context::main::input_reads::environment_read::environment_read; /// Record native template environment and context reads for project reuse, in order. -#[inline] pub fn record_reads<'a>( py: Python<'_>, reads: impl IntoIterator, ) -> PyResult<()> { - template_data::record_reads(py, reads) + for (kind, name) in reads { + if kind == ENVIRONMENT_READ { + environment_read(py, name)?; + } else { + context_read(py, name)?; + } + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs index 975be395fa..5916330ff0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs @@ -1,11 +1,10 @@ //! `record_template_reads` as a Python function. -use crate::model_config::_helpers::config::template_data; +use crate::model_config::_helpers::config::template_data::record_template_reads; /// `record_template_reads` as a Python function. -#[inline] pub fn record_template_reads_function( py: pyo3::Python<'_>, ) -> pyo3::PyResult> { - template_data::record_template_reads_function(py) + pyo3::wrap_pyfunction!(record_template_reads, py) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs index bfb2b4dff2..302aea46a9 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs @@ -1,12 +1,10 @@ //! Register the config errors functions and classes. -use crate::model_config::_helpers::config::config_errors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::errors::NativeConfigError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the config errors functions and classes. -#[inline] pub fn register_config_errors(module: &Bound<'_, PyModule>) -> PyResult<()> { - config_errors::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs index 209cb3c148..0b1daefb3a 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs @@ -1,12 +1,19 @@ //! Register the config values functions and classes. -use crate::model_config::_helpers::config::config_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_values::config_bool; +use crate::model_config::_helpers::config::config_values::config_cursor_bound; +use crate::model_config::_helpers::config::config_values::config_int; +use crate::model_config::_helpers::config::config_values::config_string_tuple; +use crate::model_config::main::config::config_str::config_str; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the config values functions and classes. -#[inline] pub fn register_config_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - config_values::register(module) + module.add_function(wrap_pyfunction!(config_str, module)?)?; + module.add_function(wrap_pyfunction!(config_bool, module)?)?; + module.add_function(wrap_pyfunction!(config_string_tuple, module)?)?; + module.add_function(wrap_pyfunction!(config_int, module)?)?; + module.add_function(wrap_pyfunction!(config_cursor_bound, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs index 916f95a334..cb44476667 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs @@ -1,12 +1,17 @@ //! Register the template data functions and classes. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::expand_effective_vars; +use crate::model_config::_helpers::config::template_data::expand_template_data; +use crate::model_config::_helpers::config::template_data::record_template_reads; +use crate::model_config::_helpers::config::template_data::resolve_run_id; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the template data functions and classes. -#[inline] pub fn register_template_data(module: &Bound<'_, PyModule>) -> PyResult<()> { - template_data::register(module) + module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; + module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; + module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; + module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs index 22552f8b28..189ac6700a 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs @@ -1,13 +1,26 @@ //! Attach the built tree to its public record, rather than reconstructing it from that view. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::TYPED_VALUES_SLOT; +use crate::model_config::models::TypedConfigValues; use crate::model_config::types::BuiltConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::store_cached::store_cached; +use sqlbuild_python_base::records::models::NativeRecord; +use std::sync::{Arc, Mutex}; /// Attach the built tree to its public record, rather than reconstructing it from that view. -#[inline] pub fn retain_values(config: &Bound<'_, PyAny>, built: Option<&BuiltConfigValues>) -> PyResult<()> { - typed_config::retain_values(config, built) + if let Some(built) = built + && let Ok(record) = config.cast::() + { + let typed = Bound::new( + config.py(), + TypedConfigValues { + built: Some(Arc::clone(built)), + read: Mutex::new(Vec::new()), + }, + )?; + store_cached(record, TYPED_VALUES_SLOT, &typed.into_any().unbind())?; + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs index c10fda69f6..e2f112aba9 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs @@ -1,12 +1,30 @@ //! The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::RUN_ID_SUFFIX_LENGTH; +use crate::model_config::_helpers::config::template_data::TIMESTAMP_SEPARATORS; +use crate::model_config::_helpers::config::template_data::UUID_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use sqlbuild_python_base::boundary::main::imported::imported; +use std::time::{SystemTime, UNIX_EPOCH}; /// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -#[inline] pub fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { - template_data::run_id(selected_run_id) + let py: Python<'py> = selected_run_id.py(); + if !selected_run_id.is_none() { + return Ok(selected_run_id.clone()); + } + let epoch_seconds: i64 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); + let suffix: String = imported(py, UUID_MODULE, "uuid4")? + .call0()? + .getattr("hex")? + .extract()?; + let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); + Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs index edc8f6f2a5..de6b631cb2 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs @@ -1,11 +1,10 @@ //! The `CompileInputError` Python raises for a template error in the context `label`. -use crate::model_config::_helpers::config::config_templates; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::templates::errors::TemplateError; +use sqlbuild_model_config::templates::main::template_error_message::template_error_message; /// The `CompileInputError` Python raises for a template error in the context `label`. -#[inline] pub fn template_error(error: &TemplateError, label: &str) -> ConfigError { - config_templates::template_error(error, label) + ConfigError::compile(template_error_message(error, label)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs index ca4d969669..63ff6d8a02 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs @@ -1,13 +1,44 @@ //! A config record's values, their typed nodes kept on the record so each key converts once. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::TYPED_VALUES_SLOT; +use crate::model_config::_helpers::config::typed_config::VALUES_FIELD; +use crate::model_config::models::TypedConfigValues; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::cached_value::cached_value; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::store_cached::store_cached; +use sqlbuild_python_base::records::models::NativeRecord; +use std::sync::Mutex; /// A config record's values, their typed nodes kept on the record so each key converts once. -#[inline] pub fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { - typed_config::typed_values(config) + let py: Python<'py> = config.py(); + let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; + let fresh = || { + Bound::new( + py, + TypedConfigValues { + built: None, + read: Mutex::new(Vec::new()), + }, + ) + }; + let Ok(record) = config.cast::() else { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + return Ok((values, typed)); + }; + let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { + Some(typed) => typed.into_bound(py).cast_into()?, + None => { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + store_cached( + record, + TYPED_VALUES_SLOT, + &typed.clone().into_any().unbind(), + )?; + typed + } + }; + Ok((values, typed)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/view.rs b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs index 2f6ed7ebdd..e9eb61520e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/view.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs @@ -1,20 +1,22 @@ //! A `CompileModelConfig` of `values` and the other `fields`. -use crate::model_config::_helpers::config::config_classes; +use crate::model_config::_helpers::config::config_classes::VALUES_FIELD; +use crate::model_config::main::config::retain_values::retain_values; use crate::model_config::models::ConfigClasses; use crate::model_config::types::BuiltConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; /// A `CompileModelConfig` of `values` and the other `fields`. -#[inline] pub fn view<'py>( classes: &ConfigClasses<'py>, values: &Bound<'py, PyDict>, fields: &Bound<'py, PyDict>, nodes: Option<&BuiltConfigValues>, ) -> PyResult> { - config_classes::view(classes, values, fields, nodes) + let arguments: Bound<'py, PyDict> = fields.copy()?; + arguments.set_item(VALUES_FIELD, values)?; + let config = classes.compile_model_config.call((), Some(&arguments))?; + retain_values(&config, nodes)?; + Ok(config) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs index b933a5c862..f234b07fcf 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs @@ -1,20 +1,17 @@ //! Audit tuple. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::audit_instances; use crate::model_config::models::ContractClasses; use crate::model_config::models::PyNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyTuple; use sqlbuild_model_config::header_metadata::models::ParsedAudit; /// Audit tuple. -#[inline] pub fn audit_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: &[ParsedAudit>], ) -> PyResult> { - header_metadata::audit_tuple(py, classes, parsed) + PyTuple::new(py, audit_instances(py, classes, parsed)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs index 040da859c0..510291ef68 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs @@ -1,13 +1,23 @@ //! A Python-authored family contract crosses into native ownership once. -use crate::model_config::_helpers::headers::header_families; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_python_base::records::main::fields::field::field; /// A Python-authored family contract crosses into native ownership once. -#[inline] pub fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { - header_families::family_specs(families) + families + .try_iter()? + .map(|family| { + let family = family?; + Ok(DynamicFamily { + name: field(&family, "name")?.extract()?, + pivot_column: field(&family, "pivot_column")?.extract()?, + value_column: field(&family, "value_column")?.extract()?, + aggregate: field(&family, "aggregate")?.extract()?, + data_type: field(&family, "type")?.extract()?, + name_pattern: field(&family, "name_pattern")?.extract()?, + }) + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs index 78e06cd911..43b13afaae 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs @@ -1,19 +1,26 @@ //! The `SchemaColumn` a native column projects to when its location names `location`'s path. -use crate::model_config::_helpers::headers::header_metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::headers::header_metadata::column_values; +use crate::model_config::_helpers::headers::header_metadata::contract_object; +use crate::model_config::_helpers::headers::header_metadata::location_record; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_discovery::model_files::models::NativeColumn; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// The `SchemaColumn` a native column projects to when its location names `location`'s path. -#[inline] pub fn native_column_record<'py>( py: Python<'py>, column: &NativeColumn, location: Option<(Bound<'py, PyAny>, LineColumnSpan)>, ) -> PyResult> { - header_metadata::native_column_record(py, column, location) + let location: Bound<'py, PyAny> = match location { + Some((path, span)) => location_record(py, path, span)?, + None => py.None().into_bound(py), + }; + contract_object( + &record_class(py, RecordId::SchemaColumn)?, + column_values(py, column, location)?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs index 5fe9654ff2..0810369a23 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs @@ -1,22 +1,51 @@ //! Native schema column. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::ColumnFields; +use crate::model_config::_helpers::headers::header_metadata::optional; use crate::model_config::models::ContractClasses; +use crate::model_config::models::HeaderAudits; use crate::model_config::models::HeaderColumn; use crate::model_config::types::HeaderValue; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_model_config::header_metadata::models::ParsedColumn; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// Native schema column. -#[inline] pub fn native_schema_column<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, column: &ParsedColumn, locations: &Bound<'py, PyDict>, ) -> PyResult { - header_metadata::native_schema_column(py, classes, column, locations) + let location = locations + .get_item(column.name.object(py)?)? + .unwrap_or_else(|| py.None().into_bound(py)); + let audits = HeaderAudits::new(py, classes, &column.audits, &location)?; + let projected_audits = if classes + .schema_column + .is(&record_class(py, RecordId::SchemaColumn)?) + { + PyTuple::empty(py) + } else { + audits.project(py)? + }; + let mut result = HeaderColumn::new( + &classes.schema_column, + [ + column.name.object(py)?, + optional(py, column.column_type.as_ref())?, + optional(py, column.nullable.as_ref())?, + optional(py, column.description.as_ref())?, + PyDict::new(py).into_any(), + projected_audits.into_any(), + location, + optional(py, column.migrate_from.as_ref())?, + ], + )?; + if !matches!(result.fields, ColumnFields::Projected) { + result.audits = Some(audits); + } + Ok(result) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs index aa2fc6838a..a8223d0eb2 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs @@ -1,16 +1,18 @@ //! `audit.name or audit.definition_name` of the instance `audit` builds. -use crate::model_config::_helpers::headers::header_metadata; use crate::model_config::types::HeaderValue; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; use sqlbuild_model_config::header_metadata::models::ParsedAudit; /// `audit.name or audit.definition_name` of the instance `audit` builds. -#[inline] pub fn parsed_audit_identity( py: Python<'_>, audit: &ParsedAudit, ) -> PyResult { - header_metadata::parsed_audit_identity(py, audit) + if let Some(name) = &audit.name + && name.object(py)?.is_truthy()? + { + return name.exact_text(py); + } + audit.definition_name.exact_text(py) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs index 398b17e024..af24bfc037 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs @@ -1,12 +1,10 @@ //! Register the header columns functions and classes. -use crate::model_config::_helpers::headers::header_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::headers::header_columns::parse_schema_columns; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header columns functions and classes. -#[inline] pub fn register_header_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_columns::register(module) + module.add_function(wrap_pyfunction!(parse_schema_columns, module)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs index 6c5318feff..8a6a90a109 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs @@ -1,21 +1,35 @@ //! The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -use crate::model_config::_helpers::headers::header_columns; use crate::model_config::_helpers::headers::header_columns::ColumnOptions; use crate::model_config::_helpers::headers::header_columns::ColumnSite; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::model_config::_helpers::headers::header_columns::raised; +use crate::model_config::_helpers::headers::header_metadata::column_tuple; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::PyNode; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_schema_columns as parse_columns; +use sqlbuild_model_config::header_metadata::models::ColumnRequest; /// The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -#[inline] pub fn schema_columns<'py>( raw_columns: &Bound<'py, PyAny>, site: ColumnSite<'_>, options: ColumnOptions, column_locations: &Bound<'py, PyDict>, ) -> PyResult> { - header_columns::schema_columns(raw_columns, site, options, column_locations) + let py: Python<'py> = raw_columns.py(); + let (path, label) = site; + let (require_columns, allow_migrate_from, declaration_errors) = options; + let request = ColumnRequest { + path, + label, + require_columns, + allow_migrate_from, + }; + let columns = parse_columns(&PyNode(raw_columns.clone()), request) + .map_err(|error| raised(py, error, declaration_errors))?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; + column_tuple(py, &classes, &columns, column_locations) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs index 74ba4ef14c..8524b7a440 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs @@ -1,19 +1,25 @@ //! `MeasurementThresholds` of `parsed`, or `None`. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::contract_object; +use crate::model_config::_helpers::headers::header_metadata::threshold_bound; use crate::model_config::models::ContractClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_model_config::header_metadata::models::ParsedThresholds; /// `MeasurementThresholds` of `parsed`, or `None`. -#[inline] pub fn thresholds<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: Option<&ParsedThresholds>, ) -> PyResult> { - header_metadata::thresholds(py, classes, parsed) + let Some(parsed) = parsed else { + return Ok(py.None().into_bound(py)); + }; + contract_object( + &classes.measurement_thresholds, + [ + threshold_bound(py, classes, parsed.warn)?, + threshold_bound(py, classes, parsed.error)?, + ], + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs index 583bffaebc..81c6ac15f5 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs @@ -1,18 +1,35 @@ //! The first column-rename problem for a model with these config values, if any. -use crate::model_config::_helpers::model_nodes::column_migrations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::model_config::_helpers::model_nodes::column_migrations::MATERIALIZED_KEY; +use crate::model_config::_helpers::model_nodes::column_migrations::MIGRATE_FROM_KEY; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::model_validation::main::validate_column_migrations::validate_column_migrations; use sqlbuild_model_config::model_validation::models::ColumnMigration; /// The first column-rename problem for a model with these config values, if any. -#[inline] pub fn column_migration_error( model_name: &str, values: &Bound<'_, PyDict>, columns: &[ColumnMigration], ) -> PyResult> { - column_migrations::column_migration_error(model_name, values, columns) + if columns.iter().all(|column| column.migrate_from.is_none()) { + return Ok(None); + } + let materialized: Option = match values.get_item(MATERIALIZED_KEY)? { + None => None, + Some(value) if value.is_none() => None, + Some(value) => match value.downcast::() { + Ok(text) => Some(text.to_str()?.to_owned()), + Err(_) => return Ok(Some(ConfigError::config_value_type(MATERIALIZED_KEY))), + }, + }; + let model_migrates: bool = values + .get_item(MIGRATE_FROM_KEY)? + .is_some_and(|value| !value.is_none()); + Ok( + validate_column_migrations(model_name, materialized.as_deref(), model_migrates, columns) + .err(), + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs index 47eaec549e..46b6ce4e6c 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs @@ -1,13 +1,13 @@ //! The contract classes native header parsing builds columns and audits as. -use crate::model_config::_helpers::model_nodes::contract_classes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::contract_classes::CONTRACT_CLASS_TABLE; +use crate::model_config::_helpers::model_nodes::contract_classes::built_contract_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The contract classes native header parsing builds columns and audits as. -#[inline] pub fn contract_classes(py: Python<'_>) -> PyResult> { - contract_classes::contract_classes(py) + CONTRACT_CLASS_TABLE + .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs index 699db0625e..17504a2ff0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs @@ -1,9 +1,7 @@ //! The file path of each model whose layered config or model schema gives it a description. use crate::model_config::_helpers::model_nodes::described_models; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// The file path of each model whose layered config or model schema gives it a description. diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs index 62a6e2593b..76fe232d6d 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs @@ -1,12 +1,20 @@ //! The Python exception a native config error raises as. -use crate::model_config::_helpers::model_nodes::model_config_builder; +use crate::model_config::_helpers::config::config_errors::config_exception; +use crate::model_config::errors::NativeConfigError; +use crate::model_config::main::config::native_config_error::native_config_error; use pyo3::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{Py, PyResult, Python}; use sqlbuild_model_config::errors::ConfigError; /// The Python exception a native config error raises as. -#[inline] pub fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { - model_config_builder::raised_config_error(py, error) + let raised = || -> PyResult { + let error: Py = native_config_error(py, error)?; + Ok(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, + )?)) + }; + raised().unwrap_or_else(|error| error) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs index c191e8b135..c36feeb817 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs @@ -1,12 +1,10 @@ //! Register the model config builder functions and classes. -use crate::model_config::_helpers::model_nodes::model_config_builder; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::models::NativeModelConfigBuilder; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the model config builder functions and classes. -#[inline] pub fn register_model_config_builder(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_config_builder::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs index bd5ccb345e..eb1686b972 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs @@ -1,12 +1,10 @@ //! Register the model validator functions and classes. -use crate::model_config::_helpers::model_nodes::model_validator; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::models::NativeModelValidator; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the model validator functions and classes. -#[inline] pub fn register_model_validator(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_validator::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs index 6399b67927..c1af74c7ab 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs @@ -1,12 +1,9 @@ //! Register the unicode environment functions and classes. -use crate::model_config::_helpers::model_nodes::unicode_environment; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::model_nodes::unicode_environment::UnicodeEnvironment; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the unicode environment functions and classes. -#[inline] pub fn register_unicode_environment(module: &Bound<'_, PyModule>) -> PyResult<()> { - unicode_environment::register(module) + module.add_class::() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs index 59c24504a6..d15d1bc440 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs @@ -1,13 +1,9 @@ //! A new view of the process environment as compile text reads it. -use crate::model_config::_helpers::model_nodes::unicode_environment; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::unicode_environment::UnicodeEnvironment; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// A new view of the process environment as compile text reads it. -#[inline] pub fn unicode_environment(py: Python<'_>) -> PyResult> { - unicode_environment::unicode_environment(py) + Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs index bc68a1655a..f0e9b701ee 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs @@ -1,13 +1,13 @@ //! The classes parsed seed and source declarations become. -use crate::model_config::_helpers::model_nodes::contract_classes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::contract_classes::YAML_ENTRY_CLASS_TABLE; +use crate::model_config::_helpers::model_nodes::contract_classes::built_yaml_entry_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The classes parsed seed and source declarations become. -#[inline] pub fn yaml_entry_classes(py: Python<'_>) -> PyResult> { - contract_classes::yaml_entry_classes(py) + YAML_ENTRY_CLASS_TABLE + .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/models.rs b/crates/sqlbuild-python-front/src/model_config/models.rs index 2b2d80cc80..c1b99e21ea 100644 --- a/crates/sqlbuild-python-front/src/model_config/models.rs +++ b/crates/sqlbuild-python-front/src/model_config/models.rs @@ -6,12 +6,9 @@ use crate::authored_values::main::project_var_text::project_var_text; use crate::discovery::main::project::each::each; use crate::discovery::models::NativeModelFile; use crate::model_config::_helpers::config::config_classes::built_config; -use crate::model_config::_helpers::config::config_errors::native_config_error; use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; use crate::model_config::_helpers::config::config_nodes::config_dict; use crate::model_config::_helpers::config::config_nodes::config_entries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::_helpers::config::config_templates::CONTEXT_READ; use crate::model_config::_helpers::config::config_templates::EFFECTIVE_VARS_LABEL; use crate::model_config::_helpers::config::config_templates::ENVIRONMENT_READ; @@ -19,7 +16,6 @@ use crate::model_config::_helpers::config::config_templates::EffectiveVars; use crate::model_config::_helpers::headers::header_audits::authored_audit; use crate::model_config::_helpers::headers::header_audits::config_json_value; use crate::model_config::_helpers::headers::header_audits::retain_audit; -use crate::model_config::_helpers::headers::header_families::family_specs; use crate::model_config::_helpers::headers::header_metadata::ColumnFields; use crate::model_config::_helpers::headers::header_metadata::SCHEMA_COLUMN_FIELDS; use crate::model_config::_helpers::headers::header_metadata::audit_instance; @@ -28,9 +24,7 @@ use crate::model_config::_helpers::headers::header_metadata::contract_object; use crate::model_config::_helpers::headers::header_metadata::native_location; use crate::model_config::_helpers::headers::header_metadata::native_value; use crate::model_config::_helpers::headers::header_metadata::native_values; -use crate::model_config::_helpers::headers::header_metadata::thresholds; use crate::model_config::_helpers::model_nodes::authored_nodes::entry_nodes; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; use crate::model_config::_helpers::model_nodes::model_config_builder::CONTEXT_KEYS; use crate::model_config::_helpers::model_nodes::model_config_builder::HeaderBuildInputs; use crate::model_config::_helpers::model_nodes::model_config_builder::PythonConfigHost; @@ -46,6 +40,12 @@ use crate::model_config::_helpers::model_nodes::model_validator::discovered_name use crate::model_config::_helpers::model_nodes::model_validator::stop_object; use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; use crate::model_config::errors::NativeConfigError; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::headers::family_specs::family_specs; +use crate::model_config::main::headers::thresholds::thresholds; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use crate::model_config::types::BuiltConfigValues; use crate::model_config::types::HeaderValue; use crate::model_config::types::PyConfigNode; diff --git a/crates/sqlbuild-python-front/src/model_config/types.rs b/crates/sqlbuild-python-front/src/model_config/types.rs index bf2205a325..26a64f4fcc 100644 --- a/crates/sqlbuild-python-front/src/model_config/types.rs +++ b/crates/sqlbuild-python-front/src/model_config/types.rs @@ -1,6 +1,6 @@ //! Public traits and type aliases of model config. -use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::main::config::config_node::config_node; use crate::model_config::models::HostHandle; use crate::model_config::models::TypedConfigValues; use pyo3::prelude::Bound; diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs index 2a2ecb2147..f6947339b1 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs @@ -1,16 +1,15 @@ //! Pruning of the audit results state table through the adapter. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::native_host::main::state_table_name::state_table_name; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::constants::AUDIT_RESULTS_TABLE_NAME; use sqlbuild_state::audit_results::main::prune_audit_results_sql::prune_audit_results_sql; -use crate::native_host::_helpers::consumption_state::state_table_name; - /// SQL deleting audit results beyond the newest `retain_versions` per audit binding. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_audit_results_prune_sql( +pub(crate) fn build_audit_results_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -24,7 +23,3 @@ fn build_audit_results_prune_sql( .map(|table| prune_audit_results_sql(&table, retain_versions)) .unwrap_or_default()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 59e36ee016..4414633a9f 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -1,16 +1,22 @@ //! The consumption table through the adapter, and the connection and callbacks a recording build holds. use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::execute::execute; +use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use crate::native_host::main::integer_literal::integer_literal; +use crate::native_host::main::literal::literal; +use crate::native_host::main::optional_text::optional_text; +use crate::native_host::main::state_table_name::state_table_name; use crate::native_host::models::Location; use chrono::{DateTime, NaiveDateTime, Utc}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyInt, PyString}; use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_state::consumption::constants::{ - CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, + CONSUMPTION_COLUMN_TYPES, NULLABLE_COLUMNS, RECENT_EXECUTIONS, }; use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; @@ -18,10 +24,8 @@ use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_ use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, }; -use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::state_rows::main::tables::state_table_create_sql::state_table_create_sql; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,27 +41,6 @@ pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -/// The adapter's qualified name of a state table in `(database, schema)`, if it has one. -pub(crate) fn state_table_name( - render_qualified_name: &Bound<'_, PyAny>, - (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - table: &str, -) -> PyResult> { - let py: Python<'_> = render_qualified_name.py(); - let name: Bound<'_, PyAny> = call_with( - render_qualified_name, - &[ - ("database", database.clone()), - ("schema", schema.clone()), - ("name", text(py, table)), - ], - )?; - if name.is_none() { - return Ok(None); - } - Ok(Some(name.extract()?)) -} - fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { let py: Python<'_> = adapter.py(); let database: Bound<'_, PyAny> = location @@ -74,7 +57,7 @@ fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -91,43 +74,10 @@ fn build_consumption_create_table_sql( ) } -/// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -pub(crate) fn fixed_state_table_sql( - (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), -) -> PyResult { - let py: Python<'_> = database.py(); - let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? - else { - return Ok(String::new()); - }; - let columns: Bound<'_, PyAny> = - PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); - let types: Bound<'_, PyDict> = PyDict::new(py); - for (column, declared) in column_types { - types.set_item(column, declared)?; - } - let required: Vec<&str> = column_types - .iter() - .map(|(column, _)| *column) - .filter(|column| !nullable.contains(column)) - .collect(); - state_table_create_sql( - &table, - ( - &columns, - &types.into_any(), - &PyFrozenSet::new(py, &required)?.into_any(), - ), - render_framework_type, - ) -} - /// SQL deleting whole consumption executions beyond the newest `retain_versions` per relation. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_consumption_prune_sql( +pub(crate) fn build_consumption_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -142,20 +92,6 @@ fn build_consumption_prune_sql( .unwrap_or_default()) } -pub(crate) fn execute<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - sql: &str, -) -> PyResult> { - call_with( - &native_field(adapter, "execute")?, - &[ - ("connection", connection.clone()), - ("sql", text(adapter.py(), sql)), - ], - ) -} - /// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. pub(crate) fn ensure_table( adapter: &Bound<'_, PyAny>, @@ -248,13 +184,6 @@ pub(crate) fn read_executions( Ok(rows) } -pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - Ok(Some(value.str()?.extract()?)) -} - fn required_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(optional_text(value)?.unwrap_or_default()) } @@ -323,7 +252,7 @@ fn read_row(row: &Bound<'_, PyAny>) -> PyResult { } /// A nullable state value as a SQL literal of `declared_type`. -fn state_literal( +pub(crate) fn state_literal( py: Python<'_>, value: Option>, declared_type: &str, @@ -332,22 +261,6 @@ fn state_literal( render_state_sql_literal(py, &value, declared_type, None) } -pub(crate) fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { - state_literal( - py, - value.map(|value| text(py, value)), - STRING_STATE_VALUE_TYPE, - ) -} - -pub(crate) fn integer_literal(py: Python<'_>, value: Option) -> PyResult { - state_literal( - py, - value.map(|value| PyInt::new(py, value).into_any()), - INTEGER_STATE_VALUE_TYPE, - ) -} - fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyResult> { let input = row.input.as_ref(); Ok(vec![ @@ -398,21 +311,3 @@ pub(crate) fn append( } Ok(()) } - -/// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -pub(crate) fn recorder_callbacks<'py>( - recorder: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok(( - recorder.getattr("on_node_complete")?, - recorder.getattr("before_node_write")?, - )) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_consumption_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs index bb8276aadc..72254144d4 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs @@ -1,29 +1,19 @@ //! DDL and pruning of the display-only observations table through the adapter. -use crate::native_host::models::ObservationAppend; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use crate::native_host::main::state_table_name::state_table_name; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; use sqlbuild_state::observations::constants::{ NULLABLE_OBSERVATION_COLUMNS, OBSERVATION_COLUMN_TYPES, }; use sqlbuild_state::observations::main::prune_observations_sql::prune_observations_sql; -use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use uuid::Uuid; - -use crate::native_host::_helpers::consumption_state::execute; -use crate::native_host::_helpers::consumption_state::fixed_state_table_sql; -use crate::native_host::_helpers::consumption_state::literal; -use crate::native_host::_helpers::consumption_state::state_table_name; -use crate::native_host::constants::ROWS_PER_STATEMENT; -use crate::native_host::models::Location; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Create-if-missing DDL for the observations table, with adapter-rendered column types. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] -fn build_observations_create_table_sql( +pub(crate) fn build_observations_create_table_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -43,7 +33,7 @@ fn build_observations_create_table_sql( /// SQL deleting observations beyond the newest `retain_versions` per kind and node. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_observations_prune_sql( +pub(crate) fn build_observations_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -57,71 +47,3 @@ fn build_observations_prune_sql( .map(|table| prune_observations_sql(&table, retain_versions)) .unwrap_or_default()) } - -/// Create the schema and observations table in `location` if missing, then append `rows`. -pub(crate) fn append_observations( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, - rows: &[ObservationAppend], -) -> PyResult<()> { - let py: Python<'_> = adapter.py(); - let keywords = location.keywords(py); - let Some(table) = state_table_name( - &native_field(adapter, "render_qualified_name")?, - (&keywords[0].1, &keywords[1].1), - OBSERVATIONS_TABLE_NAME, - )? - else { - return Ok(()); - }; - for statement in - call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? - { - execute(adapter, connection, &statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(adapter, "render_create_observations_table_sql")?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - execute(adapter, connection, &create)?; - } - let observed_at: String = - call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; - let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .collect(); - for chunk in rows.chunks(ROWS_PER_STATEMENT) { - let cells: Vec> = chunk - .iter() - .map(|row| { - Ok(vec![ - literal(py, Some(&Uuid::now_v7().to_string()))?, - literal(py, Some(row.kind))?, - literal(py, Some(&row.node))?, - literal(py, row.cursor_type.as_deref())?, - literal(py, Some(&row.value))?, - literal(py, Some(&row.run_id))?, - observed_at.clone(), - ]) - }) - .collect::>()?; - execute( - adapter, - connection, - &values_insert_sql(&table, &columns, &cells), - )?; - } - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_observations_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs index 5d63443aa6..8fe6343d54 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs @@ -1,19 +1,75 @@ //! Create the schema and observations table in `location` if missing, then append `rows`. -use crate::native_host::_helpers::observations_state; +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::execute::execute; +use crate::native_host::main::literal::literal; +use crate::native_host::main::state_table_name::state_table_name; use crate::native_host::models::Location; use crate::native_host::models::ObservationAppend; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use uuid::Uuid; /// Create the schema and observations table in `location` if missing, then append `rows`. -#[inline] pub fn append_observations( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, rows: &[ObservationAppend], ) -> PyResult<()> { - observations_state::append_observations(adapter, connection, location, rows) + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + let Some(table) = state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + OBSERVATIONS_TABLE_NAME, + )? + else { + return Ok(()); + }; + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_observations_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + let observed_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in rows.chunks(ROWS_PER_STATEMENT) { + let cells: Vec> = chunk + .iter() + .map(|row| { + Ok(vec![ + literal(py, Some(&Uuid::now_v7().to_string()))?, + literal(py, Some(row.kind))?, + literal(py, Some(&row.node))?, + literal(py, row.cursor_type.as_deref())?, + literal(py, Some(&row.value))?, + literal(py, Some(&row.run_id))?, + observed_at.clone(), + ]) + }) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/execute.rs b/crates/sqlbuild-python-front/src/native_host/main/execute.rs index 1c9e3b6ff0..b0b3f91534 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/execute.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/execute.rs @@ -1,16 +1,21 @@ //! Execute. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Execute. -#[inline] pub fn execute<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, sql: &str, ) -> PyResult> { - consumption_state::execute(adapter, connection, sql) + call_with( + &native_field(adapter, "execute")?, + &[ + ("connection", connection.clone()), + ("sql", text(adapter.py(), sql)), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs index 91cfd5cb21..93299a7407 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs @@ -1,20 +1,39 @@ //! Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::main::state_table_name::state_table_name; +use crate::state_rows::main::tables::state_table_create_sql::state_table_create_sql; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; /// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -#[inline] pub fn fixed_state_table_sql( (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), ) -> PyResult { - consumption_state::fixed_state_table_sql( - (database, schema), - (render_qualified_name, render_framework_type), - (table_name, column_types, nullable), + let py: Python<'_> = database.py(); + let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? + else { + return Ok(String::new()); + }; + let columns: Bound<'_, PyAny> = + PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); + let types: Bound<'_, PyDict> = PyDict::new(py); + for (column, declared) in column_types { + types.set_item(column, declared)?; + } + let required: Vec<&str> = column_types + .iter() + .map(|(column, _)| *column) + .filter(|column| !nullable.contains(column)) + .collect(); + state_table_create_sql( + &table, + ( + &columns, + &types.into_any(), + &PyFrozenSet::new(py, &required)?.into_any(), + ), + render_framework_type, ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs index 290ad1dd2a..b20b3d4ebf 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs @@ -1,11 +1,15 @@ //! Integer literal. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::native_host::_helpers::consumption_state::state_literal; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyInt; +use sqlbuild_state::consumption::constants::INTEGER_STATE_VALUE_TYPE; /// Integer literal. -#[inline] pub fn integer_literal(py: Python<'_>, value: Option) -> PyResult { - consumption_state::integer_literal(py, value) + state_literal( + py, + value.map(|value| PyInt::new(py, value).into_any()), + INTEGER_STATE_VALUE_TYPE, + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/literal.rs b/crates/sqlbuild-python-front/src/native_host/main/literal.rs index 5a274ff9fb..c322e768a0 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/literal.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/literal.rs @@ -1,11 +1,15 @@ //! Literal. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::native_host::_helpers::consumption_state::state_literal; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; /// Literal. -#[inline] pub fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { - consumption_state::literal(py, value) + state_literal( + py, + value.map(|value| text(py, value)), + STRING_STATE_VALUE_TYPE, + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs index 1babb19549..affa4292b7 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs @@ -1,12 +1,11 @@ //! Optional text. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// Optional text. -#[inline] pub fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { - consumption_state::optional_text(value) + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs index 3a3544c506..a9e0baa682 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs @@ -1,14 +1,13 @@ //! The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -#[inline] pub fn recorder_callbacks<'py>( recorder: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - consumption_state::recorder_callbacks(recorder) + Ok(( + recorder.getattr("on_node_complete")?, + recorder.getattr("before_node_write")?, + )) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs index d7cdba3673..1fd72949b3 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs @@ -1,12 +1,10 @@ //! Register the audit results state functions and classes. -use crate::native_host::_helpers::audit_results_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::audit_results_state::build_audit_results_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the audit results state functions and classes. -#[inline] pub fn register_audit_results_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - audit_results_state::register(module) + module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs index 84da4f5782..6fc37e3d1f 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs @@ -1,12 +1,15 @@ //! Register the consumption state functions and classes. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::build_consumption_create_table_sql; +use crate::native_host::_helpers::consumption_state::build_consumption_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the consumption state functions and classes. -#[inline] pub fn register_consumption_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - consumption_state::register(module) + module.add_function(wrap_pyfunction!( + build_consumption_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs index 97ab0d3395..d94dbf8327 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs @@ -1,12 +1,15 @@ //! Register the observations state functions and classes. -use crate::native_host::_helpers::observations_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::observations_state::build_observations_create_table_sql; +use crate::native_host::_helpers::observations_state::build_observations_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the observations state functions and classes. -#[inline] pub fn register_observations_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - observations_state::register(module) + module.add_function(wrap_pyfunction!( + build_observations_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs index 5d1c8fd59c..26fb727b81 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs @@ -1,16 +1,26 @@ //! The adapter's qualified name of a state table in `(database, schema)`, if it has one. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::call_with::call_with; /// The adapter's qualified name of a state table in `(database, schema)`, if it has one. -#[inline] pub fn state_table_name( render_qualified_name: &Bound<'_, PyAny>, (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), table: &str, ) -> PyResult> { - consumption_state::state_table_name(render_qualified_name, (database, schema), table) + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, table)), + ], + )?; + if name.is_none() { + return Ok(None); + } + Ok(Some(name.extract()?)) } diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs index 374c433043..81b7873466 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs @@ -1,9 +1,9 @@ //! Contract adoption's comparison of authored contracts with physical warehouse columns. use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::AdapterHost; @@ -24,7 +24,7 @@ const SOURCE_RESOURCE: &str = "source"; /// Python callables and classes the comparison builds findings with. struct Vocabulary<'py> { finding: Bound<'py, PyAny>, - kinds: Bound<'py, PyAny>, + pub(crate) kinds: Bound<'py, PyAny>, } impl<'py> Vocabulary<'py> { @@ -435,7 +435,7 @@ fn compared<'py>( /// `item` compared with its physical columns, or marked missing when there are none. #[pyfunction] #[pyo3(signature = (*, item, physical_columns, adapter))] -fn compare_contract_evidence<'py>( +pub(crate) fn compare_contract_evidence<'py>( py: Python<'py>, item: &Bound<'py, PyAny>, physical_columns: &Bound<'py, PyAny>, @@ -508,9 +508,3 @@ pub(crate) fn compare_contracts<'py>( } PyTuple::new(py, evidence) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compare_contracts, module)?)?; - module.add_function(wrap_pyfunction!(compare_contract_evidence, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs index 02a4a5b0a4..9a2217131f 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs @@ -1,36 +1,24 @@ //! Contract adoption's MODEL header and YAML source edits, computed natively. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; -use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; -use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; -use sqlbuild_refactor::refactoring::models::{ - ContractEdit, ContractModelColumn, ContractModelRequest, ContractSourceRequest, -}; +use sqlbuild_refactor::refactoring::models::ContractEdit; use sqlbuild_refactor::refactoring::types::PhysicalColumn; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; /// Finding kinds an edit resolves without overwrite, then the ones overwrite also resolves. -const RESOLVED_KINDS: ([&str; 1], [&str; 2]) = ( +pub(crate) const RESOLVED_KINDS: ([&str; 1], [&str; 2]) = ( ["missing_declaration"], ["type_mismatch", "missing_physical_column"], ); /// `(edited contents, ownership conflict, ContractAdoptionError message)`. -type EditRow = (Option, Option, Option); -fn edit_row(result: Result) -> PyResult { +pub(crate) type EditRow = (Option, Option, Option); +pub(crate) fn edit_row(result: Result) -> PyResult { match result { Ok(ContractEdit::Edited(contents)) => Ok((Some(contents), None, None)), Ok(ContractEdit::Conflict(conflict)) => Ok((None, Some(conflict), None)), @@ -42,12 +30,15 @@ fn edit_row(result: Result) -> PyResult { } /// `ContractAdoptionError(message)`. -fn adoption_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn adoption_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::ContractAdoption, &message) } /// `(edited contents, ownership conflict)`, raising the edit's adoption failure. -fn edit_outcome(py: Python<'_>, row: EditRow) -> PyResult<(Option, Option)> { +pub(crate) fn edit_outcome( + py: Python<'_>, + row: EditRow, +) -> PyResult<(Option, Option)> { let (contents, conflict, failure) = row; match failure { Some(failure) => Err(adoption_error(py, failure)), @@ -55,7 +46,7 @@ fn edit_outcome(py: Python<'_>, row: EditRow) -> PyResult<(Option, Optio } } -fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { let mut rows: Vec = Vec::new(); for column in columns.try_iter()? { let column: Bound<'_, PyAny> = column?; @@ -66,130 +57,3 @@ fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { } Ok(rows) } - -/// The model's MODEL header with physical columns adopted, or the ownership conflict. -#[pyfunction] -#[pyo3(signature = (*, model, physical_columns, overwrite))] -pub(crate) fn edit_contract_model( - py: Python<'_>, - model: &Bound<'_, PyAny>, - physical_columns: &Bound<'_, PyAny>, - overwrite: bool, -) -> PyResult<(Option, Option)> { - let schema: Bound<'_, PyAny> = native_field(model, "schema_entry")?; - let mut columns: Vec = Vec::new(); - let mut dynamic_patterns: Vec> = Vec::new(); - let shared_schema: bool = - !schema.is_none() && !native_field(&schema, "model_schema")?.is_none(); - if !schema.is_none() { - for column in native_field(&schema, "columns")?.try_iter()? { - let column: Bound<'_, PyAny> = column?; - let location: Bound<'_, PyAny> = native_field(&column, "location")?; - let location_path: Option = if location.is_none() { - None - } else { - Some(native_field(&location, "path")?.str()?.extract()?) - }; - columns.push(ContractModelColumn { - name: native_field(&column, "name")?.extract()?, - declared_type: native_field(&column, "type")?.extract()?, - location_path, - }); - } - for family in native_field(&schema, "dynamic_columns")?.try_iter()? { - dynamic_patterns.push(native_field(&family?, "name_pattern")?.extract()?); - } - } - let request = ContractModelRequest { - contents: native_field(model, "authored_sql")?.extract()?, - relative_path: native_field(model, "relative_path")?.str()?.extract()?, - shared_schema, - columns, - dynamic_patterns, - physical_columns: physical_rows(physical_columns)?, - overwrite, - }; - let row: EditRow = compiler_guard(|| edit_row(edit_model_contract(&request)))?; - edit_outcome(py, row) -} - -/// The source's YAML declaration with physical columns adopted, or the ownership conflict. -#[pyfunction] -#[pyo3(signature = (*, source, physical_columns, overwrite, contents = None))] -pub(crate) fn edit_contract_source( - source: &Bound<'_, PyAny>, - physical_columns: &Bound<'_, PyAny>, - overwrite: bool, - contents: Option, -) -> PyResult<(Option, Option)> { - let discovered_contents: String = - native_field(&native_field(source, "source_file")?, "contents")?.extract()?; - let mut columns: Vec<(String, Option)> = Vec::new(); - for column in native_field(&native_field(source, "source_entry")?, "columns")?.try_iter()? { - let column: Bound<'_, PyAny> = column?; - columns.push(( - native_field(&column, "name")?.extract()?, - native_field(&column, "type")?.extract()?, - )); - } - let request = ContractSourceRequest { - contents: contents.unwrap_or_else(|| discovered_contents.clone()), - discovered_contents, - name: native_field(source, "name")?.extract()?, - columns, - physical_columns: physical_rows(physical_columns)?, - overwrite, - }; - let row: EditRow = compiler_guard(|| edit_row(Ok(edit_source_contract(&request))))?; - edit_outcome(source.py(), row) -} - -/// The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -#[pyfunction] -#[pyo3(signature = (*, findings, overwrite, conflict))] -pub(crate) fn remaining_contract_findings<'py>( - py: Python<'py>, - findings: &Bound<'py, PyAny>, - overwrite: bool, - conflict: Option, -) -> PyResult> { - let findings: Vec> = findings.try_iter()?.collect::>()?; - if let Some(conflict) = conflict { - let Some(first) = findings.first() else { - return Err(adoption_error( - py, - "contract ownership conflict requires resource evidence".to_owned(), - )); - }; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("resource_type", native_field(first, "resource_type")?)?; - keywords.set_item("resource_name", native_field(first, "resource_name")?)?; - keywords.set_item( - "kind", - enum_member(py, RecordId::ContractFindingKind, "OWNERSHIP_CONFLICT")?, - )?; - keywords.set_item("message", conflict)?; - let ownership: Bound<'py, PyAny> = - new_record(py, RecordId::ContractFinding, (), Some(&keywords))?; - let mut remaining: Vec> = findings.clone(); - remaining.push(ownership); - return PyTuple::new(py, remaining); - } - let mut remaining: Vec> = Vec::new(); - for finding in findings { - let kind: String = native_field(&finding, "kind")?.str()?.extract()?; - let resolved: bool = RESOLVED_KINDS.0.contains(&kind.as_str()) - || (overwrite && RESOLVED_KINDS.1.contains(&kind.as_str())); - if !resolved { - remaining.push(finding); - } - } - PyTuple::new(py, remaining) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(edit_contract_model, module)?)?; - module.add_function(wrap_pyfunction!(edit_contract_source, module)?)?; - module.add_function(wrap_pyfunction!(remaining_contract_findings, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs index a4df31dcdc..bfbd057003 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs @@ -2,9 +2,9 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_pysource::string_literals::main::string_literal_locations::string_literal_locations; use sqlbuild_pysource::string_literals::models::StringQuery; @@ -100,8 +100,3 @@ pub(crate) fn find_python_string_locations<'py>( } PyTuple::new(py, locations) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(find_python_string_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs index bdee9fe23e..e2c2fb5d6d 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_refactor::refactoring::models::{ AuthoredFile, DeclarationPlacement, DependencyFacts, DestinationFacts, DiscoveredFile, ExpansionFacts, ExpansionSpan, LocationFacts, ManualLocation, ModelFacts, PlacedDeclaration, - RefactorFacts, RefactorOperation, RefactorRequest, RelocatedDeclaration, ResourceColumnsFacts, + RefactorOperation, RefactorRequest, RelocatedDeclaration, ResourceColumnsFacts, SchemaColumnFacts, SqlFileRole, UnitTestFacts, }; use sqlbuild_scopes::scope_index::models::{ResourceIdentity, ResourceKind}; @@ -17,11 +17,10 @@ use crate::refactor::_helpers::python_strings::find_python_string_locations; use crate::scopes::main::checks::relocated_declarations::relocated_declarations; use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; use crate::scopes::main::reports::scope_move_preview::scope_move_preview; -use crate::scopes::main::reports::scope_rebound_usages::scope_rebound_usages; use sqlbuild_python_base::records::main::fields::field::field as native_field; const SQL_SUFFIX: &str = ".sql"; -const GENERIC_DIALECT: &str = "generic"; +pub(crate) const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; const MODEL_KIND_PREFIX: &str = "model:"; @@ -30,7 +29,7 @@ fn posix(path: &Bound<'_, PyAny>) -> PyResult { path.call_method0("as_posix")?.extract() } -fn discovered_files(files: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn discovered_files(files: &Bound<'_, PyAny>) -> PyResult> { let mut discovered: Vec = Vec::new(); for item in files.try_iter()? { let item: Bound<'_, PyAny> = item?; @@ -49,7 +48,7 @@ fn column_names(columns: &Bound<'_, PyAny>) -> PyResult> { .collect() } -fn resource_columns( +pub(crate) fn resource_columns( resources: &Bound<'_, PyAny>, entry: &str, ) -> PyResult> { @@ -113,7 +112,10 @@ fn schema_columns(entry: &Bound<'_, PyAny>) -> PyResult, expansion: Option<&Bound<'_, PyAny>>) -> PyResult { +pub(crate) fn model( + model: &Bound<'_, PyAny>, + expansion: Option<&Bound<'_, PyAny>>, +) -> PyResult { let py: Python<'_> = model.py(); let values: Bound<'_, PyDict> = PyDict::new(py); values.call_method1( @@ -158,38 +160,6 @@ fn model(model: &Bound<'_, PyAny>, expansion: Option<&Bound<'_, PyAny>>) -> PyRe }) } -/// Every compiled model's refactoring facts, or none without a project. -pub(crate) fn model_facts( - refactor_project: &Bound<'_, PyAny>, -) -> PyResult>> { - if refactor_project.is_none() { - return Ok(None); - } - let project: Bound<'_, PyAny> = - native_field(&native_field(refactor_project, "graph")?, "project")?; - let project_dir: Bound<'_, PyAny> = native_field(refactor_project, "project_dir")?; - let mut expansions: HashMap> = HashMap::new(); - let items: Vec> = native_field(&project, "sql_expansions")? - .call_method0("items")? - .try_iter()? - .collect::>()?; - for item in items.iter().rev() { - let (path, expansion): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; - expansions.insert(path.call_method0("resolve")?.str()?.extract()?, expansion); - } - let mut facts: Vec = Vec::new(); - for compiled in native_field(&project, "models")?.try_iter()? { - let compiled: Bound<'_, PyAny> = compiled?; - let resolved: String = project_dir - .div(native_field(&compiled, "relative_path")?)? - .call_method0("resolve")? - .str()? - .extract()?; - facts.push(model(&compiled, expansions.get(&resolved))?); - } - Ok(Some(facts)) -} - fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { blocks .try_iter()? @@ -205,7 +175,7 @@ struct FileGroup { blocks: bool, } -fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { let groups: [FileGroup; 6] = [ FileGroup { role: SqlFileRole::Test, @@ -270,7 +240,7 @@ fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> Ok(files) } -fn python_locations( +pub(crate) fn python_locations( project: &Bound<'_, PyAny>, request: &RefactorRequest, ) -> PyResult> { @@ -308,7 +278,7 @@ fn python_locations( } /// Whether `request` moves a model without renaming it: a folder, or a file of the same name. -fn move_keeps_name(request: &RefactorRequest) -> bool { +pub(crate) fn move_keeps_name(request: &RefactorRequest) -> bool { if request.operation != RefactorOperation::MoveModel { return false; } @@ -319,44 +289,8 @@ fn move_keeps_name(request: &RefactorRequest) -> bool { } } -/// The facts one refactoring plan of `requests` reads. -pub(crate) fn refactor_facts( - project: &Bound<'_, PyAny>, - requests: &[RefactorRequest], -) -> PyResult { - let mut locations: Vec = Vec::new(); - for request in requests { - if !move_keeps_name(request) { - locations.extend(python_locations(project, request)?); - } - } - let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; - let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; - let dialect: Option = native_field(&compiled, "sql_analysis_dialect")?.extract()?; - let mut yaml_files: Vec = - discovered_files(&native_field(&discovered, "source_files")?)?; - yaml_files.extend(discovered_files(&native_field( - &discovered, - "schema_files", - )?)?); - Ok(RefactorFacts { - project_dir: native_field(project, "project_dir")?.str()?.extract()?, - dialect: dialect - .filter(|dialect| !dialect.is_empty()) - .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), - models: model_facts(project)?.unwrap_or_default(), - sources: resource_columns(&native_field(&compiled, "sources")?, "source_entry")?, - seeds: resource_columns(&native_field(&compiled, "seeds")?, "schema_entry")?, - model_files: discovered_files(&native_field(&discovered, "model_files")?)?, - authored_files: authored_files(&discovered)?, - yaml_files, - unit_tests: unit_tests(&compiled)?, - python_locations: locations, - }) -} - /// Every compiled SQL unit test with its file, header name and target models. -fn unit_tests(compiled: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn unit_tests(compiled: &Bound<'_, PyAny>) -> PyResult> { let mut tests: Vec = Vec::new(); for test in native_field(compiled, "sql_tests")?.try_iter()? { let test: Bound<'_, PyAny> = test?; @@ -376,20 +310,6 @@ fn identity_key(identity: &Bound<'_, PyAny>) -> PyResult { identity.repr()?.extract() } -/// Scoped declarations the test file at `path` uses that resolve differently from `destination`. -pub(crate) fn unit_test_rebindings( - project: &Bound<'_, PyAny>, - paths: (&str, &str), -) -> PyResult>> { - let py: Python<'_> = project.py(); - let index: Bound<'_, PyAny> = native_field( - &native_field(&native_field(project, "graph")?, "project")?, - "scope_index", - )?; - let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); - scope_rebound_usages(py, &lookup, paths) -} - /// Where scope placement puts declarations once every `(model, destination)` moves together. pub(crate) fn declaration_placement( project: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs index ef10768f9f..7a56513c46 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs @@ -1,10 +1,8 @@ //! Selected declarations compared with one bulk read of physical columns. use crate::refactor::_helpers::contract_compare; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyTuple}; /// Selected declarations compared with one bulk read of physical columns. #[inline] diff --git a/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs index 14ff083598..806dd00a21 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs @@ -1,9 +1,7 @@ //! Where scope placement puts declarations once every `(model, destination)` moves together. use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_refactor::refactoring::models::DeclarationPlacement; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs index c924eea433..faeec2ffd9 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs @@ -1,18 +1,58 @@ //! The model's MODEL header with physical columns adopted, or the ownership conflict. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::refactor::_helpers::contract_edits::EditRow; +use crate::refactor::_helpers::contract_edits::edit_outcome; +use crate::refactor::_helpers::contract_edits::edit_row; +use crate::refactor::_helpers::contract_edits::physical_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; +use sqlbuild_refactor::refactoring::models::{ContractModelColumn, ContractModelRequest}; /// The model's MODEL header with physical columns adopted, or the ownership conflict. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, physical_columns, overwrite))] pub fn edit_contract_model( py: Python<'_>, model: &Bound<'_, PyAny>, physical_columns: &Bound<'_, PyAny>, overwrite: bool, ) -> PyResult<(Option, Option)> { - contract_edits::edit_contract_model(py, model, physical_columns, overwrite) + let schema: Bound<'_, PyAny> = native_field(model, "schema_entry")?; + let mut columns: Vec = Vec::new(); + let mut dynamic_patterns: Vec> = Vec::new(); + let shared_schema: bool = + !schema.is_none() && !native_field(&schema, "model_schema")?.is_none(); + if !schema.is_none() { + for column in native_field(&schema, "columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let location: Bound<'_, PyAny> = native_field(&column, "location")?; + let location_path: Option = if location.is_none() { + None + } else { + Some(native_field(&location, "path")?.str()?.extract()?) + }; + columns.push(ContractModelColumn { + name: native_field(&column, "name")?.extract()?, + declared_type: native_field(&column, "type")?.extract()?, + location_path, + }); + } + for family in native_field(&schema, "dynamic_columns")?.try_iter()? { + dynamic_patterns.push(native_field(&family?, "name_pattern")?.extract()?); + } + } + let request = ContractModelRequest { + contents: native_field(model, "authored_sql")?.extract()?, + relative_path: native_field(model, "relative_path")?.str()?.extract()?, + shared_schema, + columns, + dynamic_patterns, + physical_columns: physical_rows(physical_columns)?, + overwrite, + }; + let row: EditRow = compiler_guard(|| edit_row(edit_model_contract(&request)))?; + edit_outcome(py, row) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs index ca1b85b4d6..13755fd84e 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs @@ -1,17 +1,43 @@ //! The source's YAML declaration with physical columns adopted, or the ownership conflict. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::contract_edits::EditRow; +use crate::refactor::_helpers::contract_edits::edit_outcome; +use crate::refactor::_helpers::contract_edits::edit_row; +use crate::refactor::_helpers::contract_edits::physical_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; +use sqlbuild_refactor::refactoring::models::ContractSourceRequest; /// The source's YAML declaration with physical columns adopted, or the ownership conflict. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, source, physical_columns, overwrite, contents = None))] pub fn edit_contract_source( source: &Bound<'_, PyAny>, physical_columns: &Bound<'_, PyAny>, overwrite: bool, contents: Option, ) -> PyResult<(Option, Option)> { - contract_edits::edit_contract_source(source, physical_columns, overwrite, contents) + let discovered_contents: String = + native_field(&native_field(source, "source_file")?, "contents")?.extract()?; + let mut columns: Vec<(String, Option)> = Vec::new(); + for column in native_field(&native_field(source, "source_entry")?, "columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + columns.push(( + native_field(&column, "name")?.extract()?, + native_field(&column, "type")?.extract()?, + )); + } + let request = ContractSourceRequest { + contents: contents.unwrap_or_else(|| discovered_contents.clone()), + discovered_contents, + name: native_field(source, "name")?.extract()?, + columns, + physical_columns: physical_rows(physical_columns)?, + overwrite, + }; + let row: EditRow = compiler_guard(|| edit_row(Ok(edit_source_contract(&request))))?; + edit_outcome(source.py(), row) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs index 315bfc6065..1e48d9f70f 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs @@ -1,13 +1,37 @@ //! Every compiled model's refactoring facts, or none without a project. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::refactor_facts::model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_refactor::refactoring::models::ModelFacts; +use std::collections::HashMap; /// Every compiled model's refactoring facts, or none without a project. -#[inline] pub fn model_facts(refactor_project: &Bound<'_, PyAny>) -> PyResult>> { - refactor_facts::model_facts(refactor_project) + if refactor_project.is_none() { + return Ok(None); + } + let project: Bound<'_, PyAny> = + native_field(&native_field(refactor_project, "graph")?, "project")?; + let project_dir: Bound<'_, PyAny> = native_field(refactor_project, "project_dir")?; + let mut expansions: HashMap> = HashMap::new(); + let items: Vec> = native_field(&project, "sql_expansions")? + .call_method0("items")? + .try_iter()? + .collect::>()?; + for item in items.iter().rev() { + let (path, expansion): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; + expansions.insert(path.call_method0("resolve")?.str()?.extract()?, expansion); + } + let mut facts: Vec = Vec::new(); + for compiled in native_field(&project, "models")?.try_iter()? { + let compiled: Bound<'_, PyAny> = compiled?; + let resolved: String = project_dir + .div(native_field(&compiled, "relative_path")?)? + .call_method0("resolve")? + .str()? + .extract()?; + facts.push(model(&compiled, expansions.get(&resolved))?); + } + Ok(Some(facts)) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs index dc984f9324..ca930f5dfa 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs @@ -1,17 +1,51 @@ //! The facts one refactoring plan of `requests` reads. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_refactor::refactoring::models::RefactorFacts; -use sqlbuild_refactor::refactoring::models::RefactorRequest; +use crate::refactor::_helpers::refactor_facts::GENERIC_DIALECT; +use crate::refactor::_helpers::refactor_facts::authored_files; +use crate::refactor::_helpers::refactor_facts::discovered_files; +use crate::refactor::_helpers::refactor_facts::move_keeps_name; +use crate::refactor::_helpers::refactor_facts::python_locations; +use crate::refactor::_helpers::refactor_facts::resource_columns; +use crate::refactor::_helpers::refactor_facts::unit_tests; +use crate::refactor::main::model_facts::model_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::models::{ + DiscoveredFile, ManualLocation, RefactorFacts, RefactorRequest, +}; /// The facts one refactoring plan of `requests` reads. -#[inline] pub fn refactor_facts( project: &Bound<'_, PyAny>, requests: &[RefactorRequest], ) -> PyResult { - refactor_facts::refactor_facts(project, requests) + let mut locations: Vec = Vec::new(); + for request in requests { + if !move_keeps_name(request) { + locations.extend(python_locations(project, request)?); + } + } + let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; + let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; + let dialect: Option = native_field(&compiled, "sql_analysis_dialect")?.extract()?; + let mut yaml_files: Vec = + discovered_files(&native_field(&discovered, "source_files")?)?; + yaml_files.extend(discovered_files(&native_field( + &discovered, + "schema_files", + )?)?); + Ok(RefactorFacts { + project_dir: native_field(project, "project_dir")?.str()?.extract()?, + dialect: dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), + models: model_facts(project)?.unwrap_or_default(), + sources: resource_columns(&native_field(&compiled, "sources")?, "source_entry")?, + seeds: resource_columns(&native_field(&compiled, "seeds")?, "schema_entry")?, + model_files: discovered_files(&native_field(&discovered, "model_files")?)?, + authored_files: authored_files(&discovered)?, + yaml_files, + unit_tests: unit_tests(&compiled)?, + python_locations: locations, + }) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs index bdf7573d6a..1f57377114 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs @@ -1,12 +1,13 @@ //! Register the contract compare functions and classes. -use crate::refactor::_helpers::contract_compare; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::contract_compare::compare_contract_evidence; +use crate::refactor::_helpers::contract_compare::compare_contracts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the contract compare functions and classes. -#[inline] pub fn register_contract_compare(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_compare::register(module) + module.add_function(wrap_pyfunction!(compare_contracts, module)?)?; + module.add_function(wrap_pyfunction!(compare_contract_evidence, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs index 0b3046740f..09c58d09da 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs @@ -1,12 +1,15 @@ //! Register the contract edits functions and classes. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::main::edit_contract_model::edit_contract_model; +use crate::refactor::main::edit_contract_source::edit_contract_source; +use crate::refactor::main::remaining_contract_findings::remaining_contract_findings; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the contract edits functions and classes. -#[inline] pub fn register_contract_edits(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_edits::register(module) + module.add_function(wrap_pyfunction!(edit_contract_model, module)?)?; + module.add_function(wrap_pyfunction!(edit_contract_source, module)?)?; + module.add_function(wrap_pyfunction!(remaining_contract_findings, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs index e5f93791ba..c06586d870 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs @@ -1,12 +1,11 @@ //! Register the python strings functions and classes. -use crate::refactor::_helpers::python_strings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::python_strings::find_python_string_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python strings functions and classes. -#[inline] pub fn register_python_strings(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_strings::register(module) + module.add_function(wrap_pyfunction!(find_python_string_locations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs index f8ef7ff7d7..40482ea74f 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs @@ -1,19 +1,54 @@ //! The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyTuple; +use crate::refactor::_helpers::contract_edits::RESOLVED_KINDS; +use crate::refactor::_helpers::contract_edits::adoption_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, findings, overwrite, conflict))] pub fn remaining_contract_findings<'py>( py: Python<'py>, findings: &Bound<'py, PyAny>, overwrite: bool, conflict: Option, ) -> PyResult> { - contract_edits::remaining_contract_findings(py, findings, overwrite, conflict) + let findings: Vec> = findings.try_iter()?.collect::>()?; + if let Some(conflict) = conflict { + let Some(first) = findings.first() else { + return Err(adoption_error( + py, + "contract ownership conflict requires resource evidence".to_owned(), + )); + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("resource_type", native_field(first, "resource_type")?)?; + keywords.set_item("resource_name", native_field(first, "resource_name")?)?; + keywords.set_item( + "kind", + enum_member(py, RecordId::ContractFindingKind, "OWNERSHIP_CONFLICT")?, + )?; + keywords.set_item("message", conflict)?; + let ownership: Bound<'py, PyAny> = + new_record(py, RecordId::ContractFinding, (), Some(&keywords))?; + let mut remaining: Vec> = findings.clone(); + remaining.push(ownership); + return PyTuple::new(py, remaining); + } + let mut remaining: Vec> = Vec::new(); + for finding in findings { + let kind: String = native_field(&finding, "kind")?.str()?.extract()?; + let resolved: bool = RESOLVED_KINDS.0.contains(&kind.as_str()) + || (overwrite && RESOLVED_KINDS.1.contains(&kind.as_str())); + if !resolved { + remaining.push(finding); + } + } + PyTuple::new(py, remaining) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs index 9b31ee349d..72624eeb1a 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs @@ -1,15 +1,20 @@ //! Scoped declarations the test file at `path` uses that resolve differently from `destination`. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use crate::scopes::main::reports::scope_rebound_usages::scope_rebound_usages; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Scoped declarations the test file at `path` uses that resolve differently from `destination`. -#[inline] pub fn unit_test_rebindings( project: &Bound<'_, PyAny>, paths: (&str, &str), ) -> PyResult>> { - refactor_facts::unit_test_rebindings(project, paths) + let py: Python<'_> = project.py(); + let index: Bound<'_, PyAny> = native_field( + &native_field(&native_field(project, "graph")?, "project")?, + "scope_index", + )?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); + scope_rebound_usages(py, &lookup, paths) } diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs index b416e2e2aa..e28096d38e 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs @@ -2,11 +2,9 @@ use pyo3::PyErr; use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_render::macro_arguments::errors::ArgumentError; use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments as parse; use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; @@ -62,7 +60,7 @@ impl ArgumentHost for UnicodeHost<'_> { /// `("ok", positional, keywords, typed references)` or `("error", detail, help, line, column)`. #[pyfunction] -fn parse_macro_arguments( +pub(crate) fn parse_macro_arguments( py: Python<'_>, text: &str, nested: Vec<(usize, usize)>, @@ -146,8 +144,3 @@ fn value_row<'py>(py: Python<'py>, value: &ArgumentValue) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_macro_arguments, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs index 6d5b316bd9..ce05b3c248 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs @@ -1,10 +1,7 @@ //! Python rows of macro call sites and splice spans. -use crate::render::models::SpanTuple; -use crate::render::types::SpanRow; -use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_python_base::positional_record; -use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; +use sqlbuild_render::macro_calls::models::MacroCallSite; positional_record! { into; @@ -19,26 +16,6 @@ positional_record! { } } -/// A span as the tuple Python reads. -pub(crate) fn span_tuple(span: &SpanRow) -> SpanTuple { - SpanTuple { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} - -/// A span as the expansion semantic validation maps offsets through. -pub(crate) fn span_expansion(span: SpanRow) -> Expansion { - Expansion { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} - pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { SiteRow { start: site.start, @@ -49,12 +26,3 @@ pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { typed_reference_text: site.typed_reference_text, } } - -pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { - SpanRow { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs index 3a574a0d63..584ff42afd 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs @@ -1,28 +1,22 @@ //! Macro call-site scanning and splicing for the Python macro bridge, with the GIL released. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::calls::top_level_macro_call_names::top_level_macro_call_names; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroScanFailure}; -use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; -use crate::render::_helpers::calls::macro_call_rows::SiteRow; -use crate::render::_helpers::calls::macro_call_rows::site_row; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; -use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::calls::macro_call_rows::{SiteRow, site_row}; use crate::render::models::SpanTuple; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; +pub(crate) const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; /// `(call start or None, call name or None, message)` of the error Python's scan raises. type FailureRow = (Option, Option, &'static str); @@ -48,33 +42,12 @@ pub(crate) fn scan_macro_call_sites( )) } -/// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -pub(crate) fn top_level_macro_call_names<'py>( +/// Unique authored macro call names in encounter order, declaration calls excluded. +#[pyfunction] +pub(crate) fn find_macro_call_names<'py>( py: Python<'py>, sql: &str, - syntax: Option<&LexicalSyntax>, ) -> PyResult> { - let names = py - .compiler_detach(|| Ok(scan_names(sql, false, syntax))) - .map_err(compiler_error)?; - match names { - Ok(names) => PyTuple::new( - py, - names - .into_iter() - .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) - .collect::>(), - ), - Err(error) => Err(raised( - &error_class(py, ErrorId::CompileInput)?, - error.message(), - )), - } -} - -/// Unique authored macro call names in encounter order, declaration calls excluded. -#[pyfunction] -fn find_macro_call_names<'py>(py: Python<'py>, sql: &str) -> PyResult> { top_level_macro_call_names(py, sql, None) } @@ -99,11 +72,3 @@ pub(crate) fn splice_macro_calls( .collect(), )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; - module.add_function(wrap_pyfunction!(find_macro_call_names, module)?)?; - module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs index 386a946494..46cb26fb9f 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs @@ -1,35 +1,31 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::declarations::expand_with_inputs::expand_with_inputs; +use crate::render::main::declarations::span_rows::span_rows; use crate::render::models::AuthoredExpansion; use crate::render::models::SqlExpansionInputs; -use pyo3::prelude::{ - Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::model_config::main::config::record_reads::record_reads; use crate::model_config::main::model_nodes::unicode_environment::unicode_environment; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; -use crate::render::_helpers::declarations::declaration_expansion::span_rows; use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; use crate::render::_helpers::expansion::macro_expansion_result::macro_output; use crate::render::models::ConstantRendering; -use crate::render::models::DeclarationExpansion; use crate::render::models::DeclarationExpansionClasses; -use crate::render::models::ExpandedDeclarations; -use crate::render::models::ExpansionConsumer; use crate::render::models::ExpansionRequest; use crate::render::models::SqlMacroSource; +use crate::render::models::{DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer}; use crate::render::types::SpanRow; use crate::sqltext::main::declaration_scan::declaration_scan; use crate::sqltext::main::interpolation_batch::interpolation_batch; use crate::sqltext::main::optional_record_syntax::optional_record_syntax; -use crate::sqltext::models::InterpolationRow; -use crate::sqltext::models::InterpolationSources; +use crate::sqltext::models::{InterpolationRow, InterpolationSources}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -185,20 +181,20 @@ fn declarations_expanded<'py>( } /// What one authored SQL expansion reads. -struct AuthoredSources<'a, 'py> { - sql: &'a Bound<'py, PyAny>, - file_path: &'a Bound<'py, PyAny>, - effective_vars: &'a Bound<'py, PyAny>, - context_values: &'a Bound<'py, PyAny>, - declarations: &'a Bound<'py, PyAny>, - value_renderer: &'a Bound<'py, PyAny>, - collection_rendering: &'a Bound<'py, PyAny>, - loaded_macros: &'a Bound<'py, PyAny>, - macro_context: &'a Bound<'py, PyAny>, - declaration_resolver: Option>, +pub(crate) struct AuthoredSources<'a, 'py> { + pub(crate) sql: &'a Bound<'py, PyAny>, + pub(crate) file_path: &'a Bound<'py, PyAny>, + pub(crate) effective_vars: &'a Bound<'py, PyAny>, + pub(crate) context_values: &'a Bound<'py, PyAny>, + pub(crate) declarations: &'a Bound<'py, PyAny>, + pub(crate) value_renderer: &'a Bound<'py, PyAny>, + pub(crate) collection_rendering: &'a Bound<'py, PyAny>, + pub(crate) loaded_macros: &'a Bound<'py, PyAny>, + pub(crate) macro_context: &'a Bound<'py, PyAny>, + pub(crate) declaration_resolver: Option>, } -fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { +pub(crate) fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { let py: Python<'_> = sources.sql.py(); let syntax: Option = context_syntax(sources.macro_context)?; let (interpolated, interpolation_spans) = substituted( @@ -283,39 +279,17 @@ fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult, -) -> PyResult { - let py: Python<'_> = file_path.py(); - let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); - authored_expansion(AuthoredSources { - sql: &PyString::new(py, sql).into_any(), - file_path, - effective_vars: inputs.effective_vars.bind(py), - context_values: &py.None().into_bound(py), - declarations: &input_declarations(inputs, file_path)?, - value_renderer: inputs.value_renderer.bind(py), - collection_rendering: inputs.collection_rendering.bind(py), - loaded_macros: inputs.loaded_macros.bind(py), - macro_context: inputs.macro_context.bind(py), - declaration_resolver: (!resolver.is_none()).then_some(resolver), - }) -} - /// Expand authored SQL, returning each pass's substitution spans in order. #[pyfunction] #[pyo3(signature = (**request))] -fn expand_authored_sql_with_spans<'py>( +pub(crate) fn expand_authored_sql_with_spans<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) } /// The declarations visible to `file_path`: the resolver's, else the inputs', plus model-local. -fn input_declarations<'py>( +pub(crate) fn input_declarations<'py>( inputs: &SqlExpansionInputs, file_path: &Bound<'py, PyAny>, ) -> PyResult> { @@ -363,7 +337,7 @@ fn input_declarations<'py>( /// Expand authored SQL with a project's expansion context and report each pass's spans. #[pyfunction] #[pyo3(signature = (*, sql, file_path, context))] -fn expand_sql_with_spans<'py>( +pub(crate) fn expand_sql_with_spans<'py>( sql: &str, file_path: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, @@ -378,7 +352,7 @@ fn expand_sql_with_spans<'py>( /// Replace SQL interpolation tokens, returning the span of every substitution. #[pyfunction] #[pyo3(signature = (*, sql, file_path, effective_vars, context_values=None))] -fn substitute_sql_vars_with_spans<'py>( +pub(crate) fn substitute_sql_vars_with_spans<'py>( sql: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, @@ -391,10 +365,3 @@ fn substitute_sql_vars_with_spans<'py>( substituted(sql, file_path, (effective_vars, &context_values), None)?; Ok((substituted_sql, span_objects(py, &spans)?)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; - module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; - module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs index a641972038..aab5b52137 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs @@ -3,7 +3,6 @@ use crate::render::models::DeclarationExpansion; use crate::render::models::DeclarationExpansionClasses; use crate::render::models::ExpandedDeclarations; -use crate::render::types::SpanRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; use sqlbuild_analysis::compiled_project::errors::DeclarationError; @@ -15,9 +14,9 @@ use sqlbuild_analysis::compiled_project::models::{ }; use sqlbuild_analysis::sql_macros::models::{ExpandedText, TypedCall}; -use crate::render::_helpers::sql_macros::sql_macro_host::expanded_sql_macro_calls; -use crate::render::_helpers::sql_macros::sql_macro_host::is_sql_macro_reference; -use crate::render::_helpers::sql_macros::sql_macro_host::reject_bare_macro_calls; +use crate::render::_helpers::sql_macros::sql_macro_host::{ + expanded_sql_macro_calls, is_sql_macro_reference, reject_bare_macro_calls, +}; use crate::sqltext::models::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; @@ -329,16 +328,3 @@ pub(crate) fn compile_error( Err(failure) => failure, } } - -pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { - spliced - .spans - .iter() - .map(|span| SpanRow { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - }) - .collect() -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs index 07a9f4a102..5c186d30cb 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs @@ -3,26 +3,23 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::HashMap; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; /// `CompileInputError(message)`. -fn compile_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::CompileInput, &message) } -fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { +pub(crate) fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { Ok(native_field(owner, name)?.str()?.to_string()) } /// Add each declaration of `files` to `index`, rejecting a name declared twice. -fn index_declarations<'py>( +pub(crate) fn index_declarations<'py>( files: &Bound<'py, PyAny>, kind: &str, index: &Bound<'py, PyDict>, @@ -49,36 +46,23 @@ fn index_declarations<'py>( Ok(()) } -/// Every public enum and constant by name, collision-checked. -#[pyfunction] -pub(crate) fn public_declaration_indexes<'py>( - inputs: Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - let py: Python<'py> = inputs.py(); - let enums: Bound<'py, PyDict> = PyDict::new(py); - let constants: Bound<'py, PyDict> = PyDict::new(py); - index_declarations(&native_field(&inputs, "enum_files")?, "enum", &enums)?; - index_declarations( - &native_field(&inputs, "constant_files")?, - "constant", - &constants, - )?; - Ok((enums, constants)) -} - /// Resolving `extends` chains across the authored public model schemas. -struct SchemaResolution<'py> { - py: Python<'py>, - authored: Vec<(String, Bound<'py, PyAny>)>, - by_name: HashMap, - resolved: HashMap>, +pub(crate) struct SchemaResolution<'py> { + pub(crate) py: Python<'py>, + pub(crate) authored: Vec<(String, Bound<'py, PyAny>)>, + pub(crate) by_name: HashMap, + pub(crate) resolved: HashMap>, /// Names in the order Python's `resolved` dict gains them. - order: Vec, + pub(crate) order: Vec, } impl<'py> SchemaResolution<'py> { /// Python's `_resolve_model_schema_declaration`. - fn resolve(&mut self, name: &str, resolving: &[String]) -> PyResult> { + pub(crate) fn resolve( + &mut self, + name: &str, + resolving: &[String], + ) -> PyResult> { if let Some(existing) = self.resolved.get(name) { return Ok(existing.clone()); } @@ -165,52 +149,3 @@ fn lowered_name(column: &Bound<'_, PyAny>) -> PyResult { .call_method0("lower")? .extract() } - -/// Every public model schema by name, with its inherited columns first, collision-checked. -#[pyfunction] -pub(crate) fn public_model_schema_index<'py>( - inputs: Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - let mut by_name: HashMap = HashMap::new(); - for file in native_field(&inputs, "model_schema_files")?.try_iter()? { - for declaration in native_field(&file?, "declarations")?.try_iter()? { - let declaration: Bound<'py, PyAny> = declaration?; - let name: String = native_field(&declaration, "name")?.extract()?; - if let Some(index) = by_name.get(&name) { - return Err(compile_input_error( - py, - format!( - "Duplicate public schema '{name}' in {} and {}", - text(&authored[*index].1, "relative_path")?, - text(&declaration, "relative_path")? - ), - )); - } - by_name.insert(name.clone(), authored.len()); - authored.push((name, declaration)); - } - } - let names: Vec = authored.iter().map(|(name, _)| name.clone()).collect(); - let mut resolution: SchemaResolution<'py> = SchemaResolution { - py, - authored, - by_name, - resolved: HashMap::new(), - order: Vec::new(), - }; - let index: Bound<'py, PyDict> = PyDict::new(py); - for name in &names { - resolution.resolve(name, &[])?; - } - for name in &resolution.order { - index.set_item(name, &resolution.resolved[name])?; - } - Ok(index) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(public_declaration_indexes, module)?)?; - module.add_function(wrap_pyfunction!(public_model_schema_index, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs index 73f68d7474..c66351e5bc 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs @@ -1,21 +1,14 @@ //! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_host::entries::models::ExecutorHost; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; -use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; use crate::scopes::models::NativeDeclarationContexts; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -29,7 +22,7 @@ use sqlbuild_python_base::records::types::RecordId; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; -const TYPES_MODULE: &str = "types"; +pub(crate) const TYPES_MODULE: &str = "types"; const SCALAR_KINDS: [&str; 6] = ["string", "integer", "boolean", "float", "decimal", "null"]; const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; const ARRAY_RENDERING: &str = "array"; @@ -51,7 +44,7 @@ fn declaration_identity<'py>( } /// Each project declaration's identity mapped to its process-local value. -fn runtime_declarations<'py>( +pub(crate) fn runtime_declarations<'py>( discovered: &Bound<'py, PyAny>, loaded_macros: &Bound<'py, PyAny>, ) -> PyResult> { @@ -119,7 +112,9 @@ fn runtime_declarations<'py>( } /// Resources whose declarations differ from their directory's: private owners and grantees. -fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn resource_specific<'py>( + scope_index: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = scope_index.py(); let private: Bound<'py, PyAny> = enum_member(py, RecordId::ScopeKind, "PRIVATE")?; let mut resources: Vec> = Vec::new(); @@ -137,45 +132,6 @@ fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult( - discovered: &Bound<'py, PyAny>, - scope_index: &Bound<'py, PyAny>, - (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = discovered.py(); - let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; - let lookup: Bound<'py, PyAny> = if lookup.is_none() { - build_scope_lookup(py, scope_index)?.into_bound(py) - } else { - lookup.clone() - }; - let projection: Bound<'py, PyAny> = new_record_with( - py, - RecordId::DeclarationRuntimeProjection, - &[( - "declarations", - imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, - )], - )?; - let native_contexts: Bound<'py, PyAny> = - native_declaration_contexts(&lookup, declarations.as_any())?; - new_record_with( - py, - RecordId::DeclarationScopeResolver, - &[ - ("project_dir", native_field(discovered, "project_dir")?), - ("lookup", lookup), - ("projection", projection), - ( - "resource_specific", - resource_specific(scope_index)?.into_any(), - ), - ("native_contexts", native_contexts), - ], - ) -} - /// The native code of declaration kind `kind`, or the no-kind code. fn kind_code(kinds: &[Bound<'_, PyAny>], kind: &Bound<'_, PyAny>) -> u8 { for (code, known) in (0_u8..).zip(kinds.iter()) { @@ -331,7 +287,7 @@ pub(crate) fn native_declaration_contexts<'py>( /// Hand one scope lookup's positions and runtime values to the native classifier. #[pyfunction] #[pyo3(signature = (*, lookup, declarations))] -fn build_native_declaration_contexts<'py>( +pub(crate) fn build_native_declaration_contexts<'py>( lookup: &Bound<'py, PyAny>, declarations: &Bound<'py, PyAny>, ) -> PyResult> { @@ -353,7 +309,7 @@ pub(crate) fn resolved_declaration_context<'py>( } /// The adapter's SQL for one constant under `selected` collection rendering, or its error. -fn rendered_constant<'py>( +pub(crate) fn rendered_constant<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, selected: &Bound<'py, PyAny>, @@ -390,73 +346,13 @@ fn rendered_constant<'py>( Ok(rendered) } -/// Render one validated constant with the active adapter's typed-value contract. -pub(crate) fn constant_declaration_sql<'py>( - declaration: &Bound<'py, PyAny>, - value_renderer: &Bound<'py, PyAny>, - (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), -) -> PyResult { - let py: Python<'py> = declaration.py(); - let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; - let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { - render_as - } else { - collection_rendering.clone() - }; - let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { - Ok(rendered) => return Ok(rendered), - Err(error) => error, - }; - let rendering_error: Bound<'py, PyAny> = host_entry(py, ExecutorHost::SqlValueRenderingError)?; - let validation_error: Bound<'py, PyAny> = native_attr(py, NativeAttr::SqlValueValidationError)?; - let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); - if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { - return Err(error); - } - let relative_path: Bound<'py, PyAny> = native_field(declaration, "relative_path")?; - let location: String = match file_path { - Some(file_path) => file_path.to_owned(), - None => relative_path.str()?.to_string(), - }; - let compile_error: Bound<'py, PyAny> = - error_class(py, ErrorId::CompileInput)?.call1((format!( - "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", - relative_path.str()?, - native_field(declaration, "name")?.str()?, - location, - native_field(value_renderer, "adapter_name")?.str()?, - native_field(&selected, "value")?.str()?, - raised.str()?, - ),))?; - compile_error.setattr("__cause__", &raised)?; - Err(PyErr::from_value(compile_error)) -} - -/// One enum member value as a SQL scalar literal. -#[pyfunction] -#[pyo3(signature = (*, value))] -pub(crate) fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_instance_of::() { - return value.str()?.extract(); - } - let text: String = value.extract()?; - Ok(format!("'{}'", text.replace('\'', "''"))) -} - /// Canonical visibility for one authored path, projected onto runtime declaration values. #[pyfunction] #[pyo3(signature = (*, resolver, file_path, resource=None))] -fn resolve_declaration_context<'py>( +pub(crate) fn resolve_declaration_context<'py>( resolver: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, resource: Option>, ) -> PyResult> { resolved_declaration_context(resolver, file_path, resource.as_ref()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; - module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; - module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs index ce9fa04516..5c73a741c1 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs @@ -15,8 +15,7 @@ use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_s use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; use crate::render::_helpers::calls::macro_arguments::parsed_arguments; -use crate::render::_helpers::expansion::macro_expansion::ExpansionClasses; -use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::expansion::macro_expansion::{ExpansionClasses, raised}; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::models::NativeMacroExpansion; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs index 9c23011592..69cd824d73 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs @@ -6,9 +6,7 @@ use sqlbuild_python_base::positional_record; use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList}; use sqlbuild_render::macro_calls::errors::ScanError; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; @@ -16,11 +14,8 @@ use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls a use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; use crate::render::_helpers::calls::macro_call_classes::CallClass; -use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; -use crate::render::_helpers::calls::macro_call_memo::poisoned; -use crate::render::_helpers::expansion::macro_evaluation::CallText; -use crate::render::_helpers::expansion::macro_evaluation::Evaluation; -use crate::render::_helpers::expansion::macro_evaluation::evaluate_call; +use crate::render::_helpers::calls::macro_call_memo::{MacroCallMemo, poisoned}; +use crate::render::_helpers::expansion::macro_evaluation::{CallText, Evaluation, evaluate_call}; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::_helpers::expansion::macro_generated_references::GeneratedReferenceClasses; use crate::render::models::SpanTuple; @@ -254,11 +249,6 @@ pub(crate) fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { } } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} - /// Whether a macro's result calls any reference function. pub(crate) fn result_has_markers(classes: &ExpansionClasses<'_>, result: &str) -> bool { classes diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs index 83221cb5ba..327101a06a 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs @@ -1,20 +1,19 @@ //! Expand one authored SQL string's macros into a `MacroExpansionResult`. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::expansion::macro_expansion_classes::macro_expansion_classes; use crate::render::models::ExpansionRequest; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use crate::discovery::main::failures::explicit_references_help::explicit_references_help; -use crate::render::_helpers::calls::macro_call_rows::span_row; use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; use crate::render::_helpers::expansion::macro_expansion::expanded; -use crate::render::models::ExpansionOutput; -use crate::render::models::NativeMacroExpansion; -use crate::render::models::StateInput; +use crate::render::models::{ExpansionOutput, NativeMacroExpansion, StateInput}; use crate::render::types::SpanRow; use crate::sqltext::main::extract_references::extract_references; use sqlbuild_python_base::boundary::main::imported::imported; @@ -48,7 +47,7 @@ const BUILTINS_MODULE: &str = "builtins"; const MACRO_TOKEN: &str = "@"; const GENERATED_ALLOWED: &str = "macro-generated references"; -static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); +pub(crate) static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); fn generated_classes(py: Python<'_>) -> PyResult> { let help: Bound<'_, PyAny> = @@ -82,7 +81,7 @@ fn generated_classes(py: Python<'_>) -> PyResult> { ) } -fn built_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_classes(py: Python<'_>) -> PyResult> { let declaration_kind: Bound<'_, PyAny> = record_class(py, RecordId::DeclarationKind)?; let ref_kind: Bound<'_, PyAny> = host_entry(py, UserCodeHost::SqlResourceRefKind)?; let ref_kinds: Bound<'_, PyDict> = PyDict::new(py); @@ -143,13 +142,6 @@ fn built_classes(py: Python<'_>) -> PyResult> { ) } -/// The classes, enums and helpers the native expansion builds facts and errors from, built once. -pub(crate) fn macro_expansion_classes(py: Python<'_>) -> PyResult> { - EXPANSION_CLASSES - .get_or_try_init(py, || built_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - /// `tuple(dict.fromkeys(items))`: the items without repeats, in first-seen order. fn unique<'py>(py: Python<'py>, items: Vec>) -> PyResult> { let seen: Bound<'py, PyDict> = PyDict::new(py); @@ -294,7 +286,3 @@ pub(crate) fn expand_sql_macros<'py>( consumer: optional(arguments.value("consumer")?), }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_sql_macros, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs index 2343604556..c1486d2617 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs @@ -1,7 +1,7 @@ //! Static inventory of project macro modules for the Python macro loader, with the GIL released. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_pysource::macro_modules::main::inventory_macro_modules::inventory_macro_modules as inventory; use sqlbuild_pysource::macro_modules::main::macro_load_plan::macro_load_plan; use sqlbuild_pysource::macro_modules::models::{ @@ -47,7 +47,7 @@ positional_record! { positional_record! { extract; /// `(name, path, contents, dependencies)` of one macro module to load. - struct ModuleSpecRow { + pub(crate) struct ModuleSpecRow { name: String, file_path: String, contents: String, @@ -146,7 +146,9 @@ pub(crate) fn inventory_macro_modules( /// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. #[pyfunction] -fn macro_module_load_plan(modules: Vec) -> (String, Vec, Vec) { +pub(crate) fn macro_module_load_plan( + modules: Vec, +) -> (String, Vec, Vec) { let specs: Vec = modules .into_iter() .map( @@ -170,9 +172,3 @@ fn macro_module_load_plan(modules: Vec) -> (String, Vec, } = macro_load_plan(&specs); (root, packages, order) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; - module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs index 1d2445ca42..ef2be07911 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs @@ -28,10 +28,10 @@ use sqlbuild_analysis::sql_macros::models::{ use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; -use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; -use crate::render::models::DeclarationExpansion; -use crate::render::models::ExpandedDeclarations; +use crate::render::_helpers::declarations::declaration_expansion::{ + expand_declarations, usage_visibility, +}; +use crate::render::models::{DeclarationExpansion, ExpandedDeclarations}; use crate::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs index cd02a4a7ae..3f9f194765 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs @@ -1,7 +1,7 @@ //! SQL macro files parsed and checked when the project's macros load. -use pyo3::prelude::{Bound, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::sql_macros::constants::CYCLE_CODE; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; @@ -37,7 +37,10 @@ fn definition_row(definition: SqlMacroDefinition) -> DefinitionRow { /// Each SQL macro's parameters and calls, or the first file that is not a valid SQL macro. #[pyfunction] -fn inventory_sql_macros(py: Python<'_>, sources: Vec) -> PyResult> { +pub(crate) fn inventory_sql_macros( + py: Python<'_>, + sources: Vec, +) -> PyResult> { let mut definitions: Vec = Vec::with_capacity(sources.len()); for (name, path, contents) in &sources { definitions @@ -70,7 +73,3 @@ fn inventory_sql_macros(py: Python<'_>, sources: Vec) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inventory_sql_macros, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs index 5370a6a5a7..582749d28e 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs @@ -1,48 +1,11 @@ //! CTE-list SQL macro calls in the `WITH` list of a macro-mode SQL test. -use sqlbuild_analysis::sql_macros::main::with_list_sql_macro_calls::with_list_sql_macro_calls; -use sqlbuild_analysis::sql_macros::models::SqlMacroCall; - -/// The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -pub(crate) fn without_with_list_calls(sql: &str) -> (String, Vec) { - let calls: Vec = with_list_sql_macro_calls(sql); - if calls.is_empty() { - return (sql.to_owned(), Vec::new()); - } - let mut blanked: Vec<(usize, usize)> = Vec::new(); - for call in &calls { - blanked.push((call.start, call.end)); - let after: usize = call.end + leading_space(&sql[call.end..]); - if sql[after..].starts_with(',') { - blanked.push((after, after + 1)); - } else if ends_list(&sql[after..]) { - let before: &str = sql[..call.start].trim_end(); - if before.ends_with(',') { - blanked.push((before.len() - 1, before.len())); - } - } - } - let text: String = sql - .char_indices() - .map(|(offset, character)| { - let inside: bool = is_blanked(offset, &blanked); - if inside && !character.is_whitespace() { - ' ' - } else { - character - } - }) - .collect(); - let names: Vec = calls.into_iter().map(|call| call.name).collect(); - (text, names) -} - -fn leading_space(text: &str) -> usize { +pub(crate) fn leading_space(text: &str) -> usize { text.len() - text.trim_start().len() } /// Whether the text after a call ends the `WITH` list: nothing, a statement end or `SELECT`. -fn ends_list(text: &str) -> bool { +pub(crate) fn ends_list(text: &str) -> bool { let rest: &str = text.trim_start(); let word: String = rest .chars() @@ -52,7 +15,7 @@ fn ends_list(text: &str) -> bool { } /// Whether byte `offset` falls in one of the blanked spans. -fn is_blanked(offset: usize, blanked: &[(usize, usize)]) -> bool { +pub(crate) fn is_blanked(offset: usize, blanked: &[(usize, usize)]) -> bool { blanked .iter() .any(|(start, end)| *start <= offset && offset < *end) diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs index 1f7be17726..4d8b07ce14 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs @@ -1,12 +1,11 @@ //! Register the macro arguments functions and classes. -use crate::render::_helpers::calls::macro_arguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::calls::macro_arguments::parse_macro_arguments; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro arguments functions and classes. -#[inline] pub fn register_macro_arguments(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_arguments::register(module) + module.add_function(wrap_pyfunction!(parse_macro_arguments, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs index d44a654e62..a5133f1909 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs @@ -1,12 +1,17 @@ //! Register the macro calls functions and classes. -use crate::render::_helpers::calls::macro_calls; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; +use crate::render::_helpers::calls::macro_calls::find_macro_call_names; +use crate::render::_helpers::calls::macro_calls::scan_macro_call_sites; +use crate::render::_helpers::calls::macro_calls::splice_macro_calls; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro calls functions and classes. -#[inline] pub fn register_macro_calls(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_calls::register(module) + module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; + module.add_function(wrap_pyfunction!(find_macro_call_names, module)?)?; + module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs index e30ecf2a63..e95e4f952c 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs @@ -1,11 +1,14 @@ //! A span as the expansion semantic validation maps offsets through. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::types::SpanRow; use sqlbuild_analysis::semantic_validation::models::Expansion; /// A span as the expansion semantic validation maps offsets through. -#[inline] pub fn span_expansion(span: SpanRow) -> Expansion { - macro_call_rows::span_expansion(span) + Expansion { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs index 5a2e19132f..9ae3ef675b 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs @@ -1,11 +1,14 @@ //! Span row. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::types::SpanRow; use sqlbuild_render::macro_calls::models::MacroSpliceSpan; /// Span row. -#[inline] pub fn span_row(span: MacroSpliceSpan) -> SpanRow { - macro_call_rows::span_row(span) + SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs index d1c2173566..c5396fb6c5 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs @@ -1,11 +1,14 @@ //! A span as the tuple Python reads. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::models::SpanTuple; use crate::render::types::SpanRow; /// A span as the tuple Python reads. -#[inline] pub fn span_tuple(span: &SpanRow) -> SpanTuple { - macro_call_rows::span_tuple(span) + SpanTuple { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs index f740688342..302e7185de 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs @@ -1,18 +1,36 @@ //! Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -use crate::render::_helpers::calls::macro_calls; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::render::_helpers::calls::macro_calls::DECLARATION_CALLS; +use crate::render::_helpers::expansion::macro_expansion::raised; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -#[inline] pub fn top_level_macro_call_names<'py>( py: Python<'py>, sql: &str, syntax: Option<&LexicalSyntax>, ) -> PyResult> { - macro_calls::top_level_macro_call_names(py, sql, syntax) + let names = py + .compiler_detach(|| Ok(scan_names(sql, false, syntax))) + .map_err(compiler_error)?; + match names { + Ok(names) => PyTuple::new( + py, + names + .into_iter() + .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) + .collect::>(), + ), + Err(error) => Err(raised( + &error_class(py, ErrorId::CompileInput)?, + error.message(), + )), + } } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs index 98c8ffe2cd..dd4556b8eb 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs @@ -1,20 +1,53 @@ //! Render one validated constant with the active adapter's typed-value contract. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::rendered_constant; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::NativeAttr; /// Render one validated constant with the active adapter's typed-value contract. -#[inline] pub fn constant_declaration_sql<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), ) -> PyResult { - declaration_values::constant_declaration_sql( - declaration, - value_renderer, - (collection_rendering, file_path), - ) + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; + let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { + render_as + } else { + collection_rendering.clone() + }; + let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { + Ok(rendered) => return Ok(rendered), + Err(error) => error, + }; + let rendering_error: Bound<'py, PyAny> = host_entry(py, ExecutorHost::SqlValueRenderingError)?; + let validation_error: Bound<'py, PyAny> = native_attr(py, NativeAttr::SqlValueValidationError)?; + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { + return Err(error); + } + let relative_path: Bound<'py, PyAny> = native_field(declaration, "relative_path")?; + let location: String = match file_path { + Some(file_path) => file_path.to_owned(), + None => relative_path.str()?.to_string(), + }; + let compile_error: Bound<'py, PyAny> = + error_class(py, ErrorId::CompileInput)?.call1((format!( + "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", + relative_path.str()?, + native_field(declaration, "name")?.str()?, + location, + native_field(value_renderer, "adapter_name")?.str()?, + native_field(&selected, "value")?.str()?, + raised.str()?, + ),))?; + compile_error.setattr("__cause__", &raised)?; + Err(PyErr::from_value(compile_error)) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs index 0f8d8e6cbd..93578455e7 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs @@ -1,16 +1,52 @@ //! Pair the serializable static index with original process-local declaration values. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::TYPES_MODULE; +use crate::render::_helpers::declarations::declaration_values::native_declaration_contexts; +use crate::render::_helpers::declarations::declaration_values::resource_specific; +use crate::render::_helpers::declarations::declaration_values::runtime_declarations; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Pair the serializable static index with original process-local declaration values. -#[inline] pub fn declaration_scope_resolver<'py>( discovered: &Bound<'py, PyAny>, scope_index: &Bound<'py, PyAny>, (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - declaration_values::declaration_scope_resolver(discovered, scope_index, (loaded_macros, lookup)) + let py: Python<'py> = discovered.py(); + let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; + let lookup: Bound<'py, PyAny> = if lookup.is_none() { + build_scope_lookup(py, scope_index)?.into_bound(py) + } else { + lookup.clone() + }; + let projection: Bound<'py, PyAny> = new_record_with( + py, + RecordId::DeclarationRuntimeProjection, + &[( + "declarations", + imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, + )], + )?; + let native_contexts: Bound<'py, PyAny> = + native_declaration_contexts(&lookup, declarations.as_any())?; + new_record_with( + py, + RecordId::DeclarationScopeResolver, + &[ + ("project_dir", native_field(discovered, "project_dir")?), + ("lookup", lookup), + ("projection", projection), + ( + "resource_specific", + resource_specific(scope_index)?.into_any(), + ), + ("native_contexts", native_contexts), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs index 4b1bac8d3d..d2cdaade28 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs @@ -1,18 +1,31 @@ //! `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. -use crate::render::_helpers::declarations::authored_sql; +use crate::render::_helpers::declarations::authored_sql::AuthoredSources; +use crate::render::_helpers::declarations::authored_sql::authored_expansion; +use crate::render::_helpers::declarations::authored_sql::input_declarations; use crate::render::models::AuthoredExpansion; use crate::render::models::SqlExpansionInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; /// `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. -#[inline] pub fn expand_with_inputs( inputs: &SqlExpansionInputs, sql: &str, file_path: &Bound<'_, PyAny>, ) -> PyResult { - authored_sql::expand_with_inputs(inputs, sql, file_path) + let py: Python<'_> = file_path.py(); + let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); + authored_expansion(AuthoredSources { + sql: &PyString::new(py, sql).into_any(), + file_path, + effective_vars: inputs.effective_vars.bind(py), + context_values: &py.None().into_bound(py), + declarations: &input_declarations(inputs, file_path)?, + value_renderer: inputs.value_renderer.bind(py), + collection_rendering: inputs.collection_rendering.bind(py), + loaded_macros: inputs.loaded_macros.bind(py), + macro_context: inputs.macro_context.bind(py), + declaration_resolver: (!resolver.is_none()).then_some(resolver), + }) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs index aada555fa0..77813c5753 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs @@ -1,15 +1,24 @@ //! Every public enum and constant by name, collision-checked. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_indexes::index_declarations; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Every public enum and constant by name, collision-checked. -#[inline] +#[pyfunction] pub fn public_declaration_indexes<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - declaration_indexes::public_declaration_indexes(inputs) + let py: Python<'py> = inputs.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + let constants: Bound<'py, PyDict> = PyDict::new(py); + index_declarations(&native_field(&inputs, "enum_files")?, "enum", &enums)?; + index_declarations( + &native_field(&inputs, "constant_files")?, + "constant", + &constants, + )?; + Ok((enums, constants)) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs index 1c1d35991e..19aefe81c6 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs @@ -1,13 +1,52 @@ //! Every public model schema by name, with its inherited columns first, collision-checked. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::render::_helpers::declarations::declaration_indexes::SchemaResolution; +use crate::render::_helpers::declarations::declaration_indexes::compile_input_error; +use crate::render::_helpers::declarations::declaration_indexes::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; /// Every public model schema by name, with its inherited columns first, collision-checked. -#[inline] +#[pyfunction] pub fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { - declaration_indexes::public_model_schema_index(inputs) + let py: Python<'py> = inputs.py(); + let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + let mut by_name: HashMap = HashMap::new(); + for file in native_field(&inputs, "model_schema_files")?.try_iter()? { + for declaration in native_field(&file?, "declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let name: String = native_field(&declaration, "name")?.extract()?; + if let Some(index) = by_name.get(&name) { + return Err(compile_input_error( + py, + format!( + "Duplicate public schema '{name}' in {} and {}", + text(&authored[*index].1, "relative_path")?, + text(&declaration, "relative_path")? + ), + )); + } + by_name.insert(name.clone(), authored.len()); + authored.push((name, declaration)); + } + } + let names: Vec = authored.iter().map(|(name, _)| name.clone()).collect(); + let mut resolution: SchemaResolution<'py> = SchemaResolution { + py, + authored, + by_name, + resolved: HashMap::new(), + order: Vec::new(), + }; + let index: Bound<'py, PyDict> = PyDict::new(py); + for name in &names { + resolution.resolve(name, &[])?; + } + for name in &resolution.order { + index.set_item(name, &resolution.resolved[name])?; + } + Ok(index) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs index 1a77c6da6c..a5852cb504 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs @@ -1,12 +1,15 @@ //! Register the authored sql functions and classes. -use crate::render::_helpers::declarations::authored_sql; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::authored_sql::expand_authored_sql_with_spans; +use crate::render::_helpers::declarations::authored_sql::expand_sql_with_spans; +use crate::render::_helpers::declarations::authored_sql::substitute_sql_vars_with_spans; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the authored sql functions and classes. -#[inline] pub fn register_authored_sql(module: &Bound<'_, PyModule>) -> PyResult<()> { - authored_sql::register(module) + module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs index dab0eed265..70395954f7 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs @@ -1,12 +1,12 @@ //! Register the declaration indexes functions and classes. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::main::declarations::public_declaration_indexes::public_declaration_indexes; +use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration indexes functions and classes. -#[inline] pub fn register_declaration_indexes(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_indexes::register(module) + module.add_function(wrap_pyfunction!(public_declaration_indexes, module)?)?; + module.add_function(wrap_pyfunction!(public_model_schema_index, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs index 92d91956ef..feb05ea17a 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs @@ -1,12 +1,15 @@ //! Register the declaration values functions and classes. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::build_native_declaration_contexts; +use crate::render::_helpers::declarations::declaration_values::resolve_declaration_context; +use crate::render::main::declarations::render_enum_member_value::render_enum_member_value; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration values functions and classes. -#[inline] pub fn register_declaration_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_values::register(module) + module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; + module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; + module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs index 18b3148a1e..fd3822fa90 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs @@ -1,12 +1,16 @@ //! One enum member value as a SQL scalar literal. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyInt; /// One enum member value as a SQL scalar literal. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, value))] pub fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { - declaration_values::render_enum_member_value(value) + if value.is_instance_of::() { + return value.str()?.extract(); + } + let text: String = value.extract()?; + Ok(format!("'{}'", text.replace('\'', "''"))) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs index 69f717b456..69bf7246ce 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs @@ -1,11 +1,18 @@ //! Span rows. -use crate::render::_helpers::declarations::declaration_expansion; use crate::render::types::SpanRow; use sqlbuild_analysis::compiled_project::models::SplicedSql; /// Span rows. -#[inline] pub fn span_rows(spliced: &SplicedSql) -> Vec { - declaration_expansion::span_rows(spliced) + spliced + .spans + .iter() + .map(|span| SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs index f3493f3740..1981296de6 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs @@ -3,9 +3,7 @@ use crate::render::_helpers::expansion::macro_expansion; use crate::render::models::ExpansionOutput; use crate::render::models::NativeMacroExpansion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. #[inline] diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs index 3cc313db31..f32602167e 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs @@ -2,9 +2,7 @@ use crate::render::_helpers::expansion::macro_expansion_result; use crate::render::models::ExpansionRequest; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Expansion result. #[inline] diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs index ceebea1865..1aa3e08c4a 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs @@ -1,13 +1,13 @@ //! The classes, enums and helpers the native expansion builds facts and errors from, built once. -use crate::render::_helpers::expansion::macro_expansion_result; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::render::_helpers::expansion::macro_expansion_result::EXPANSION_CLASSES; +use crate::render::_helpers::expansion::macro_expansion_result::built_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The classes, enums and helpers the native expansion builds facts and errors from, built once. -#[inline] pub fn macro_expansion_classes(py: Python<'_>) -> PyResult> { - macro_expansion_result::macro_expansion_classes(py) + EXPANSION_CLASSES + .get_or_try_init(py, || built_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs index 14a84995b5..a3fd9b002c 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs @@ -1,12 +1,10 @@ //! Register the macro expansion functions and classes. -use crate::render::_helpers::expansion::macro_expansion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::models::NativeMacroExpansion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the macro expansion functions and classes. -#[inline] pub fn register_macro_expansion(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_expansion::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs index bd34acc97d..4807d9a877 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs @@ -1,12 +1,10 @@ //! Register the macro expansion result functions and classes. -use crate::render::_helpers::expansion::macro_expansion_result; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::expansion::macro_expansion_result::expand_sql_macros; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro expansion result functions and classes. -#[inline] pub fn register_macro_expansion_result(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_expansion_result::register(module) + module.add_function(wrap_pyfunction!(expand_sql_macros, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs index abe97ff144..39cf08b062 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs @@ -1,12 +1,13 @@ //! Register the macro modules functions and classes. -use crate::render::_helpers::expansion::macro_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::expansion::macro_modules::inventory_macro_modules; +use crate::render::_helpers::expansion::macro_modules::macro_module_load_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro modules functions and classes. -#[inline] pub fn register_macro_modules(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_modules::register(module) + module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; + module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs index 2242424725..46d20a9dbb 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs @@ -1,12 +1,10 @@ //! Register the sql macro inventory functions and classes. -use crate::render::_helpers::sql_macros::sql_macro_inventory; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::sql_macros::sql_macro_inventory::inventory_sql_macros; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql macro inventory functions and classes. -#[inline] pub fn register_sql_macro_inventory(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_macro_inventory::register(module) + module.add_function(wrap_pyfunction!(inventory_sql_macros, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs index 951d255eb0..53b2a7076d 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs @@ -1,9 +1,7 @@ //! `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. use crate::render::_helpers::sql_macros::sql_macro_mocks; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs index 4f4d902195..b2743ce49c 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs @@ -1,9 +1,41 @@ //! The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -use crate::render::_helpers::sql_macros::sql_macro_tests; +use crate::render::_helpers::sql_macros::sql_macro_tests::ends_list; +use crate::render::_helpers::sql_macros::sql_macro_tests::is_blanked; +use crate::render::_helpers::sql_macros::sql_macro_tests::leading_space; +use sqlbuild_analysis::sql_macros::main::with_list_sql_macro_calls::with_list_sql_macro_calls; +use sqlbuild_analysis::sql_macros::models::SqlMacroCall; /// The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -#[inline] pub fn without_with_list_calls(sql: &str) -> (String, Vec) { - sql_macro_tests::without_with_list_calls(sql) + let calls: Vec = with_list_sql_macro_calls(sql); + if calls.is_empty() { + return (sql.to_owned(), Vec::new()); + } + let mut blanked: Vec<(usize, usize)> = Vec::new(); + for call in &calls { + blanked.push((call.start, call.end)); + let after: usize = call.end + leading_space(&sql[call.end..]); + if sql[after..].starts_with(',') { + blanked.push((after, after + 1)); + } else if ends_list(&sql[after..]) { + let before: &str = sql[..call.start].trim_end(); + if before.ends_with(',') { + blanked.push((before.len() - 1, before.len())); + } + } + } + let text: String = sql + .char_indices() + .map(|(offset, character)| { + let inside: bool = is_blanked(offset, &blanked); + if inside && !character.is_whitespace() { + ' ' + } else { + character + } + }) + .collect(); + let names: Vec = calls.into_iter().map(|call| call.name).collect(); + (text, names) } diff --git a/crates/sqlbuild-python-front/src/render/models.rs b/crates/sqlbuild-python-front/src/render/models.rs index a350fc00b4..4fc45f6c6e 100644 --- a/crates/sqlbuild-python-front/src/render/models.rs +++ b/crates/sqlbuild-python-front/src/render/models.rs @@ -4,10 +4,7 @@ use crate::render::_helpers::calls::macro_call_classes::CallClass; use crate::render::_helpers::calls::macro_call_classes::ClassInputs; use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; use crate::render::_helpers::calls::macro_call_memo::poisoned; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; -use crate::render::_helpers::declarations::declaration_values::constant_declaration_sql; use crate::render::_helpers::expansion::macro_evaluation::CallText; use crate::render::_helpers::expansion::macro_evaluation::Evaluation; use crate::render::_helpers::expansion::macro_evaluation::macro_argument_error; @@ -36,6 +33,9 @@ use crate::render::_helpers::expansion::macro_expansion_facts::ExpansionFacts; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::_helpers::expansion::macro_generated_references::GeneratedOrigin; use crate::render::_helpers::expansion::macro_generated_references::report_generated_references; +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::declarations::constant_declaration_sql::constant_declaration_sql; use crate::render::types::MacroScopes; use crate::render::types::SpanRow; use crate::render::types::StateTuple; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs index 6f05103216..b6dc17d6cd 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs @@ -9,12 +9,9 @@ use sqlbuild_scopes::scope_index::models::{ }; use crate::scopes::_helpers::checks::scope_validation::replace; -use crate::scopes::_helpers::index::scope_objects::enum_text; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::read_resource; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::_helpers::index::scope_objects::{ + enum_text, field, optional_item, read_resource, root_object, tuple_attribute, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs index ab84f15f4d..c6fcb98f4c 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs @@ -1,48 +1,35 @@ //! Compile usage merging, placement validation and move relocation over a Python `ScopeIndex`. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::scopes::main::checks::relocated_declarations::relocated_declarations; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::compile_usages::models::UsageKey; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; -use sqlbuild_scopes::placement::main::relocate_for_resource_moves::relocate_for_resource_moves; use sqlbuild_scopes::placement::models::{ - DiagnosticOrderKey, PlacementCode, PlacementDiagnostic, PlacementIndex, Relocation, + DiagnosticOrderKey, PlacementCode, PlacementDiagnostic, PlacementIndex, }; use sqlbuild_scopes::scope_index::models::ResourceIdentity; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::resource_identity; -use crate::scopes::_helpers::index::scope_objects::enum_text; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::identity_row; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::qualified_row; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_placement_index; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; -use crate::scopes::_helpers::index::scope_objects::value_of; -use crate::scopes::_helpers::relationships::compile_usages::MergeRows; -use crate::scopes::_helpers::relationships::compile_usages::SourceRow; -use crate::scopes::_helpers::relationships::compile_usages::merged_usage_rows; -use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use crate::scopes::_helpers::declarations::placement::{QualifiedRow, resource_identity}; +use crate::scopes::_helpers::index::scope_objects::{ + enum_text, field, identity_row, optional_item, qualified_row, read_completeness, + read_placement_index, tuple_attribute, value_of, +}; +use crate::scopes::_helpers::relationships::compile_usages::{ + MergeRows, SourceRow, merged_usage_rows, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; -const RUNTIME_USAGE: usize = 2; +pub(crate) const RUNTIME_USAGE: usize = 2; /// Compile input groups whose items carry declaration usages, after the model inputs. -const COMPILE_USAGE_GROUPS: [&str; 5] = [ +pub(crate) const COMPILE_USAGE_GROUPS: [&str; 5] = [ "source_inputs", "sql_function_inputs", "audit_inputs", @@ -306,138 +293,6 @@ pub(crate) fn scope_relocated_declarations( Ok(relocated.map(|(records, _)| records.unbind())) } -/// Relocated records after every move in `moves`, and per record whether it lacks a needed group. -pub(crate) fn relocated_declarations<'py>( - index: &Bound<'py, PyAny>, - moves: &[(ResourceIdentity, String)], -) -> PyResult, Vec)>> { - let py: Python<'py> = index.py(); - if !read_completeness(index)?[RUNTIME_USAGE] { - return Ok(None); - } - let placement: PlacementIndex = read_placement_index(index)?; - let relocations: Option> = py - .compiler_detach(|| Ok(relocate_for_resource_moves(&placement, moves))) - .map_err(compiler_error)?; - let Some(relocations) = relocations else { - return Ok(None); - }; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let records: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; - let mut moved: Vec> = Vec::with_capacity(relocations.len()); - let mut grouping: Vec = Vec::with_capacity(relocations.len()); - for item in relocations { - let owning_path: Bound<'_, PyAny> = item.owning_path.into_bound_py_any(py)?; - let record: Bound<'py, PyAny> = replace( - &records.get_item(item.declaration)?, - &[ - ("path", item.path.into_bound_py_any(py)?), - ( - "scope", - classes.member(RecordId::ScopeKind, item.scope.as_str())?, - ), - ( - "ownership_root", - root_object(&classes, &item.ownership_root)?, - ), - ("owning_path", owning_path), - ], - )?; - moved.push(record); - grouping.push(item.requires_group); - } - Ok(Some((PyTuple::new(py, moved)?, grouping))) -} - -fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { records.try_iter()?.collect() } - -/// Compile inputs' canonical index with all usage and placement facts. -pub(crate) fn scope_index_with_compile_usages<'py>( - inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; - let resolver: Bound<'py, PyAny> = if scope.is_none() { - scope - } else { - native_field(&scope, "resolver")? - }; - let schema_files: Bound<'py, PyAny> = native_field( - &native_field(inputs, "discovered_inputs")?, - "model_schema_files", - )?; - let mut usages: Vec> = Vec::new(); - usages.extend(usage_records( - model_schema_scope_usages(&schema_files, &resolver)?.as_any(), - )?); - let digests: bool = sqlbuild_python_base::compile_context::main::delta::capturing::capturing(); - for model in native_field(inputs, "model_inputs")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let macro_usages: Vec> = - usage_records(&native_field(&model, "macro_usages")?)?; - let declaration_usages: Vec> = - usage_records(&native_field(&model, "declaration_usages")?)?; - if digests { - sqlbuild_python_base::compile_context::main::image_facts::record_model_usages::record_model_usages( - &model, - (¯o_usages, &declaration_usages), - )?; - } - usages.extend(macro_usages); - usages.extend(declaration_usages); - } - for group in COMPILE_USAGE_GROUPS { - for item in native_field(inputs, group)?.try_iter()? { - usages.extend(usage_records(&native_field(&item?, "declaration_usages")?)?); - } - } - let enforce_placement: bool = native_field( - &native_field(&native_field(inputs, "project_config")?, "scopes")?, - "enforce_placement", - )? - .extract()?; - Ok(scope_placement_validated_index( - &native_field(inputs, "scope_index")?, - enforce_placement, - Some(PyTuple::new(py, usages)?), - )? - .into_bound(py)) -} - -/// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -#[pyfunction] -#[pyo3(signature = (*, index))] -pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { - let py: Python<'_> = index.py(); - let error: Bound<'_, PyAny> = enum_member(py, RecordId::ScopeDiagnosticSeverity, "ERROR")?; - let mut errors: Vec> = Vec::new(); - for item in native_field(index, "diagnostics")?.try_iter()? { - let item: Bound<'_, PyAny> = item?; - if native_field(&item, "severity")?.is(&error) { - errors.push(item); - } - } - if errors.is_empty() { - return Ok(()); - } - let mut message: String = String::from("Invalid declaration scope index:"); - for item in &errors { - let code: String = native_field(&native_field(item, "code")?, "value")?.extract()?; - let text: String = native_field(item, "message")?.extract()?; - message.push_str(&format!("\n[{code}] {text}")); - } - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("diagnostics", PyTuple::new(py, errors)?)?; - Err(PyErr::from_value( - error_class(py, ErrorId::ScopeValidation)?.call((message,), Some(&options))?, - )) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; - module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; - module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs index ba33e6ec00..024b9d0001 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs @@ -1,23 +1,20 @@ //! Static declaration visibility of scope targets and authored paths over a `ScopeLookup`. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, }; -use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; -use crate::scopes::_helpers::index::scope_lookup::ResolvedRows; -use crate::scopes::_helpers::index::scope_lookup::TargetRows; -use crate::scopes::_helpers::index::scope_lookup::scope_path_error; -use crate::scopes::_helpers::index::scope_objects::qualified_row; -use crate::scopes::_helpers::index::scope_objects::read_resource; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::_helpers::index::scope_lookup::{ + NativeScopeLookup, ResolvedRows, TargetRows, scope_path_error, +}; +use crate::scopes::_helpers::index::scope_objects::{ + qualified_row, read_resource, tuple_attribute, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -101,7 +98,7 @@ fn target_query<'py>( /// `query_target`: the `ScopeTargetQuery` of `target`; invalid text raises the scope error. #[pyfunction] #[pyo3(name = "query_scope_target", signature = (*, lookup, target))] -fn scope_query_target( +pub(crate) fn scope_query_target( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -124,7 +121,7 @@ fn resolved( /// Python's `resolve_scope_visibility`: visible facts and inaccessible reasons per target. #[pyfunction] #[pyo3(signature = (*, lookup, target))] -fn resolve_scope_visibility( +pub(crate) fn resolve_scope_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -135,7 +132,7 @@ fn resolve_scope_visibility( /// Python's `resolve_scope_declaration_visibility`: visible facts and inaccessible identities. #[pyfunction] #[pyo3(signature = (*, lookup, target))] -fn resolve_scope_declaration_visibility( +pub(crate) fn resolve_scope_declaration_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -208,7 +205,7 @@ fn resolved_visibility( /// Visible and inaccessible declarations for an authored path, including declaration files. #[pyfunction] #[pyo3(name = "resolve_scope_path_visibility", signature = (*, lookup, path))] -fn scope_resolve_path_visibility( +pub(crate) fn scope_resolve_path_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, path: &Bound<'_, PyAny>, @@ -261,14 +258,3 @@ fn scope_resolve_path_visibility( )?, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; - module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_scope_declaration_visibility, - module - )?)?; - module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs index fafbf520b4..b726c94ae8 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs @@ -1,10 +1,9 @@ //! Declaration resolution contexts built natively from a Python scope lookup's positions. -use crate::scopes::models::NativeDeclarationContexts; use std::collections::HashMap; use pyo3::FromPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::types::PyDict; use sqlbuild_scopes::scope_index::models::VisibleEntry; @@ -59,8 +58,3 @@ pub(crate) struct ContextParts<'py> { pub(crate) inaccessible: [Bound<'py, PyDict>; 3], pub(crate) visibility: [Bound<'py, PyDict>; 3], } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs index 83357b83b3..cae5e29d27 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs @@ -1,31 +1,19 @@ //! The persistent scope index cache: fingerprint, verified read and atomic write. +use crate::scopes::main::index::project_fingerprint::project_fingerprint; use std::path::PathBuf; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; -use sqlbuild_scopes::scope_cache::main::project_fingerprint_inputs::project_fingerprint_inputs; -use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; -use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; -use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; -use sqlbuild_scopes::scope_cache::models::{CachedIndex, Fingerprint, FingerprintInputs}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::pyfunction; -use crate::scopes::_helpers::index::scope_objects::index_object; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_index; -use crate::scopes::models::ScopeClasses; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::types::CompilerDetach; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; const UNKNOWN_VERSION: &str = "0+unknown"; /// `importlib.metadata.version("sqlbuild")`, or `0+unknown` when it is not installed. -fn sqlbuild_version(py: Python<'_>) -> PyResult { +pub(crate) fn sqlbuild_version(py: Python<'_>) -> PyResult { let metadata: Bound<'_, PyModule> = py.import(METADATA_MODULE)?; match native_field(&metadata, "version")?.call1((DISTRIBUTION_NAME,)) { Ok(version) => version.extract(), @@ -36,72 +24,12 @@ fn sqlbuild_version(py: Python<'_>) -> PyResult { } } -/// The scope index fingerprint of `project_dir`, `None` when it cannot be read. -pub(crate) fn project_fingerprint( - py: Python<'_>, - project_dir: PathBuf, -) -> PyResult> { - let inputs: FingerprintInputs = project_fingerprint_inputs(sqlbuild_version(py)?); - let fingerprint: Fingerprint = py - .compiler_detach(|| Ok(scope_fingerprint(&project_dir, &inputs))) - .map_err(compiler_error)?; - match fingerprint { - Fingerprint::Hash(hash) => Ok(Some(hash)), - Fingerprint::Unavailable => Ok(None), - Fingerprint::Unserializable => Err(PyTypeError::new_err( - "a scope-relevant config value is not JSON serializable", - )), - } -} - /// Hash the canonical scope-relevant authored inputs, or `None` when they cannot be read. #[pyfunction] #[pyo3(signature = (*, project_dir))] -fn scope_index_fingerprint(py: Python<'_>, project_dir: PathBuf) -> PyResult> { - project_fingerprint(py, project_dir) -} - -/// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -pub(crate) fn read_scope_index_cache( - py: Python<'_>, - path: PathBuf, - fingerprint: String, -) -> PyResult>> { - let cached = py - .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) - .map_err(compiler_error)?; - let Ok(index) = cached else { - return Ok(None); - }; - match index_object(&ScopeClasses::new(py)?, &index) { - Ok(object) => Ok(Some(object.unbind())), - Err(error) if error.is_instance_of::(py) => Ok(None), - Err(error) => Err(error), - } -} - -/// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -pub(crate) fn write_scope_index_cache( +pub(crate) fn scope_index_fingerprint( py: Python<'_>, - location: (PathBuf, String), - fingerprint: String, - index: &Bound<'_, PyAny>, -) -> PyResult { - if !read_completeness(index)?.iter().all(|done| *done) { - return Ok(false); - } - let index: CachedIndex = read_index(index)?; - let (directory, file_name) = location; - py.compiler_detach(|| { - Ok(match encode_scope_cache(&index, &fingerprint) { - Ok(envelope) => matches!(write_scope_cache(&directory, &file_name, &envelope), Ok(())), - Err(_) => false, - }) - }) - .map_err(compiler_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_index_fingerprint, module)?)?; - Ok(()) + project_dir: PathBuf, +) -> PyResult> { + project_fingerprint(py, project_dir) } diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs index 1716405bb2..c771639a5b 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs @@ -1,41 +1,32 @@ //! The static scope index built natively from discovered inputs, as the Python scope models. +use crate::scopes::main::index::open_native_scope_index::open_native_scope_index; use crate::scopes::models::NativeScopeIndex; use std::collections::HashMap; use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationInput, DeclarationKind, RejectedPath, ResourceIdentity, - ResourceInput, ResourceKind, ResourceRoot, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, + ResourceInput, ResourceKind, ResourceRoot, ScopeIndex, ScopeInputs, ScopeKind, }; -use crate::discovery::models::ModelFileTexts; -use crate::discovery::models::NativeModelFile; +use crate::discovery::models::{ModelFileTexts, NativeModelFile}; use crate::scopes::_helpers::index::scope_lookup::scope_path_error; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::value_of; -use crate::scopes::_helpers::relationships::relationship_scan::ScenarioRow; -use crate::scopes::_helpers::relationships::relationship_scan::TestFileRow; +use crate::scopes::_helpers::index::scope_objects::{field, optional_item, root_object, value_of}; +use crate::scopes::_helpers::relationships::relationship_scan::{ScenarioRow, TestFileRow}; use crate::scopes::models::ScopeClasses; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::types::CompilerDetach; /// Why `UnindexableScopeError` is raised; discovery only yields normalized project paths. const UNINDEXABLE_SCOPE_MESSAGE: &str = "Native scope indexing could not place a discovered path or resource kind. Discovery always yields normalized project-relative paths, so this is a SQLBuild bug; please file an issue with the project layout"; @@ -75,15 +66,15 @@ struct Placement { } /// The builder's walk of discovered inputs: native inputs and the Python values beside them. -struct Walk<'py> { +pub(crate) struct Walk<'py> { classes: ScopeClasses<'py>, - inputs: ScopeInputs, + pub(crate) inputs: ScopeInputs, resource_identities: Vec>, /// Each resource's kind and root, which pick its ownership root. resource_roots: Vec<(ResourceKind, ResourceRoot)>, declarations: Vec>, /// Whether every walked kind, scope and root parsed natively. - indexable: bool, + pub(crate) indexable: bool, } fn text(value: &Bound<'_, PyAny>) -> PyResult { @@ -308,7 +299,7 @@ impl<'py> Walk<'py> { )) } - fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for model_file in items(inputs, "model_files")? { let header: Bound<'py, PyAny> = header_name(&model_file)?; let (model_name, path, folder): (String, String, String) = @@ -390,7 +381,7 @@ impl<'py> Walk<'py> { ) } - fn add_resources(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_resources(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for test_file in items(inputs, "test_files")? { let path: Bound<'py, PyAny> = native_field(&test_file, "relative_path")?; for block in items(&test_file, "blocks")? { @@ -458,7 +449,7 @@ impl<'py> Walk<'py> { }) } - fn add_declaration_files(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_declaration_files(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for enum_file in items(inputs, "enum_files")? { for declaration in items(&enum_file, "declarations")? { let identity: Bound<'py, PyAny> = self.declaration_identity( @@ -512,7 +503,7 @@ impl<'py> Walk<'py> { self.add_declaration(values, placement) } - fn add_named_declarations(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_named_declarations(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { let audit_kind: String = DeclarationKind::Audit.as_str().to_owned(); for audit_file in items(inputs, "audit_files")? { let stem_name: Bound<'py, PyAny> = @@ -621,7 +612,7 @@ impl<'py> Walk<'py> { Ok((line, metadata)) } - fn add_macros( + pub(crate) fn add_macros( &mut self, inputs: &Bound<'py, PyAny>, macros: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -701,7 +692,7 @@ fn resource_root<'py>( )) } -fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult { +pub(crate) fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult { let classes: &ScopeClasses<'py> = &walk.classes; let py: Python<'py> = classes.py(); let mut resources: Vec> = Vec::with_capacity(native.resources.len()); @@ -853,46 +844,13 @@ pub(crate) fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult )) } -/// Python's `open_native_scope_index`: the natively built index of discovered inputs. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] -pub(crate) fn open_native_scope_index( - py: Python<'_>, - discovered_inputs: &Bound<'_, PyAny>, - loaded_macros: &Bound<'_, PyAny>, - static_macros: Option>, -) -> PyResult { - let static_macros: Bound<'_, PyAny> = match static_macros { - Some(static_macros) => static_macros, - None => PyTuple::empty(py).into_any(), - }; - let static_macros: &Bound<'_, PyAny> = &static_macros; - let mut walk: Walk<'_> = Walk::new(py)?; - walk.add_models(discovered_inputs)?; - walk.add_resources(discovered_inputs)?; - walk.add_declaration_files(discovered_inputs)?; - walk.add_named_declarations(discovered_inputs)?; - walk.add_macros(discovered_inputs, (loaded_macros, static_macros))?; - if !walk.indexable { - return Err(unindexable(py)?); - } - let inputs: ScopeInputs = std::mem::take(&mut walk.inputs); - let built: Result = py - .compiler_detach(|| Ok(build_scope_index(inputs))) - .map_err(compiler_error)?; - let Ok(native) = built else { - return Err(unindexable(py)?); - }; - materialise(walk, native) -} - /// Python's `build_scope_index`: the natively built, partially valid index of discovered inputs. #[pyfunction] #[pyo3( name = "build_scope_index", signature = (*, discovered_inputs, loaded_macros = None, static_macros = None) )] -fn build_scope_index_binding( +pub(crate) fn build_scope_index_binding( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: Option>, @@ -906,10 +864,3 @@ fn build_scope_index_binding( open_native_scope_index(py, discovered_inputs, &loaded_macros, static_macros)?; Ok(native.index.clone_ref(py)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; - module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs index e0ea677f39..519c22847b 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs @@ -1,14 +1,11 @@ //! The scope lookup of a full scope index: canonical orders, mappings, targets and visibility. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{pyclass, pyfunction}; use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; -use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; use sqlbuild_scopes::scope_index::main::report_target::report_target; use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; use sqlbuild_scopes::scope_index::models::{ @@ -16,13 +13,9 @@ use sqlbuild_scopes::scope_index::models::{ ResolvedVisibility, ResolvedVisible, ResourceRecord, TargetFailure, TargetMatch, VisibilityGroups, }; -use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::qualified_identity; -use crate::scopes::_helpers::index::scope_objects::read_index; +use crate::scopes::_helpers::declarations::placement::{QualifiedRow, qualified_identity}; use crate::scopes::_helpers::index::scope_objects::resource_identity_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -53,17 +46,17 @@ pub(crate) struct NativeScopeLookup { pub(crate) records: ReportRecords, pub(crate) lookup: ReportLookup, /// Each input resource, declaration and grant's canonical position. - canonical: CanonicalPositions, + pub(crate) canonical: CanonicalPositions, } /// The inverse of the lookup's canonical orders. -struct CanonicalPositions { - resources: Vec, - declarations: Vec, - grants: Vec, +pub(crate) struct CanonicalPositions { + pub(crate) resources: Vec, + pub(crate) declarations: Vec, + pub(crate) grants: Vec, } -fn inverse(order: &[usize]) -> Vec { +pub(crate) fn inverse(order: &[usize]) -> Vec { let mut positions: Vec = vec![0; order.len()]; for (canonical, input) in order.iter().enumerate() { positions[*input] = canonical; @@ -82,7 +75,7 @@ pub(crate) fn grant_kind(value: &str) -> PyResult { } /// Input positions as canonical positions. -fn canonical(order: &[usize], positions: Vec) -> Vec { +pub(crate) fn canonical(order: &[usize], positions: Vec) -> Vec { positions .into_iter() .map(|position| order[position]) @@ -171,7 +164,10 @@ fn resolved_rows(resolved: ResolvedVisibility, grants: &[usize]) -> ResolvedRows ) } -fn picked<'py>(values: &Bound<'py, PyTuple>, order: &[usize]) -> PyResult> { +pub(crate) fn picked<'py>( + values: &Bound<'py, PyTuple>, + order: &[usize], +) -> PyResult> { let items: Vec> = order .iter() .map(|position| values.get_item(*position)) @@ -188,7 +184,7 @@ fn mapping_proxy<'py>(entries: &Bound<'py, PyDict>) -> PyResult( +pub(crate) fn grouped<'py>( groups: &KeyedGroups, records: &Bound<'py, PyTuple>, key: &str, @@ -201,7 +197,7 @@ fn grouped<'py>( mapping_proxy(&entries) } -fn path_grouped<'py>( +pub(crate) fn path_grouped<'py>( groups: &KeyedGroups, records: &Bound<'py, PyTuple>, paths: &[String], @@ -224,7 +220,7 @@ fn positions_by<'py>( mapping_proxy(&mapping) } -fn visibility_index<'py>( +pub(crate) fn visibility_index<'py>( classes: &ScopeClasses<'py>, canonical_declarations: &Bound<'py, PyTuple>, visibility: &VisibilityGroups, @@ -256,91 +252,10 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { - let records: ReportRecords = cached_report_records(&read_index(index)?); - let built = py - .compiler_detach(|| Ok(report_lookup(&records))) - .map_err(compiler_error)?; - let lookup: ReportLookup = match built { - Ok(lookup) => lookup, - Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), - }; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; - let declarations: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; - let usages: Bound<'_, PyTuple> = tuple_attribute(index, "usages")?; - let grants: Bound<'_, PyTuple> = tuple_attribute(index, "grants")?; - let canonical_declarations: Bound<'_, PyTuple> = picked(&declarations, &lookup.declarations)?; - let canonical_index: Bound<'_, PyAny> = classes.model(RecordId::ScopeIndex)?.call1(( - picked(&tuple_attribute(index, "ownership_roots")?, &lookup.roots)?, - picked(&resources, &lookup.resources)?, - &canonical_declarations, - picked(&usages, &lookup.usages)?, - picked(&grants, &lookup.grants)?, - native_field(index, "visibility")?, - native_field(index, "inaccessible")?, - picked(&tuple_attribute(index, "diagnostics")?, &lookup.diagnostics)?, - native_field(index, "completeness")?, - ))?; - let groups = &lookup.groups; - let mappings: [Bound<'_, PyAny>; 6] = [ - grouped(&groups.resources, &resources, "identity")?, - path_grouped( - &groups.resources_by_path, - &resources, - &lookup.resource_paths, - )?, - grouped(&groups.declarations, &declarations, "identity")?, - grouped(&groups.usages_by_consumer, &usages, "consumer")?, - grouped(&groups.usages_by_declaration, &usages, "declaration")?, - grouped(&groups.grants_by_resource, &grants, "resource")?, - ]; - let visibility: Bound<'_, PyAny> = - visibility_index(&classes, &canonical_declarations, &lookup.visibility)?; - let canonical: CanonicalPositions = CanonicalPositions { - resources: inverse(&lookup.resources), - declarations: inverse(&lookup.declarations), - grants: inverse(&lookup.grants), - }; - let native: Py = Py::new( - py, - NativeScopeLookup { - records, - lookup, - canonical, - }, - )?; - let [ - by_identity, - by_path, - by_declaration, - by_consumer, - by_used, - by_resource, - ] = mappings; - Ok(classes - .model(RecordId::ScopeLookup)? - .call1(( - canonical_index, - by_identity, - by_path, - by_declaration, - by_consumer, - by_used, - by_resource, - visibility, - native, - ))? - .unbind()) -} - /// The canonical project-relative POSIX path; absolute or escaping paths raise. #[pyfunction] #[pyo3(name = "normalize_scope_path", signature = (*, path))] -fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { let path: String = path.str()?.extract()?; match normalize_scope_path(&path) { Ok(normalized) => Ok(normalized), @@ -350,10 +265,3 @@ fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_scope_lookup, module)?)?; - module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs index 016b1beb3e..993733662e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs @@ -13,15 +13,10 @@ use sqlbuild_scopes::scope_index::models::{ ResourceKind, ResourceRecord, }; -use crate::scopes::_helpers::declarations::placement::DeclarationRow; -use crate::scopes::_helpers::declarations::placement::IdentityRow; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::RootRow; -use crate::scopes::_helpers::declarations::placement::declaration_identity; -use crate::scopes::_helpers::declarations::placement::declaration_record; -use crate::scopes::_helpers::declarations::placement::ownership_root; -use crate::scopes::_helpers::declarations::placement::qualified_identity; -use crate::scopes::_helpers::declarations::placement::resource_identity; +use crate::scopes::_helpers::declarations::placement::{ + DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, + ownership_root, qualified_identity, resource_identity, +}; use crate::scopes::_helpers::index::scope_lookup::grant_kind; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs index b84011d0f1..4d5ea1010e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs @@ -3,11 +3,9 @@ use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::compile_usages::main::merged_usages::merged_usages; use sqlbuild_scopes::compile_usages::main::schema_scope_usages::schema_scope_usages; use sqlbuild_scopes::compile_usages::models::{ @@ -276,8 +274,3 @@ pub(crate) fn merged_usage_rows( combined, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs index eb801aa90f..ae27ddc070 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs @@ -1,7 +1,7 @@ //! Expected-model relationship names and top-level CTEs of SQL tests and scenarios, scanned natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; use sqlbuild_scopes::relationship_names::models::{ @@ -26,7 +26,7 @@ fn source(scenario: bool) -> RelationshipSource { /// Each `(sql, file label)` body's expected models or the scan's error. #[pyfunction] -fn scope_expected_model_names( +pub(crate) fn scope_expected_model_names( py: Python<'_>, texts: Vec<(String, String)>, scenario: bool, @@ -47,7 +47,7 @@ fn scope_expected_model_names( /// Each `(sql, file label)` test body's top-level CTEs or the scan's error. #[pyfunction] -fn scope_test_ctes( +pub(crate) fn scope_test_ctes( py: Python<'_>, texts: Vec<(String, String)>, syntax: LexicalSyntaxInput, @@ -71,9 +71,3 @@ fn scope_test_ctes( }) .collect()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; - module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs index 396b548083..35dc69385e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs @@ -1,5 +1,6 @@ //! Scope report requests read from, and results built as, the Python report models. +use crate::scopes::main::reports::json_to_python::json_to_python; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -12,14 +13,12 @@ use sqlbuild_scopes::scope_report::main::cached_diagnostic_facts::cached_diagnos use sqlbuild_scopes::scope_report::models::{ BrowseResult, DeclarationReport, DeclarationTarget, Explanation, ListResult, MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, ScopeChainEntry, - ScopeFolder, ScopeReport, ScopeResult, ScopeSection, SourceLocation, + ScopeFolder, ScopeReport, ScopeSection, SourceLocation, }; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::read_declaration_identity; -use crate::scopes::_helpers::index::scope_objects::read_diagnostic; -use crate::scopes::_helpers::index::scope_objects::read_qualified; +use crate::scopes::_helpers::index::scope_objects::{ + field, optional_item, read_declaration_identity, read_diagnostic, read_qualified, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; @@ -98,43 +97,6 @@ pub(crate) fn read_filters(filters: &Bound<'_, PyAny>) -> PyResult, value: JsonValue, tuples: bool) -> PyResult { - match value { - JsonValue::Null => Ok(py.None()), - JsonValue::Bool(flag) => flag.into_py_any(py), - JsonValue::Integer(integer) => match integer.to_i128() { - Some(number) => number.into_py_any(py), - None => Ok(py - .import("builtins")? - .getattr("int")? - .call1((integer.as_str(),))? - .unbind()), - }, - JsonValue::Float(number) => number.into_py_any(py), - JsonValue::String(text) => text.into_py_any(py), - JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), - JsonValue::Array(items) => { - let values: Vec = items - .into_iter() - .map(|item| json_to_python(py, item, tuples)) - .collect::>()?; - if tuples { - PyTuple::new(py, values)?.into_py_any(py) - } else { - PyList::new(py, values)?.into_py_any(py) - } - } - JsonValue::Object(entries) => { - let dictionary: Bound<'_, PyDict> = PyDict::new(py); - for (key, item) in entries { - dictionary.set_item(key, json_to_python(py, item, tuples)?)?; - } - dictionary.into_py_any(py) - } - } -} - /// `plain_value(int(integer))`: a 64-bit signed or unsigned number, else the wide-integer error. fn plain_integer(integer: &JsonInteger) -> PyResult { let Some(number) = integer.to_i128() else { @@ -504,7 +466,7 @@ fn read_declaration_report(report: &Bound<'_, PyAny>) -> PyResult, name: &str, ) -> PyResult> { @@ -514,7 +476,7 @@ fn read_declaration_reports( .collect() } -fn read_section(section: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_section(section: &Bound<'_, PyAny>) -> PyResult { Ok(ScopeSection { name: field(section, "name")?, total: field(section, "total")?, @@ -529,7 +491,7 @@ fn read_section(section: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { let mut kind_counts: Vec<(String, i64)> = Vec::new(); for entry in native_field(folder, "kind_counts")?.try_iter()? { let (kind, count): (Bound<'_, PyAny>, i64) = entry?.extract()?; @@ -552,7 +514,7 @@ fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> { native_field(item, "diagnostics")? .try_iter()? .map(|diagnostic| { @@ -587,7 +549,7 @@ fn read_move(preview: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_report(report: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_report(report: &Bound<'_, PyAny>) -> PyResult { let resource: Bound<'_, PyAny> = native_field(report, "resource")?; Ok(ScopeReport { resource: ResourceReport { @@ -628,30 +590,3 @@ fn read_report(report: &Bound<'_, PyAny>) -> PyResult { complete: field(report, "complete")?, }) } - -/// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -pub(crate) fn read_result( - classes: &ScopeClasses<'_>, - result: &Bound<'_, PyAny>, -) -> PyResult { - if result.is_instance(&classes.model(RecordId::ScopeReport)?)? { - return Ok(ScopeResult::Report(Box::new(read_report(result)?))); - } - if result.is_instance(&classes.model(RecordId::ScopeBrowseResult)?)? { - return Ok(ScopeResult::Browse(BrowseResult { - folder: field(result, "folder")?, - folders: native_field(result, "folders")? - .try_iter()? - .map(|folder| read_folder(&folder?)) - .collect::>()?, - diagnostics: read_diagnostics(result)?, - complete: field(result, "complete")?, - })); - } - Ok(ScopeResult::List(ListResult { - folder: field(result, "folder")?, - declarations: read_declaration_reports(result, "declarations")?, - section: read_section(&native_field(result, "section")?)?, - diagnostics: read_diagnostics(result)?, - })) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs index cfb49a189b..1494f26d96 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs @@ -1,70 +1,54 @@ //! `sqb scope` reports over a `ScopeLookup`, their serialization and the metadata projection. +use crate::scopes::main::reports::json_to_python::json_to_python; +use crate::scopes::main::reports::read_result::read_result; +use crate::scopes::main::reports::scope_browse::scope_browse; +use crate::scopes::main::reports::scope_explain::scope_explain; +use crate::scopes::main::reports::scope_list::scope_list; +use crate::scopes::main::reports::scope_report::scope_report; use crate::scopes::models::ExplainArguments; use crate::scopes::models::ListArguments; use crate::scopes::models::ReportArguments; use pyo3::exceptions::{PyIndexError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; -use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; -use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; use sqlbuild_scopes::scope_report::main::cached_completeness::cached_completeness; use sqlbuild_scopes::scope_report::main::cached_declaration_values::cached_declaration_values; use sqlbuild_scopes::scope_report::main::cached_projection_index::cached_projection_index; -use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; -use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; -use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; -use sqlbuild_scopes::scope_report::main::rebound_scope_usages::rebound_scope_usages; use sqlbuild_scopes::scope_report::main::scope_metadata_projection::scope_metadata_projection; use sqlbuild_scopes::scope_report::main::serialize_scope_result::serialize_scope_result; use sqlbuild_scopes::scope_report::models::{ - DeclarationValues, DiagnosticFacts, ExplainRequest, FolderRequest, MoveRequest, - ProjectionIndex, ReportDiagnostic, ReportFailure, ReportRequest, ReportSource, ReportValues, - ScopeResult, + DeclarationValues, DiagnosticFacts, FolderRequest, ProjectionIndex, ReportDiagnostic, + ReportFailure, ReportSource, ReportValues, ScopeResult, }; use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_declaration; -use crate::scopes::_helpers::index::scope_objects::read_index; -use crate::scopes::_helpers::reports::scope_report_objects::browse_object; -use crate::scopes::_helpers::reports::scope_report_objects::diagnostics_object; -use crate::scopes::_helpers::reports::scope_report_objects::explanation_object; -use crate::scopes::_helpers::reports::scope_report_objects::json_to_plain; -use crate::scopes::_helpers::reports::scope_report_objects::json_to_python; -use crate::scopes::_helpers::reports::scope_report_objects::list_object; -use crate::scopes::_helpers::reports::scope_report_objects::move_object; -use crate::scopes::_helpers::reports::scope_report_objects::read_declaration_target; -use crate::scopes::_helpers::reports::scope_report_objects::read_filters; -use crate::scopes::_helpers::reports::scope_report_objects::read_optional_target; -use crate::scopes::_helpers::reports::scope_report_objects::read_result; -use crate::scopes::_helpers::reports::scope_report_objects::read_target; -use crate::scopes::_helpers::reports::scope_report_objects::report_object; +use crate::scopes::_helpers::index::scope_objects::{ + read_completeness, read_declaration, read_index, +}; +use crate::scopes::_helpers::reports::scope_report_objects::{read_filters, read_optional_target}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; pub(crate) type PyObject = Py; const CURRENT_FOLDER: &str = "."; /// One lookup's native lookup and every declaration's values in input order. -struct LookupReports { - lookup: Py, +pub(crate) struct LookupReports { + pub(crate) lookup: Py, values: ReportValues, } impl LookupReports { - fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { + pub(crate) fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { ReportSource { records: &lookup.records, lookup: &lookup.lookup, @@ -74,7 +58,7 @@ impl LookupReports { } /// The reports of a `ScopeLookup`: its native lookup with its canonical declarations' values. -fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { let index: Bound<'_, PyAny> = native_field(lookup, "index")?; let native: Py = native_field(lookup, "native")?.extract()?; let rows: Vec = native_field(&index, "declarations")? @@ -100,14 +84,14 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { }) } -fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult { +pub(crate) fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult { Ok(PyErr::from_value( error_class(py, class)?.call1((message,))?, )) } /// The error Python raises for a failed query. -fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { +pub(crate) fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { match failure { ReportFailure::Path(rejected) => scope_error( py, @@ -124,90 +108,21 @@ fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { } } -fn path_text(value: Option>) -> PyResult> { +pub(crate) fn path_text(value: Option>) -> PyResult> { value .filter(|value| !value.is_none()) .map(|value| value.str()?.extract()) .transpose() } -fn raised(diagnostics: Vec) -> Vec { +pub(crate) fn raised(diagnostics: Vec) -> Vec { diagnostics .into_iter() .map(ReportDiagnostic::Raised) .collect() } -/// `query_scope_report`: one existing target or prospective path; invalid paths raise. -pub(crate) fn scope_report( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ReportArguments<'_>, -) -> PyResult { - let ReportArguments { - target, - at, - directory, - filters, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { - Some(filters) => filters, - None => classes.model(RecordId::ScopeReportFilters)?.call0()?, - }; - let reports: LookupReports = lookup_reports(lookup)?; - let request: ReportRequest = ReportRequest { - target: read_optional_target(&classes, target.as_ref())?, - at: path_text(at)?, - directory, - filters: read_filters(&selected)?, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match build_scope_report(reports.source(native), &request) { - Ok(report) => { - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(report_object(&classes, report, (&selected, &diagnostics))?.unbind()) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// `explain_scope_declaration`: one qualified declaration in an optional context. -pub(crate) fn scope_explain( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ExplainArguments<'_>, -) -> PyResult<(PyObject, PyObject)> { - let ExplainArguments { - declaration, - target, - at, - directory, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: ExplainRequest = ExplainRequest { - declaration: read_declaration_target(&classes, &declaration)?, - target: read_optional_target(&classes, target.as_ref())?, - at: path_text(at)?, - directory, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match explain_scope_declaration(reports.source(native), &request) { - Ok((explanation, diagnostics)) => { - let index_diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(( - explanation_object(&classes, explanation)?.unbind(), - diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), - )) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -fn folder_request( +pub(crate) fn folder_request( classes: &ScopeClasses<'_>, location: (String, Option>, bool), filters: Option>, @@ -224,95 +139,10 @@ fn folder_request( }) } -/// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -pub(crate) fn scope_browse( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: (String, Option>, bool), -) -> PyResult { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: FolderRequest = folder_request(&classes, request, None)?; - let native: &NativeScopeLookup = reports.lookup.get(); - let result = browse_scope_folders(reports.source(native), &request); - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(browse_object(&classes, result, &diagnostics)?.unbind()) -} - -/// `list_scope_declarations`: declarations recursively beneath one definition folder. -pub(crate) fn scope_list( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ListArguments<'_>, -) -> PyResult { - let ListArguments { - folder, - filters, - target, - target_is_prospective, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: FolderRequest = - folder_request(&classes, (folder, target, target_is_prospective), filters)?; - let native: &NativeScopeLookup = reports.lookup.get(); - match list_scope_declarations(reports.source(native), &request) { - Ok(result) => { - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(list_object(&classes, result, &diagnostics)?.unbind()) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -#[pyfunction] -#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] -pub(crate) fn scope_move_preview( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - resource: &Bound<'_, PyAny>, - destination: &Bound<'_, PyAny>, -) -> PyResult<(Option, PyObject)> { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: MoveRequest = MoveRequest { - resource: read_target(&classes, resource)?, - destination: destination.str()?.extract()?, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match preview_scope_move(reports.source(native), &request) { - Ok((preview, diagnostics)) => { - let index_diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(( - preview - .map(|preview| Ok::<_, PyErr>(move_object(&classes, preview)?.unbind())) - .transpose()?, - diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), - )) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// Scoped declarations the resources at `path` use that resolve differently from `destination`. -pub(crate) fn scope_rebound_usages( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - paths: (&str, &str), -) -> PyResult>> { - let reports: LookupReports = lookup_reports(lookup)?; - let native: &NativeScopeLookup = reports.lookup.get(); - match rebound_scope_usages(reports.source(native), paths) { - Ok(rebound) => Ok(rebound), - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn serialized( + classes: &ScopeClasses<'_>, + result: &Bound<'_, PyAny>, +) -> PyResult { let result: ScopeResult = read_result(classes, result)?; serialize_scope_result(&result).map_err(|failure| match failure { ReportFailure::Serialization(message) => PyValueError::new_err(message), @@ -320,25 +150,6 @@ fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult }) } -/// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -#[pyfunction] -#[pyo3(signature = (*, report))] -pub(crate) fn serialize_scope_report( - py: Python<'_>, - report: &Bound<'_, PyAny>, -) -> PyResult { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - match serialized(&classes, report) { - Err(error) if error.is_instance_of::(py) => { - let message: String = error.value(py).str()?.extract()?; - let raised: PyErr = scope_error(py, ErrorId::Scope, message)?; - raised.set_cause(py, Some(error)); - Err(raised) - } - other => other, - } -} - /// `scope_metadata_projection` of one `ScopeIndex`, as the dictionary Python builds. #[pyfunction] #[pyo3(name = "scope_metadata_projection", signature = (*, index))] @@ -346,12 +157,7 @@ pub(crate) fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> P json_to_python(py, projection_json(index)?, false) } -/// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { - json_to_plain(projection_json(index)?) -} - -fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = cached_projection_index(read_index(index)?); Ok(scope_metadata_projection(&index)) } @@ -366,7 +172,7 @@ fn optional<'py>( /// Python's `query_scope_report`. #[pyfunction] #[pyo3(name = "query_scope_report", signature = (**options))] -fn query_scope_report_binding<'py>( +pub(crate) fn query_scope_report_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -391,7 +197,7 @@ fn query_scope_report_binding<'py>( /// Python's `explain_scope_declaration`. #[pyfunction] #[pyo3(name = "explain_scope_declaration", signature = (**options))] -fn explain_scope_declaration_binding<'py>( +pub(crate) fn explain_scope_declaration_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult<(PyObject, PyObject)> { @@ -416,7 +222,7 @@ fn explain_scope_declaration_binding<'py>( /// Python's `browse_scope_folders`. #[pyfunction] #[pyo3(name = "browse_scope_folders", signature = (**options))] -fn browse_scope_folders_binding<'py>( +pub(crate) fn browse_scope_folders_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -444,7 +250,7 @@ fn browse_scope_folders_binding<'py>( /// Python's `list_scope_declarations`. #[pyfunction] #[pyo3(name = "list_scope_declarations", signature = (**options))] -fn list_scope_declarations_binding<'py>( +pub(crate) fn list_scope_declarations_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -468,14 +274,3 @@ fn list_scope_declarations_binding<'py>( }, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; - module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; - module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; - module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; - module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; - module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; - module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs index 59bb9c9aa3..194f2fe490 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs @@ -1,12 +1,15 @@ //! Register the scope validation functions and classes. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_validation::scope_placement_validated_index; +use crate::scopes::_helpers::checks::scope_validation::scope_relocated_declarations; +use crate::scopes::main::checks::validate_scope_index::validate_scope_index; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope validation functions and classes. -#[inline] pub fn register_scope_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_validation::register(module) + module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; + module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs index 03b3da25b7..441a85470d 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs @@ -1,12 +1,20 @@ //! Register the scope visibility functions and classes. -use crate::scopes::_helpers::checks::scope_visibility; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_visibility::resolve_scope_declaration_visibility; +use crate::scopes::_helpers::checks::scope_visibility::resolve_scope_visibility; +use crate::scopes::_helpers::checks::scope_visibility::scope_query_target; +use crate::scopes::_helpers::checks::scope_visibility::scope_resolve_path_visibility; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope visibility functions and classes. -#[inline] pub fn register_scope_visibility(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_visibility::register(module) + module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; + module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_scope_declaration_visibility, + module + )?)?; + module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs index 1f67ce386f..1d60cd9dba 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs @@ -1,17 +1,60 @@ //! Relocated records after every move in `moves`, and per record whether it lacks a needed group. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::scopes::_helpers::checks::scope_validation::RUNTIME_USAGE; +use crate::scopes::_helpers::checks::scope_validation::replace; +use crate::scopes::_helpers::index::scope_objects::{ + read_completeness, read_placement_index, root_object, tuple_attribute, +}; +use crate::scopes::models::ScopeClasses; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyTuple, PyTupleMethods}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::placement::main::relocate_for_resource_moves::relocate_for_resource_moves; +use sqlbuild_scopes::placement::models::{PlacementIndex, Relocation}; use sqlbuild_scopes::scope_index::models::ResourceIdentity; /// Relocated records after every move in `moves`, and per record whether it lacks a needed group. -#[inline] pub fn relocated_declarations<'py>( index: &Bound<'py, PyAny>, moves: &[(ResourceIdentity, String)], ) -> PyResult, Vec)>> { - scope_validation::relocated_declarations(index, moves) + let py: Python<'py> = index.py(); + if !read_completeness(index)?[RUNTIME_USAGE] { + return Ok(None); + } + let placement: PlacementIndex = read_placement_index(index)?; + let relocations: Option> = py + .compiler_detach(|| Ok(relocate_for_resource_moves(&placement, moves))) + .map_err(compiler_error)?; + let Some(relocations) = relocations else { + return Ok(None); + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let records: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let mut moved: Vec> = Vec::with_capacity(relocations.len()); + let mut grouping: Vec = Vec::with_capacity(relocations.len()); + for item in relocations { + let owning_path: Bound<'_, PyAny> = item.owning_path.into_bound_py_any(py)?; + let record: Bound<'py, PyAny> = replace( + &records.get_item(item.declaration)?, + &[ + ("path", item.path.into_bound_py_any(py)?), + ( + "scope", + classes.member(RecordId::ScopeKind, item.scope.as_str())?, + ), + ( + "ownership_root", + root_object(&classes, &item.ownership_root)?, + ), + ("owning_path", owning_path), + ], + )?; + moved.push(record); + grouping.push(item.requires_group); + } + Ok(Some((PyTuple::new(py, moved)?, grouping))) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs index 8eb10e7943..1ed107998c 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs @@ -1,14 +1,62 @@ //! Compile inputs' canonical index with all usage and placement facts. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_validation::COMPILE_USAGE_GROUPS; +use crate::scopes::_helpers::checks::scope_validation::scope_placement_validated_index; +use crate::scopes::_helpers::checks::scope_validation::usage_records; +use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Compile inputs' canonical index with all usage and placement facts. -#[inline] pub fn scope_index_with_compile_usages<'py>( inputs: &Bound<'py, PyAny>, ) -> PyResult> { - scope_validation::scope_index_with_compile_usages(inputs) + let py: Python<'py> = inputs.py(); + let scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; + let resolver: Bound<'py, PyAny> = if scope.is_none() { + scope + } else { + native_field(&scope, "resolver")? + }; + let schema_files: Bound<'py, PyAny> = native_field( + &native_field(inputs, "discovered_inputs")?, + "model_schema_files", + )?; + let mut usages: Vec> = Vec::new(); + usages.extend(usage_records( + model_schema_scope_usages(&schema_files, &resolver)?.as_any(), + )?); + let digests: bool = sqlbuild_python_base::compile_context::main::delta::capturing::capturing(); + for model in native_field(inputs, "model_inputs")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let macro_usages: Vec> = + usage_records(&native_field(&model, "macro_usages")?)?; + let declaration_usages: Vec> = + usage_records(&native_field(&model, "declaration_usages")?)?; + if digests { + sqlbuild_python_base::compile_context::main::image_facts::record_model_usages::record_model_usages( + &model, + (¯o_usages, &declaration_usages), + )?; + } + usages.extend(macro_usages); + usages.extend(declaration_usages); + } + for group in COMPILE_USAGE_GROUPS { + for item in native_field(inputs, group)?.try_iter()? { + usages.extend(usage_records(&native_field(&item?, "declaration_usages")?)?); + } + } + let enforce_placement: bool = native_field( + &native_field(&native_field(inputs, "project_config")?, "scopes")?, + "enforce_placement", + )? + .extract()?; + Ok(scope_placement_validated_index( + &native_field(inputs, "scope_index")?, + enforce_placement, + Some(PyTuple::new(py, usages)?), + )? + .into_bound(py)) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs index 82e2186ba0..896d8d70de 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs @@ -1,11 +1,7 @@ //! The tolerant index: fault diagnostics, prospective resources and partial completeness. use crate::scopes::_helpers::checks::scope_tolerant; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; /// The tolerant index: fault diagnostics, prospective resources and partial completeness. #[inline] diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs index 117154f15d..60af7d0c08 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs @@ -1,12 +1,39 @@ //! Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; /// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, index))] pub fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { - scope_validation::validate_scope_index(index) + let py: Python<'_> = index.py(); + let error: Bound<'_, PyAny> = enum_member(py, RecordId::ScopeDiagnosticSeverity, "ERROR")?; + let mut errors: Vec> = Vec::new(); + for item in native_field(index, "diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if native_field(&item, "severity")?.is(&error) { + errors.push(item); + } + } + if errors.is_empty() { + return Ok(()); + } + let mut message: String = String::from("Invalid declaration scope index:"); + for item in &errors { + let code: String = native_field(&native_field(item, "code")?, "value")?.extract()?; + let text: String = native_field(item, "message")?.extract()?; + message.push_str(&format!("\n[{code}] {text}")); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("diagnostics", PyTuple::new(py, errors)?)?; + Err(PyErr::from_value( + error_class(py, ErrorId::ScopeValidation)?.call((message,), Some(&options))?, + )) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs index b613d65b38..7aad982058 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs @@ -1,12 +1,10 @@ //! Register the declaration contexts functions and classes. -use crate::scopes::_helpers::declarations::declaration_contexts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::models::NativeDeclarationContexts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the declaration contexts functions and classes. -#[inline] pub fn register_declaration_contexts(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_contexts::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs index d9260694d9..4233aa2229 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs @@ -1,14 +1,104 @@ //! Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. -use crate::scopes::_helpers::index::scope_lookup; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_lookup::CanonicalPositions; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_lookup::grouped; +use crate::scopes::_helpers::index::scope_lookup::inverse; +use crate::scopes::_helpers::index::scope_lookup::path_grouped; +use crate::scopes::_helpers::index::scope_lookup::picked; +use crate::scopes::_helpers::index::scope_lookup::scope_path_error; +use crate::scopes::_helpers::index::scope_lookup::visibility_index; +use crate::scopes::_helpers::index::scope_objects::{read_index, tuple_attribute}; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; +use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; +use sqlbuild_scopes::scope_index::models::{ReportLookup, ReportRecords}; +use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; /// Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, index))] pub fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { - scope_lookup::build_scope_lookup(py, index) + let records: ReportRecords = cached_report_records(&read_index(index)?); + let built = py + .compiler_detach(|| Ok(report_lookup(&records))) + .map_err(compiler_error)?; + let lookup: ReportLookup = match built { + Ok(lookup) => lookup, + Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; + let declarations: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let usages: Bound<'_, PyTuple> = tuple_attribute(index, "usages")?; + let grants: Bound<'_, PyTuple> = tuple_attribute(index, "grants")?; + let canonical_declarations: Bound<'_, PyTuple> = picked(&declarations, &lookup.declarations)?; + let canonical_index: Bound<'_, PyAny> = classes.model(RecordId::ScopeIndex)?.call1(( + picked(&tuple_attribute(index, "ownership_roots")?, &lookup.roots)?, + picked(&resources, &lookup.resources)?, + &canonical_declarations, + picked(&usages, &lookup.usages)?, + picked(&grants, &lookup.grants)?, + native_field(index, "visibility")?, + native_field(index, "inaccessible")?, + picked(&tuple_attribute(index, "diagnostics")?, &lookup.diagnostics)?, + native_field(index, "completeness")?, + ))?; + let groups = &lookup.groups; + let mappings: [Bound<'_, PyAny>; 6] = [ + grouped(&groups.resources, &resources, "identity")?, + path_grouped( + &groups.resources_by_path, + &resources, + &lookup.resource_paths, + )?, + grouped(&groups.declarations, &declarations, "identity")?, + grouped(&groups.usages_by_consumer, &usages, "consumer")?, + grouped(&groups.usages_by_declaration, &usages, "declaration")?, + grouped(&groups.grants_by_resource, &grants, "resource")?, + ]; + let visibility: Bound<'_, PyAny> = + visibility_index(&classes, &canonical_declarations, &lookup.visibility)?; + let canonical: CanonicalPositions = CanonicalPositions { + resources: inverse(&lookup.resources), + declarations: inverse(&lookup.declarations), + grants: inverse(&lookup.grants), + }; + let native: Py = Py::new( + py, + NativeScopeLookup { + records, + lookup, + canonical, + }, + )?; + let [ + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + ] = mappings; + Ok(classes + .model(RecordId::ScopeLookup)? + .call1(( + canonical_index, + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + visibility, + native, + ))? + .unbind()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs index 2196717847..c26ad856bc 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs @@ -1,19 +1,46 @@ //! Python's `open_native_scope_index`: the natively built index of discovered inputs. -use crate::scopes::_helpers::index::scope_index; +use crate::scopes::_helpers::index::scope_index::Walk; +use crate::scopes::_helpers::index::scope_index::materialise; +use crate::scopes::_helpers::index::scope_index::unindexable; use crate::scopes::models::NativeScopeIndex; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use sqlbuild_scopes::scope_index::models::{ScopeDeferral, ScopeIndex, ScopeInputs}; /// Python's `open_native_scope_index`: the natively built index of discovered inputs. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] pub fn open_native_scope_index( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: &Bound<'_, PyAny>, static_macros: Option>, ) -> PyResult { - scope_index::open_native_scope_index(py, discovered_inputs, loaded_macros, static_macros) + let static_macros: Bound<'_, PyAny> = match static_macros { + Some(static_macros) => static_macros, + None => PyTuple::empty(py).into_any(), + }; + let static_macros: &Bound<'_, PyAny> = &static_macros; + let mut walk: Walk<'_> = Walk::new(py)?; + walk.add_models(discovered_inputs)?; + walk.add_resources(discovered_inputs)?; + walk.add_declaration_files(discovered_inputs)?; + walk.add_named_declarations(discovered_inputs)?; + walk.add_macros(discovered_inputs, (loaded_macros, static_macros))?; + if !walk.indexable { + return Err(unindexable(py)?); + } + let inputs: ScopeInputs = std::mem::take(&mut walk.inputs); + let built: Result = py + .compiler_detach(|| Ok(build_scope_index(inputs))) + .map_err(compiler_error)?; + let Ok(native) = built else { + return Err(unindexable(py)?); + }; + materialise(walk, native) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs index ac5ddfb553..0a379823e8 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs @@ -1,12 +1,26 @@ //! The scope index fingerprint of `project_dir`, `None` when it cannot be read. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_cache::sqlbuild_version; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::project_fingerprint_inputs::project_fingerprint_inputs; +use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; +use sqlbuild_scopes::scope_cache::models::{Fingerprint, FingerprintInputs}; use std::path::PathBuf; /// The scope index fingerprint of `project_dir`, `None` when it cannot be read. -#[inline] pub fn project_fingerprint(py: Python<'_>, project_dir: PathBuf) -> PyResult> { - scope_cache::project_fingerprint(py, project_dir) + let inputs: FingerprintInputs = project_fingerprint_inputs(sqlbuild_version(py)?); + let fingerprint: Fingerprint = py + .compiler_detach(|| Ok(scope_fingerprint(&project_dir, &inputs))) + .map_err(compiler_error)?; + match fingerprint { + Fingerprint::Hash(hash) => Ok(Some(hash)), + Fingerprint::Unavailable => Ok(None), + Fingerprint::Unserializable => Err(PyTypeError::new_err( + "a scope-relevant config value is not JSON serializable", + )), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs index 146846e797..e481760b23 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs @@ -1,18 +1,29 @@ //! The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_objects::index_object; +use crate::scopes::models::ScopeClasses; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; use std::path::PathBuf; /// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -#[inline] pub fn read_scope_index_cache( py: Python<'_>, path: PathBuf, fingerprint: String, ) -> PyResult>> { - scope_cache::read_scope_index_cache(py, path, fingerprint) + let cached = py + .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) + .map_err(compiler_error)?; + let Ok(index) = cached else { + return Ok(None); + }; + match index_object(&ScopeClasses::new(py)?, &index) { + Ok(object) => Ok(Some(object.unbind())), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs index 4dcdf09e70..aa70b4cb80 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs @@ -1,12 +1,11 @@ //! Register the scope cache functions and classes. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_cache::scope_index_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope cache functions and classes. -#[inline] pub fn register_scope_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_cache::register(module) + module.add_function(wrap_pyfunction!(scope_index_fingerprint, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs index 1194dee508..3e901df83f 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs @@ -1,12 +1,15 @@ //! Register the scope index functions and classes. -use crate::scopes::_helpers::index::scope_index; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_index::build_scope_index_binding; +use crate::scopes::main::index::open_native_scope_index::open_native_scope_index; +use crate::scopes::models::NativeScopeIndex; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope index functions and classes. -#[inline] pub fn register_scope_index(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_index::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs index 4c3e76c30d..11b77031f3 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs @@ -1,12 +1,15 @@ //! Register the scope lookup functions and classes. -use crate::scopes::_helpers::index::scope_lookup; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_lookup::scope_normalize_path; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope lookup functions and classes. -#[inline] pub fn register_scope_lookup(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_lookup::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_scope_lookup, module)?)?; + module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs index 1f3e8858d2..64c5a6faf7 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs @@ -1,19 +1,31 @@ //! Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_objects::{read_completeness, read_index}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; +use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; +use sqlbuild_scopes::scope_cache::models::CachedIndex; use std::path::PathBuf; /// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -#[inline] pub fn write_scope_index_cache( py: Python<'_>, location: (PathBuf, String), fingerprint: String, index: &Bound<'_, PyAny>, ) -> PyResult { - scope_cache::write_scope_index_cache(py, location, fingerprint, index) + if !read_completeness(index)?.iter().all(|done| *done) { + return Ok(false); + } + let index: CachedIndex = read_index(index)?; + let (directory, file_name) = location; + py.compiler_detach(|| { + Ok(match encode_scope_cache(&index, &fingerprint) { + Ok(envelope) => matches!(write_scope_cache(&directory, &file_name, &envelope), Ok(())), + Err(_) => false, + }) + }) + .map_err(compiler_error) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs index 2b36f57b36..782f59408b 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs @@ -1,12 +1,11 @@ //! Register the compile usages functions and classes. -use crate::scopes::_helpers::relationships::compile_usages; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the compile usages functions and classes. -#[inline] pub fn register_compile_usages(module: &Bound<'_, PyModule>) -> PyResult<()> { - compile_usages::register(module) + module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs index 03781fc7b2..7461a22953 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs @@ -1,12 +1,13 @@ //! Register the relationship names functions and classes. -use crate::scopes::_helpers::relationships::relationship_names; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::relationships::relationship_names::scope_expected_model_names; +use crate::scopes::_helpers::relationships::relationship_names::scope_test_ctes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the relationship names functions and classes. -#[inline] pub fn register_relationship_names(module: &Bound<'_, PyModule>) -> PyResult<()> { - relationship_names::register(module) + module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; + module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs index ff32cbaeb1..cfbb87fe85 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs @@ -1,13 +1,44 @@ //! A JSON value as Python builds it; arrays become tuples or lists. -use crate::scopes::_helpers::reports::scope_report_objects; use crate::scopes::_helpers::reports::scope_report_objects::PyObject; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_core::json::models::JsonValue; /// A JSON value as Python builds it; arrays become tuples or lists. -#[inline] pub fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { - scope_report_objects::json_to_python(py, value, tuples) + match value { + JsonValue::Null => Ok(py.None()), + JsonValue::Bool(flag) => flag.into_py_any(py), + JsonValue::Integer(integer) => match integer.to_i128() { + Some(number) => number.into_py_any(py), + None => Ok(py + .import("builtins")? + .getattr("int")? + .call1((integer.as_str(),))? + .unbind()), + }, + JsonValue::Float(number) => number.into_py_any(py), + JsonValue::String(text) => text.into_py_any(py), + JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), + JsonValue::Array(items) => { + let values: Vec = items + .into_iter() + .map(|item| json_to_python(py, item, tuples)) + .collect::>()?; + if tuples { + PyTuple::new(py, values)?.into_py_any(py) + } else { + PyList::new(py, values)?.into_py_any(py) + } + } + JsonValue::Object(entries) => { + let dictionary: Bound<'_, PyDict> = PyDict::new(py); + for (key, item) in entries { + dictionary.set_item(key, json_to_python(py, item, tuples)?)?; + } + dictionary.into_py_any(py) + } + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs index 9ccc5d8479..c789554831 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs @@ -1,12 +1,10 @@ //! `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::reports::scope_report_objects::json_to_plain; +use crate::scopes::_helpers::reports::scope_reports::projection_json; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -#[inline] pub fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { - scope_reports::metadata_projection_value(index) + json_to_plain(projection_json(index)?) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs index 255b493c51..aa1a4c8054 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs @@ -1,14 +1,37 @@ //! A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -use crate::scopes::_helpers::reports::scope_report_objects; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::reports::scope_report_objects::read_declaration_reports; +use crate::scopes::_helpers::reports::scope_report_objects::read_diagnostics; +use crate::scopes::_helpers::reports::scope_report_objects::read_folder; +use crate::scopes::_helpers::reports::scope_report_objects::read_report; +use crate::scopes::_helpers::reports::scope_report_objects::read_section; use crate::scopes::models::ScopeClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_scopes::scope_report::models::ScopeResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_scopes::scope_report::models::{BrowseResult, ListResult, ScopeResult}; /// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -#[inline] pub fn read_result(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { - scope_report_objects::read_result(classes, result) + if result.is_instance(&classes.model(RecordId::ScopeReport)?)? { + return Ok(ScopeResult::Report(Box::new(read_report(result)?))); + } + if result.is_instance(&classes.model(RecordId::ScopeBrowseResult)?)? { + return Ok(ScopeResult::Browse(BrowseResult { + folder: field(result, "folder")?, + folders: native_field(result, "folders")? + .try_iter()? + .map(|folder| read_folder(&folder?)) + .collect::>()?, + diagnostics: read_diagnostics(result)?, + complete: field(result, "complete")?, + })); + } + Ok(ScopeResult::List(ListResult { + folder: field(result, "folder")?, + declarations: read_declaration_reports(result, "declarations")?, + section: read_section(&native_field(result, "section")?)?, + diagnostics: read_diagnostics(result)?, + })) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs index fe8102070b..852a9c756a 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs @@ -1,12 +1,23 @@ //! Register the scope reports functions and classes. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::reports::scope_reports::browse_scope_folders_binding; +use crate::scopes::_helpers::reports::scope_reports::explain_scope_declaration_binding; +use crate::scopes::_helpers::reports::scope_reports::list_scope_declarations_binding; +use crate::scopes::_helpers::reports::scope_reports::metadata_projection; +use crate::scopes::_helpers::reports::scope_reports::query_scope_report_binding; +use crate::scopes::main::reports::scope_move_preview::scope_move_preview; +use crate::scopes::main::reports::serialize_scope_report::serialize_scope_report; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope reports functions and classes. -#[inline] pub fn register_scope_reports(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_reports::register(module) + module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; + module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; + module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; + module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; + module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; + module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; + module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs index fb15112112..f67246a526 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs @@ -1,9 +1,7 @@ //! `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. use crate::scopes::_helpers::reports::scope_errors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. #[inline] diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs index 1a134fa5a9..05e2bfb47f 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs @@ -1,18 +1,29 @@ //! `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::browse_object; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::folder_request; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; +use sqlbuild_scopes::scope_report::models::FolderRequest; /// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -#[inline] pub fn scope_browse( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: (String, Option>, bool), ) -> PyResult { - scope_reports::scope_browse(py, lookup, request) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = folder_request(&classes, request, None)?; + let native: &NativeScopeLookup = reports.lookup.get(); + let result = browse_scope_folders(reports.source(native), &request); + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(browse_object(&classes, result, &diagnostics)?.unbind()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs index 86db324665..6045c41b99 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs @@ -1,19 +1,52 @@ //! `explain_scope_declaration`: one qualified declaration in an optional context. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + diagnostics_object, explanation_object, read_declaration_target, read_optional_target, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::path_text; +use crate::scopes::_helpers::reports::scope_reports::raised; use crate::scopes::models::ExplainArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; +use sqlbuild_scopes::scope_report::models::ExplainRequest; /// `explain_scope_declaration`: one qualified declaration in an optional context. -#[inline] pub fn scope_explain( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, ) -> PyResult<(PyObject, PyObject)> { - scope_reports::scope_explain(py, lookup, request) + let ExplainArguments { + declaration, + target, + at, + directory, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ExplainRequest = ExplainRequest { + declaration: read_declaration_target(&classes, &declaration)?, + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match explain_scope_declaration(reports.source(native), &request) { + Ok((explanation, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(( + explanation_object(&classes, explanation)?.unbind(), + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs index 7460b26b08..17aee73a7e 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs @@ -1,19 +1,42 @@ //! `list_scope_declarations`: declarations recursively beneath one definition folder. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::list_object; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::folder_request; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; use crate::scopes::models::ListArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; +use sqlbuild_scopes::scope_report::models::FolderRequest; /// `list_scope_declarations`: declarations recursively beneath one definition folder. -#[inline] pub fn scope_list( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, ) -> PyResult { - scope_reports::scope_list(py, lookup, request) + let ListArguments { + folder, + filters, + target, + target_is_prospective, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = + folder_request(&classes, (folder, target, target_is_prospective), filters)?; + let native: &NativeScopeLookup = reports.lookup.get(); + match list_scope_declarations(reports.source(native), &request) { + Ok(result) => { + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(list_object(&classes, result, &diagnostics)?.unbind()) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs index 21dbee2b9e..7df2950f65 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs @@ -1,19 +1,48 @@ //! `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + diagnostics_object, move_object, read_target, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::raised; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; +use sqlbuild_scopes::scope_report::models::MoveRequest; /// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -#[inline] +#[pyfunction] +#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] pub fn scope_move_preview( py: Python<'_>, lookup: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, destination: &Bound<'_, PyAny>, ) -> PyResult<(Option, PyObject)> { - scope_reports::scope_move_preview(py, lookup, resource, destination) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: MoveRequest = MoveRequest { + resource: read_target(&classes, resource)?, + destination: destination.str()?.extract()?, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match preview_scope_move(reports.source(native), &request) { + Ok((preview, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(( + preview + .map(|preview| Ok::<_, PyErr>(move_object(&classes, preview)?.unbind())) + .transpose()?, + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs index ac85c5fe37..c87288db12 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs @@ -1,17 +1,22 @@ //! Scoped declarations the resources at `path` use that resolve differently from `destination`. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_scopes::scope_report::main::rebound_scope_usages::rebound_scope_usages; /// Scoped declarations the resources at `path` use that resolve differently from `destination`. -#[inline] pub fn scope_rebound_usages( py: Python<'_>, lookup: &Bound<'_, PyAny>, paths: (&str, &str), ) -> PyResult>> { - scope_reports::scope_rebound_usages(py, lookup, paths) + let reports: LookupReports = lookup_reports(lookup)?; + let native: &NativeScopeLookup = reports.lookup.get(); + match rebound_scope_usages(reports.source(native), paths) { + Ok(rebound) => Ok(rebound), + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs index 5ad16340fa..a03b663ad1 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs @@ -1,19 +1,53 @@ //! `query_scope_report`: one existing target or prospective path; invalid paths raise. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + read_filters, read_optional_target, report_object, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::path_text; use crate::scopes::models::ReportArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; +use sqlbuild_scopes::scope_report::models::ReportRequest; /// `query_scope_report`: one existing target or prospective path; invalid paths raise. -#[inline] pub fn scope_report( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, ) -> PyResult { - scope_reports::scope_report(py, lookup, request) + let ReportArguments { + target, + at, + directory, + filters, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { + Some(filters) => filters, + None => classes.model(RecordId::ScopeReportFilters)?.call0()?, + }; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ReportRequest = ReportRequest { + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + filters: read_filters(&selected)?, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match build_scope_report(reports.source(native), &request) { + Ok(report) => { + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(report_object(&classes, report, (&selected, &diagnostics))?.unbind()) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs index 91429ea6c6..6949771bb3 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs @@ -1,13 +1,25 @@ //! `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::scope_error; +use crate::scopes::_helpers::reports::scope_reports::serialized; +use crate::scopes::models::ScopeClasses; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::ErrorId; /// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, report))] pub fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { - scope_reports::serialize_scope_report(py, report) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + match serialized(&classes, report) { + Err(error) if error.is_instance_of::(py) => { + let message: String = error.value(py).str()?.extract()?; + let raised: PyErr = scope_error(py, ErrorId::Scope, message)?; + raised.set_cause(py, Some(error)); + Err(raised) + } + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs index e6e78be935..40ff86c4c1 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs @@ -1,10 +1,7 @@ //! Reject authored SQL tests and scenarios whose CTE names cannot be read, at the authored text. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::TextPosition; use sqlbuild_analysis::compiler::main::sql_test_classification::extract_sql_tests; @@ -12,31 +9,14 @@ use sqlbuild_analysis::compiler::models::{SqlTestExtractionFailure, SqlTestReque use sqlbuild_analysis::sql_test_attachment::main::authored_test_offset::authored_test_offset; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; const MODEL_MODE: &str = "model"; -/// Extract fixture bodies from authored test or scenario CTEs in authored order. -pub(crate) fn prefixed_ctes<'py>( - ctes: &Bound<'py, PyAny>, - prefix: &str, -) -> PyResult> { - let result = PyDict::new(ctes.py()); - for cte in ctes.try_iter()? { - let cte = cte?; - let name: String = native_field(&cte, "name")?.extract()?; - if let Some(name) = name.strip_prefix(prefix) { - result.set_item(name, native_field(&cte, "sql_body")?)?; - } - } - Ok(result) -} - /// One authored SQL body to read: its text, file contents, relative path and body span. -struct AuthoredBody { +pub(crate) struct AuthoredBody { sql: String, contents: String, relative_path: String, @@ -46,7 +26,7 @@ struct AuthoredBody { } impl AuthoredBody { - fn read( + pub(crate) fn read( owner: &Bound<'_, PyAny>, sql: &Bound<'_, PyAny>, file: &Bound<'_, PyAny>, @@ -89,7 +69,7 @@ fn located_error(py: Python<'_>, failure: &SqlTestExtractionFailure, body: &Auth } /// Raise the located error of the first body whose authored CTEs cannot be read. -fn check_bodies( +pub(crate) fn check_bodies( py: Python<'_>, bodies: &[AuthoredBody], scenario: bool, @@ -114,41 +94,3 @@ fn check_bodies( Err(failure) => Err(located_error(py, &failure, &bodies[failure.index])), } } - -/// Reject the first authored test block, then scenario, whose CTE names cannot be read. -#[pyfunction] -pub(crate) fn validate_authored_sql_test_ctes( - py: Python<'_>, - test_files: Bound<'_, PyAny>, - scenario_files: Bound<'_, PyAny>, - syntax: LexicalSyntaxInput, -) -> Result<(), PyErr> { - let syntax: LexicalSyntax = syntax.into(); - let mut blocks: Vec = Vec::new(); - for test_file in test_files.try_iter()? { - let test_file: Bound<'_, PyAny> = test_file?; - for block in native_field(&test_file, "blocks")?.try_iter()? { - let block: Bound<'_, PyAny> = block?; - blocks.push(AuthoredBody::read( - &block, - &native_field(&block, "sql_body")?, - &test_file, - )?); - } - } - check_bodies(py, &blocks, false, &syntax)?; - let mut scenarios: Vec = Vec::new(); - for scenario_file in scenario_files.try_iter()? { - let scenario_file: Bound<'_, PyAny> = scenario_file?; - scenarios.push(AuthoredBody::read( - &scenario_file, - &native_field(&scenario_file, "sql_body")?, - &scenario_file, - )?); - } - check_bodies(py, &scenarios, true, &syntax) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validate_authored_sql_test_ctes, module)?) -} diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs index ee70b193f8..2983897917 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs @@ -1,13 +1,18 @@ //! Extract fixture bodies from authored test or scenario CTEs in authored order. -use crate::sql_test_glue::_helpers::authored_ctes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Extract fixture bodies from authored test or scenario CTEs in authored order. -#[inline] pub fn prefixed_ctes<'py>(ctes: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { - authored_ctes::prefixed_ctes(ctes, prefix) + let result = PyDict::new(ctes.py()); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = native_field(&cte, "name")?.extract()?; + if let Some(name) = name.strip_prefix(prefix) { + result.set_item(name, native_field(&cte, "sql_body")?)?; + } + } + Ok(result) } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs index 06b795b1ca..f366bef1ed 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs @@ -1,12 +1,10 @@ //! Register the authored ctes functions and classes. -use crate::sql_test_glue::_helpers::authored_ctes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sql_test_glue::main::validate_authored_sql_test_ctes::validate_authored_sql_test_ctes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the authored ctes functions and classes. -#[inline] pub fn register_authored_ctes(module: &Bound<'_, PyModule>) -> PyResult<()> { - authored_ctes::register(module) + module.add_function(wrap_pyfunction!(validate_authored_sql_test_ctes, module)?) } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs index 4fb8a7c492..ec1f1dd0e2 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs @@ -1,19 +1,43 @@ //! Reject the first authored test block, then scenario, whose CTE names cannot be read. -use crate::sql_test_glue::_helpers::authored_ctes; +use crate::sql_test_glue::_helpers::authored_ctes::AuthoredBody; +use crate::sql_test_glue::_helpers::authored_ctes::check_bodies; use crate::sqltext::models::LexicalSyntaxInput; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Reject the first authored test block, then scenario, whose CTE names cannot be read. -#[inline] +#[pyfunction] pub fn validate_authored_sql_test_ctes( py: Python<'_>, test_files: Bound<'_, PyAny>, scenario_files: Bound<'_, PyAny>, syntax: LexicalSyntaxInput, ) -> Result<(), PyErr> { - authored_ctes::validate_authored_sql_test_ctes(py, test_files, scenario_files, syntax) + let syntax: LexicalSyntax = syntax.into(); + let mut blocks: Vec = Vec::new(); + for test_file in test_files.try_iter()? { + let test_file: Bound<'_, PyAny> = test_file?; + for block in native_field(&test_file, "blocks")?.try_iter()? { + let block: Bound<'_, PyAny> = block?; + blocks.push(AuthoredBody::read( + &block, + &native_field(&block, "sql_body")?, + &test_file, + )?); + } + } + check_bodies(py, &blocks, false, &syntax)?; + let mut scenarios: Vec = Vec::new(); + for scenario_file in scenario_files.try_iter()? { + let scenario_file: Bound<'_, PyAny> = scenario_file?; + scenarios.push(AuthoredBody::read( + &scenario_file, + &native_field(&scenario_file, "sql_body")?, + &scenario_file, + )?); + } + check_bodies(py, &scenarios, true, &syntax) } diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs index d0fbd143e8..cc9c278cba 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs @@ -1,13 +1,15 @@ //! Authored header values as the Python header objects: scalars, sets, constants and hooks. +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use crate::sqltext::main::map_to_python::map_to_python; use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; -fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { +pub(crate) fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { match header_word_value(word) { HeaderWord::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()), HeaderWord::Null => Ok(py.None()), @@ -24,7 +26,7 @@ fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { } } -fn instance( +pub(crate) fn instance( py: Python<'_>, class: RecordId, arguments: &pyo3::Bound<'_, PyDict>, @@ -32,7 +34,7 @@ fn instance( Ok(new_record(py, class, (), Some(arguments))?.unbind()) } -fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { +pub(crate) fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { let projected = values .iter() .map(|item| authored_value_to_python(py, item)) @@ -40,7 +42,10 @@ fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> Ok(PyTuple::new(py, projected)?.unbind().into_any()) } -fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { +pub(crate) fn constant_call( + py: Python<'_>, + values: &[(String, AuthoredValue)], +) -> PyResult> { let arguments = values .iter() .map(|(key, value)| { @@ -53,59 +58,7 @@ fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult instance(py, RecordId::AuthoredSqlValueCall, &keywords) } -/// One authored value as the Python header object discovery hands to config builds. -pub(crate) fn authored_value_to_python( - py: Python<'_>, - value: &AuthoredValue, -) -> PyResult> { - match value { - AuthoredValue::Null => Ok(py.None()), - AuthoredValue::Boolean(value) => { - Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) - } - AuthoredValue::BareWord(value) => word_to_python(py, value), - AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), - AuthoredValue::List(values) => { - let projected = values - .iter() - .map(|item| authored_value_to_python(py, item)) - .collect::>>()?; - Ok(PyList::new(py, projected)?.unbind().into_any()) - } - AuthoredValue::Map(values) => map_to_python(py, values), - AuthoredValue::Set(values) => { - let keywords = PyDict::new(py); - keywords.set_item("values", values_tuple(py, values)?)?; - instance(py, RecordId::AuthoredSqlSet, &keywords) - } - AuthoredValue::Tuple(values) => values_tuple(py, values), - AuthoredValue::TypedConstant(values) => constant_call(py, values), - AuthoredValue::InlineSqlHook(statement) => { - let keywords = PyDict::new(py); - keywords.set_item("statement", statement)?; - instance(py, RecordId::SqlHookEntry, &keywords) - } - AuthoredValue::NamedSqlHook(name, kwargs) => { - hook(py, RecordId::NamedSqlHookEntry, name, kwargs) - } - AuthoredValue::PythonHook(name, kwargs) => { - hook(py, RecordId::PythonHookEntry, name, kwargs) - } - } -} - -pub(crate) fn map_to_python( - py: Python<'_>, - values: &[(String, AuthoredValue)], -) -> PyResult> { - let result = PyDict::new(py); - for (key, value) in values { - result.set_item(key, authored_value_to_python(py, value)?)?; - } - Ok(result.unbind().into_any()) -} - -fn hook( +pub(crate) fn hook( py: Python<'_>, class: RecordId, name: &str, @@ -116,12 +69,3 @@ fn hook( keywords.set_item("kwargs", map_to_python(py, kwargs)?)?; instance(py, class, &keywords) } - -pub(crate) fn optional_authored_value_to_python( - py: Python<'_>, - value: Option<&AuthoredValue>, -) -> PyResult>> { - value - .map(|item| authored_value_to_python(py, item)) - .transpose() -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs deleted file mode 100644 index 948a471284..0000000000 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. - -use crate::sqltext::models::SqlCodeScanner; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs index c0ec6dba04..d5c71d554b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs @@ -1,9 +1,9 @@ //! Executed-statement effects on the executor's cached relation metadata. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_sqltext::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key as native_qualified_relation_name_key; use sqlbuild_sqltext::ddl_effects::main::relation_name_key::relation_name_key as native_relation_name_key; @@ -20,7 +20,10 @@ static EFFECT_CLASS: PyOnceLock> = PyOnceLock::new(); /// Which cached relation names `sql` may have changed, as a `StatementMetadataEffect`. #[pyfunction] #[pyo3(signature = (sql, /))] -fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult> { +pub(crate) fn statement_metadata_effect<'py>( + py: Python<'py>, + sql: &str, +) -> PyResult> { let effect: StatementMetadataEffect = py.detach(|| native_statement_metadata_effect(sql)); let class: &Bound<'py, PyAny> = EFFECT_CLASS .get_or_try_init(py, || host_entry(py, EFFECT_MODEL).map(Bound::unbind))? @@ -51,20 +54,13 @@ fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult String { +pub(crate) fn relation_name_key(name: &str) -> String { native_relation_name_key(name) } /// The key for the last part of a qualified name, or `None` when it cannot be parsed. #[pyfunction] #[pyo3(signature = (qualified, /))] -fn qualified_relation_name_key(qualified: &str) -> Option { +pub(crate) fn qualified_relation_name_key(qualified: &str) -> Option { native_qualified_relation_name_key(qualified) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; - module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; - module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs index a31739eb01..d3f0ff8da0 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs @@ -1,9 +1,7 @@ //! `@enum`/`@const` reference offsets scanned natively for the model loop. use crate::sqltext::models::ScannedReference; -use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; -use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, @@ -16,19 +14,12 @@ const SQL_MACRO_REFERENCE_KIND: u8 = 2; /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. pub(crate) type ScanRow = (Vec, Option); -/// One SQL string's references and stop code, scanned without walking past malformed ones. -pub(crate) fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { - let scanned: Vec = - scan_declaration_references(&[sql.to_owned()], false, syntax); - let row: ScanRow = scanned.into_iter().next().map(scan_row).unwrap_or_default(); - if !sql.contains(SQL_CALL_PREFIX) { - return row; - } - with_sql_macro_calls(sql, row, syntax) -} - /// The scan with each outer SQL macro call as one kind-2 reference, replacing those inside it. -fn with_sql_macro_calls(sql: &str, row: ScanRow, syntax: Option<&LexicalSyntax>) -> ScanRow { +pub(crate) fn with_sql_macro_calls( + sql: &str, + row: ScanRow, + syntax: Option<&LexicalSyntax>, +) -> ScanRow { let (references, stop) = row; let Ok(calls) = sql_macro_calls(sql, syntax) else { return (references, stop); @@ -66,7 +57,7 @@ fn with_sql_macro_calls(sql: &str, row: ScanRow, syntax: Option<&LexicalSyntax>) (merged, stop) } -fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { +pub(crate) fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ( scan.references.into_iter().map(reference_row).collect(), scan.stop.map(|stop| match stop { diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs index 200ce9e1eb..1881fce178 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs @@ -1,14 +1,8 @@ //! One adapter's `SqlLexicalSyntax`, read from a Python mapping. use crate::sqltext::models::LexicalSyntaxInput; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyFrozenSet; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - impl From for LexicalSyntax { fn from(input: LexicalSyntaxInput) -> Self { Self { @@ -22,63 +16,3 @@ impl From for LexicalSyntax { } } } - -/// The sorted texts of a set of strings. -pub(crate) fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { - let mut texts: Vec = values - .try_iter()? - .map(|value| value?.extract::()) - .collect::>()?; - texts.sort_unstable(); - Ok(texts) -} - -/// The lexical rules a `SqlLexicalSyntax` record holds. -pub(crate) fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { - let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; - Ok(LexicalSyntax { - backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, - escape_string_prefix: flag("escape_string_prefix")?, - raw_string_prefix: flag("raw_string_prefix")?, - triple_quoted_strings: flag("triple_quoted_strings")?, - nested_block_comments: flag("nested_block_comments")?, - line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, - bracket_identifiers: false, - }) -} - -/// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -pub(crate) fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - record_syntax(value).map(Some) -} - -/// A `SqlLexicalSyntax` record holding `syntax`. -pub(crate) fn syntax_record<'py>( - py: Python<'py>, - syntax: &LexicalSyntax, -) -> PyResult> { - let flag = |value: bool| -> PyResult> { - Ok(pyo3::types::PyBool::new(py, value).to_owned().into_any()) - }; - new_record_with( - py, - RecordId::SqlLexicalSyntax, - &[ - ( - "backslash_escape_quotes", - PyFrozenSet::new(py, &syntax.backslash_escape_quotes)?.into_any(), - ), - ("escape_string_prefix", flag(syntax.escape_string_prefix)?), - ("raw_string_prefix", flag(syntax.raw_string_prefix)?), - ("triple_quoted_strings", flag(syntax.triple_quoted_strings)?), - ("nested_block_comments", flag(syntax.nested_block_comments)?), - ( - "line_comment_prefixes", - PyFrozenSet::new(py, &syntax.line_comment_prefixes)?.into_any(), - ), - ], - ) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs index 4a2ee8426a..97b4ef2d10 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod authored_values; -pub(crate) mod code_scanner; pub(crate) mod ddl_effects; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs index f8c8ecd8c9..56ec8fd04b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs @@ -1,71 +1,15 @@ //! Match, tokenize and parse model headers natively. +use crate::sqltext::main::location_rows::location_rows; use crate::sqltext::models::ColumnLocation; -use crate::sqltext::types::ParsedModelHeader; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::PyResult; +use pyo3::pyfunction; -use crate::sqltext::_helpers::authored_values::optional_authored_value_to_python; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::types::CompilerDetach; - -#[pyfunction] -pub(crate) fn parse_model_headers( - py: Python<'_>, - headers: Vec, -) -> PyResult> { - let parsed = py - .compiler_detach(|| { - sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) - }) - .map_err(value_error)?; - parsed - .into_iter() - .map(|(value, offsets, error)| { - Ok(( - optional_authored_value_to_python(py, value.as_ref())?, - offsets, - error, - )) - }) - .collect() -} - -pub(crate) fn location_rows( - locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, -) -> Vec { - locations - .into_iter() - .map(|(name, location)| ColumnLocation { - name, - line: location.line, - column: location.column, - end_line: location.end_line, - end_column: location.end_column, - }) - .collect() -} - -/// Simple top-level SELECT output names and their authored locations. -#[pyfunction] -pub(crate) fn model_output_columns( - contents: &str, - implicit_aliases: bool, -) -> PyResult> { - compiler_guard(|| { - Ok(location_rows( - sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( - contents, - implicit_aliases, - ), - )) - }) -} /// The authored locations of a header's column names, or the header's syntax error. #[pyfunction] -fn model_header_column_locations( +pub(crate) fn model_header_column_locations( contents: &str, header: &str, header_start: usize, @@ -83,39 +27,3 @@ fn model_header_column_locations( ) }) } - -/// A header syntax error's detail, help and file line (when `header_line` is known). -pub(crate) fn header_failure( - error: &str, - header: &str, - header_line: Option, -) -> (String, Option, Option) { - let nesting = sqlbuild_sqltext::compiler::main::model_header_errors::header_nesting_failure( - error, - header, - header_line.unwrap_or(1), - ); - if let Some(quote) = sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure( - error, - header, - (header_line.unwrap_or(1), 1), - ) { - return ( - quote.message, - Some(quote.help), - header_line.map(|_| quote.line), - ); - } - match (nesting, header_line) { - (Some(nesting), Some(_)) => (nesting.message, Some(nesting.help), Some(nesting.line)), - (Some(nesting), None) => (error.to_owned(), Some(nesting.help), None), - (None, _) => (error.to_owned(), None, None), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; - module.add_function(wrap_pyfunction!(model_output_columns, module)?)?; - module.add_function(wrap_pyfunction!(model_header_column_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs index 5ae54c3196..d44a0feb5e 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs @@ -1,7 +1,7 @@ //! Substitute `@@@name` placeholder defaults for SQL analysis. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -9,7 +9,7 @@ use sqlbuild_python_base::types::CompilerDetach; /// Python's `substitute_placeholder_defaults`: each `@@@name` becomes its default. #[pyfunction] -fn substitute_placeholder_defaults( +pub(crate) fn substitute_placeholder_defaults( py: Python<'_>, sql: &str, placeholders: Vec<(String, String)>, @@ -17,8 +17,3 @@ fn substitute_placeholder_defaults( py.compiler_detach(|| Ok(placeholder_defaults(sql, &placeholders))) .map_err(compiler_error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(substitute_placeholder_defaults, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs index c45eecade0..1caa7ef112 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs @@ -1,10 +1,8 @@ //! Logical SQL references extracted natively under one adapter's lexical rules. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods}; use sqlbuild_python_base::positional_record; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::{ @@ -14,16 +12,14 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; +pub(crate) const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; positional_record! { into; @@ -113,12 +109,15 @@ fn invalid_call_row(call: InvalidReferenceCall) -> InvalidCallRow { } } -fn text<'py>(py: Python<'py>, value: impl IntoPyObject<'py>) -> PyResult> { +pub(crate) fn text<'py>( + py: Python<'py>, + value: impl IntoPyObject<'py>, +) -> PyResult> { value.into_bound_py_any(py) } /// The references and rejected calls `sql` holds as Python objects; a scan error raises. -fn scanned<'py>( +pub(crate) fn scanned<'py>( py: Python<'py>, sql: &str, syntax: &Bound<'py, PyAny>, @@ -182,7 +181,7 @@ fn scanned<'py>( /// Valid references and rejected calls; a scan error raises unlocated. #[pyfunction] #[pyo3(signature = (*, sql, syntax))] -fn scan_sql_reference_calls<'py>( +pub(crate) fn scan_sql_reference_calls<'py>( sql: &str, syntax: &Bound<'py, PyAny>, ) -> PyResult> { @@ -197,47 +196,3 @@ fn scan_sql_reference_calls<'py>( ], ) } - -/// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -#[pyfunction(name = "extract_sql_references")] -#[pyo3(signature = (*, sql, syntax))] -pub(crate) fn extract_references<'py>( - sql: &str, - syntax: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = syntax.py(); - let (references, invalid_calls) = scanned(py, sql, syntax)?; - for call in invalid_calls { - let diagnostic: Bound<'py, PyAny> = new_record_with( - py, - RecordId::CompilerDiagnostic, - &[ - ( - "phase", - enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?, - ), - ( - "severity", - enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?, - ), - ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), - ("message", native_field(&call, "message")?), - ("help", native_field(&call, "help")?), - ], - )?; - report_compile_diagnostic( - (REFERENCE_CALL_SYNTAX_CODE, "", native_field(&call, "call")?) - .into_pyobject(py)? - .as_any(), - &diagnostic, - )?; - } - PyTuple::new(py, references) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(scan_sql_reference_calls, module)?)?; - module.add_function(wrap_pyfunction!(extract_references, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs index 1e37cd63c7..4aa3789808 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs @@ -1,25 +1,23 @@ //! Interpolate `@@` tokens in authored SQL and extract static SQL references. -use crate::sqltext::models::InterpolationRow; use crate::sqltext::models::InterpolationSources; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::cell::RefCell; use pyo3::PyErr; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; -use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -struct PythonInterpolationHost<'py> { - sources: InterpolationSources<'py>, +pub(crate) struct PythonInterpolationHost<'py> { + pub(crate) sources: InterpolationSources<'py>, /// A Python exception other than a rendering `ValueError`, raised after the scan. - raised: RefCell>, + pub(crate) raised: RefCell>, } impl PythonInterpolationHost<'_> { @@ -132,63 +130,10 @@ fn key_text(key: &Bound<'_, PyAny>) -> String { key.str().map(|text| text.to_string()).unwrap_or_default() } -/// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -pub(crate) fn interpolation_batch( - sources: InterpolationSources<'_>, - sqls: &[(String, String)], -) -> PyResult> { - let host = PythonInterpolationHost { - sources, - raised: RefCell::new(None), - }; - let mut rows = Vec::with_capacity(sqls.len()); - for (sql, file_path) in sqls { - let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => InterpolationRow { - sql: result.sql, - spans: result.spans, - reads: read_rows(result.reads), - error: None, - }, - Err(failure) => InterpolationRow { - sql: None, - spans: Vec::new(), - reads: read_rows(failure.reads), - error: Some(failure.message), - }, - }; - if let Some(error) = host.raised.borrow_mut().take() { - return Err(error); - } - rows.push(row); - } - Ok(rows) -} - /// `(sql or None when unchanged, reads, error message or None)` for one SQL text. pub(crate) type InterpolatedRow = (Option, Vec<(&'static str, String)>, Option); -/// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -pub(crate) fn interpolated( - sources: InterpolationSources<'_>, - sql: &str, - file_path: &str, -) -> PyResult { - let host = PythonInterpolationHost { - sources, - raised: RefCell::new(None), - }; - let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => (result.sql, read_rows(result.reads), None), - Err(failure) => (None, read_rows(failure.reads), Some(failure.message)), - }; - if let Some(error) = host.raised.borrow_mut().take() { - return Err(error); - } - Ok(row) -} - -fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { +pub(crate) fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { reads .into_iter() .map(|read| match read { diff --git a/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs index 4b26dde9b3..7b51d0d16d 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs @@ -1,14 +1,50 @@ //! One authored value as the Python header object discovery hands to config builds. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::_helpers::authored_values::constant_call; +use crate::sqltext::_helpers::authored_values::hook; +use crate::sqltext::_helpers::authored_values::instance; +use crate::sqltext::_helpers::authored_values::values_tuple; +use crate::sqltext::_helpers::authored_values::word_to_python; +use crate::sqltext::main::map_to_python::map_to_python; +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// One authored value as the Python header object discovery hands to config builds. -#[inline] pub fn authored_value_to_python(py: Python<'_>, value: &AuthoredValue) -> PyResult> { - authored_values::authored_value_to_python(py, value) + match value { + AuthoredValue::Null => Ok(py.None()), + AuthoredValue::Boolean(value) => { + Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) + } + AuthoredValue::BareWord(value) => word_to_python(py, value), + AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), + AuthoredValue::List(values) => { + let projected = values + .iter() + .map(|item| authored_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, projected)?.unbind().into_any()) + } + AuthoredValue::Map(values) => map_to_python(py, values), + AuthoredValue::Set(values) => { + let keywords = PyDict::new(py); + keywords.set_item("values", values_tuple(py, values)?)?; + instance(py, RecordId::AuthoredSqlSet, &keywords) + } + AuthoredValue::Tuple(values) => values_tuple(py, values), + AuthoredValue::TypedConstant(values) => constant_call(py, values), + AuthoredValue::InlineSqlHook(statement) => { + let keywords = PyDict::new(py); + keywords.set_item("statement", statement)?; + instance(py, RecordId::SqlHookEntry, &keywords) + } + AuthoredValue::NamedSqlHook(name, kwargs) => { + hook(py, RecordId::NamedSqlHookEntry, name, kwargs) + } + AuthoredValue::PythonHook(name, kwargs) => { + hook(py, RecordId::PythonHookEntry, name, kwargs) + } + } } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs index 78b1018b7c..5d5685a429 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs @@ -1,11 +1,20 @@ //! One SQL string's references and stop code, scanned without walking past malformed ones. -use crate::sqltext::_helpers::declaration_references; use crate::sqltext::_helpers::declaration_references::ScanRow; +use crate::sqltext::_helpers::declaration_references::scan_row; +use crate::sqltext::_helpers::declaration_references::with_sql_macro_calls; +use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; +use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; +use sqlbuild_sqltext::compiler::models::DeclarationReferenceScan; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// One SQL string's references and stop code, scanned without walking past malformed ones. -#[inline] pub fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { - declaration_references::declaration_scan(sql, syntax) + let scanned: Vec = + scan_declaration_references(&[sql.to_owned()], false, syntax); + let row: ScanRow = scanned.into_iter().next().map(scan_row).unwrap_or_default(); + if !sql.contains(SQL_CALL_PREFIX) { + return row; + } + with_sql_macro_calls(sql, row, syntax) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs index dd8e5d84b4..d735a9afa7 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs @@ -1,16 +1,50 @@ //! Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -use crate::sqltext::_helpers::sql_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::sql_references::REFERENCE_CALL_SYNTAX_CODE; +use crate::sqltext::_helpers::sql_references::scanned; +use crate::sqltext::_helpers::sql_references::text; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -#[inline] +#[pyfunction(name = "extract_sql_references")] +#[pyo3(signature = (*, sql, syntax))] pub fn extract_references<'py>( sql: &str, syntax: &Bound<'py, PyAny>, ) -> PyResult> { - sql_references::extract_references(sql, syntax) + let py: Python<'py> = syntax.py(); + let (references, invalid_calls) = scanned(py, sql, syntax)?; + for call in invalid_calls { + let diagnostic: Bound<'py, PyAny> = new_record_with( + py, + RecordId::CompilerDiagnostic, + &[ + ( + "phase", + enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?, + ), + ( + "severity", + enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?, + ), + ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), + ("message", native_field(&call, "message")?), + ("help", native_field(&call, "help")?), + ], + )?; + report_compile_diagnostic( + (REFERENCE_CALL_SYNTAX_CODE, "", native_field(&call, "call")?) + .into_pyobject(py)? + .as_any(), + &diagnostic, + )?; + } + PyTuple::new(py, references) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs index 3a0a42ef4a..6b50d2fab8 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs @@ -1,13 +1,30 @@ //! A header syntax error's detail, help and file line (when `header_line` is known). -use crate::sqltext::_helpers::model_headers; - /// A header syntax error's detail, help and file line (when `header_line` is known). -#[inline] pub fn header_failure( error: &str, header: &str, header_line: Option, ) -> (String, Option, Option) { - model_headers::header_failure(error, header, header_line) + let nesting = sqlbuild_sqltext::compiler::main::model_header_errors::header_nesting_failure( + error, + header, + header_line.unwrap_or(1), + ); + if let Some(quote) = sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure( + error, + header, + (header_line.unwrap_or(1), 1), + ) { + return ( + quote.message, + Some(quote.help), + header_line.map(|_| quote.line), + ); + } + match (nesting, header_line) { + (Some(nesting), Some(_)) => (nesting.message, Some(nesting.help), Some(nesting.line)), + (Some(nesting), None) => (error.to_owned(), Some(nesting.help), None), + (None, _) => (error.to_owned(), None, None), + } } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs index 37f5d41c82..418249e763 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs @@ -1,16 +1,29 @@ //! One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -use crate::sqltext::_helpers::static_sql; use crate::sqltext::_helpers::static_sql::InterpolatedRow; +use crate::sqltext::_helpers::static_sql::PythonInterpolationHost; +use crate::sqltext::_helpers::static_sql::read_rows; use crate::sqltext::models::InterpolationSources; use pyo3::prelude::PyResult; +use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; +use std::cell::RefCell; /// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -#[inline] pub fn interpolated( sources: InterpolationSources<'_>, sql: &str, file_path: &str, ) -> PyResult { - static_sql::interpolated(sources, sql, file_path) + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let row = match interpolate_sql(&host, sql, file_path) { + Ok(result) => (result.sql, read_rows(result.reads), None), + Err(failure) => (None, read_rows(failure.reads), Some(failure.message)), + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + Ok(row) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs index 5c13ea9fdf..3156622ff2 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs @@ -1,15 +1,42 @@ //! Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -use crate::sqltext::_helpers::static_sql; +use crate::sqltext::_helpers::static_sql::PythonInterpolationHost; +use crate::sqltext::_helpers::static_sql::read_rows; use crate::sqltext::models::InterpolationRow; use crate::sqltext::models::InterpolationSources; use pyo3::prelude::PyResult; +use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; +use std::cell::RefCell; /// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -#[inline] pub fn interpolation_batch( sources: InterpolationSources<'_>, sqls: &[(String, String)], ) -> PyResult> { - static_sql::interpolation_batch(sources, sqls) + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let mut rows = Vec::with_capacity(sqls.len()); + for (sql, file_path) in sqls { + let row = match interpolate_sql(&host, sql, file_path) { + Ok(result) => InterpolationRow { + sql: result.sql, + spans: result.spans, + reads: read_rows(result.reads), + error: None, + }, + Err(failure) => InterpolationRow { + sql: None, + spans: Vec::new(), + reads: read_rows(failure.reads), + error: Some(failure.message), + }, + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + rows.push(row); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs index 5788549395..bb42cb86fd 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs @@ -1,12 +1,19 @@ //! Location rows. -use crate::sqltext::_helpers::model_headers; use crate::sqltext::models::ColumnLocation; /// Location rows. -#[inline] pub fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { - model_headers::location_rows(locations) + locations + .into_iter() + .map(|(name, location)| ColumnLocation { + name, + line: location.line, + column: location.column, + end_line: location.end_line, + end_column: location.end_column, + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs index eb9e38b163..46add43371 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs @@ -1,14 +1,15 @@ //! Map to python. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// Map to python. -#[inline] pub fn map_to_python(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { - authored_values::map_to_python(py, values) + let result = PyDict::new(py); + for (key, value) in values { + result.set_item(key, authored_value_to_python(py, value)?)?; + } + Ok(result.unbind().into_any()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs index 3c8c84f92b..aa07c51d5b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs @@ -1,14 +1,23 @@ //! Simple top-level SELECT output names and their authored locations. -use crate::sqltext::_helpers::model_headers; +use crate::sqltext::main::location_rows::location_rows; use crate::sqltext::models::ColumnLocation; use pyo3::prelude::PyResult; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// Simple top-level SELECT output names and their authored locations. -#[inline] +#[pyfunction] pub fn model_output_columns( contents: &str, implicit_aliases: bool, ) -> PyResult> { - model_headers::model_output_columns(contents, implicit_aliases) + compiler_guard(|| { + Ok(location_rows( + sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( + contents, + implicit_aliases, + ), + )) + }) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs index 6385780a54..01678d05eb 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs @@ -1,17 +1,15 @@ //! Optional authored value to python. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// Optional authored value to python. -#[inline] pub fn optional_authored_value_to_python( py: Python<'_>, value: Option<&AuthoredValue>, ) -> PyResult>> { - authored_values::optional_authored_value_to_python(py, value) + value + .map(|item| authored_value_to_python(py, item)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs index d329f2acc3..3ae3df6734 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs @@ -1,13 +1,13 @@ //! The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::main::record_syntax::record_syntax; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -#[inline] pub fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { - lexical_syntax::optional_record_syntax(value) + if value.is_none() { + return Ok(None); + } + record_syntax(value).map(Some) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs index 068fe6c526..cecbed086b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs @@ -1,15 +1,31 @@ //! Parse model headers. -use crate::sqltext::_helpers::model_headers; +use crate::sqltext::main::optional_authored_value_to_python::optional_authored_value_to_python; use crate::sqltext::types::ParsedModelHeader; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; /// Parse model headers. -#[inline] +#[pyfunction] pub fn parse_model_headers( py: Python<'_>, headers: Vec, ) -> PyResult> { - model_headers::parse_model_headers(py, headers) + let parsed = py + .compiler_detach(|| { + sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) + }) + .map_err(value_error)?; + parsed + .into_iter() + .map(|(value, offsets, error)| { + Ok(( + optional_authored_value_to_python(py, value.as_ref())?, + offsets, + error, + )) + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs index a990fdc113..9ca9b52207 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs @@ -1,13 +1,20 @@ //! The lexical rules a `SqlLexicalSyntax` record holds. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::main::sorted_texts::sorted_texts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The lexical rules a `SqlLexicalSyntax` record holds. -#[inline] pub fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { - lexical_syntax::record_syntax(instance) + let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; + Ok(LexicalSyntax { + backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, + escape_string_prefix: flag("escape_string_prefix")?, + raw_string_prefix: flag("raw_string_prefix")?, + triple_quoted_strings: flag("triple_quoted_strings")?, + nested_block_comments: flag("nested_block_comments")?, + line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, + bracket_identifiers: false, + }) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs index f70b7bc051..de3c31c66e 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs @@ -1,12 +1,10 @@ //! Register the code scanner functions and classes. -use crate::sqltext::_helpers::code_scanner; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::models::SqlCodeScanner; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the code scanner functions and classes. -#[inline] pub fn register_code_scanner(module: &Bound<'_, PyModule>) -> PyResult<()> { - code_scanner::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs index 6ab4c7d18f..96941c4f62 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs @@ -1,12 +1,15 @@ //! Register the ddl effects functions and classes. -use crate::sqltext::_helpers::ddl_effects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::ddl_effects::qualified_relation_name_key; +use crate::sqltext::_helpers::ddl_effects::relation_name_key; +use crate::sqltext::_helpers::ddl_effects::statement_metadata_effect; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the ddl effects functions and classes. -#[inline] pub fn register_ddl_effects(module: &Bound<'_, PyModule>) -> PyResult<()> { - ddl_effects::register(module) + module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; + module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; + module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs index abf259e94a..a63bf0c341 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs @@ -1,12 +1,15 @@ //! Register the model headers functions and classes. -use crate::sqltext::_helpers::model_headers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::model_headers::model_header_column_locations; +use crate::sqltext::main::model_output_columns::model_output_columns; +use crate::sqltext::main::parse_model_headers::parse_model_headers; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model headers functions and classes. -#[inline] pub fn register_model_headers(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_headers::register(module) + module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; + module.add_function(wrap_pyfunction!(model_output_columns, module)?)?; + module.add_function(wrap_pyfunction!(model_header_column_locations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs index e49345fc1b..264936f90b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs @@ -1,12 +1,11 @@ //! Register the placeholders functions and classes. -use crate::sqltext::_helpers::placeholders; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::placeholders::substitute_placeholder_defaults; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the placeholders functions and classes. -#[inline] pub fn register_placeholders(module: &Bound<'_, PyModule>) -> PyResult<()> { - placeholders::register(module) + module.add_function(wrap_pyfunction!(substitute_placeholder_defaults, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs index 98649fadff..3c5370803d 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs @@ -1,12 +1,15 @@ //! Register the sql references functions and classes. -use crate::sqltext::_helpers::sql_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::sql_references::SqlReferenceScanner; +use crate::sqltext::_helpers::sql_references::scan_sql_reference_calls; +use crate::sqltext::main::extract_references::extract_references; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql references functions and classes. -#[inline] pub fn register_sql_references(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_references::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(scan_sql_reference_calls, module)?)?; + module.add_function(wrap_pyfunction!(extract_references, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs index f16064c9b8..7dfb314ad6 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs @@ -1,12 +1,13 @@ //! The sorted texts of a set of strings. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The sorted texts of a set of strings. -#[inline] pub fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { - lexical_syntax::sorted_texts(values) + let mut texts: Vec = values + .try_iter()? + .map(|value| value?.extract::()) + .collect::>()?; + texts.sort_unstable(); + Ok(texts) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs index 6cc6dfdb00..b06b0fd7dd 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs @@ -1,14 +1,32 @@ //! A `SqlLexicalSyntax` record holding `syntax`. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// A `SqlLexicalSyntax` record holding `syntax`. -#[inline] pub fn syntax_record<'py>(py: Python<'py>, syntax: &LexicalSyntax) -> PyResult> { - lexical_syntax::syntax_record(py, syntax) + let flag = |value: bool| -> PyResult> { + Ok(pyo3::types::PyBool::new(py, value).to_owned().into_any()) + }; + new_record_with( + py, + RecordId::SqlLexicalSyntax, + &[ + ( + "backslash_escape_quotes", + PyFrozenSet::new(py, &syntax.backslash_escape_quotes)?.into_any(), + ), + ("escape_string_prefix", flag(syntax.escape_string_prefix)?), + ("raw_string_prefix", flag(syntax.raw_string_prefix)?), + ("triple_quoted_strings", flag(syntax.triple_quoted_strings)?), + ("nested_block_comments", flag(syntax.nested_block_comments)?), + ( + "line_comment_prefixes", + PyFrozenSet::new(py, &syntax.line_comment_prefixes)?.into_any(), + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs index 58a33d0726..b90aab4107 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs @@ -1,11 +1,11 @@ //! Migration event IDs and the typed literals of one appended event row. +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_state::events::errors::StateSqlError; @@ -16,8 +16,6 @@ use sqlbuild_state::events::main::migration_event_id::migration_event_id; use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; use sqlbuild_state::events::models::{EventRelation, StateCell, StateColumnType, StateValue}; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,7 +35,7 @@ pub(crate) fn relation(value: &Bound<'_, PyAny>) -> PyResult { /// The content-addressed ID for one migration performed by one run. #[pyfunction(name = "deterministic_migration_event_id")] #[pyo3(signature = (*, run_id, target_name, origin, destination))] -fn migration_id( +pub(crate) fn migration_id( run_id: &str, target_name: Option<&str>, origin: &Bound<'_, PyAny>, @@ -54,7 +52,7 @@ fn migration_id( /// The content-addressed ID for one column rename performed by one run. #[pyfunction(name = "deterministic_column_migration_event_id")] #[pyo3(signature = (**request))] -fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("deterministic_column_migration_event_id", request)?; let run_id: String = request.get("run_id")?.extract()?; let target_name: Option = request.get("target_name")?.extract()?; @@ -71,7 +69,10 @@ fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { /// The content-addressed ID of one old-name step of one recorded move. #[pyfunction(name = "deterministic_old_name_view_event_id")] #[pyo3(signature = (*, event_type, migration_event_id))] -fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyResult { +pub(crate) fn old_name_id( + event_type: &Bound<'_, PyAny>, + migration_event_id: &str, +) -> PyResult { let event_type: String = native_field(event_type, "value")?.extract()?; old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) } @@ -119,10 +120,3 @@ fn rendering_error(py: Python<'_>) -> impl Fn(StateSqlError) -> PyErr + '_ { } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_id, module)?)?; - module.add_function(wrap_pyfunction!(column_id, module)?)?; - module.add_function(wrap_pyfunction!(old_name_id, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs index e3e00b6dba..17f991f70a 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs @@ -1,41 +1,27 @@ //! Source freshness data versions, their hashes, skip equivalence and age policy. +use crate::state_rows::main::event_rows::in_utc::in_utc; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use std::cell::RefCell; use pyo3::exceptions::PyOverflowError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::LocalDateTime; use sqlbuild_state::freshness::errors::FreshnessError; -use sqlbuild_state::freshness::main::age_status::age_status; -use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; -use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; -use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{ - AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, -}; +use sqlbuild_state::freshness::models::{ComparableRecord, FreshnessEventIdentity}; use sqlbuild_state::freshness::types::TimestampParser; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::main::values::local_date_time::local_date_time; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::event_rows::timestamps::in_utc; -use crate::state_rows::_helpers::event_rows::timestamps::offset_aware; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; -use sqlbuild_python_base::records::types::RecordId; -const TIMESTAMP_KIND: &str = "timestamp"; -const INTEGER_KIND: &str = "integer"; -const STRING_KIND: &str = "string"; +pub(crate) const TIMESTAMP_KIND: &str = "timestamp"; +pub(crate) const INTEGER_KIND: &str = "integer"; +pub(crate) const STRING_KIND: &str = "string"; /// Normalize timestamp offsets and precision before computing a persisted observation identity. pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { @@ -71,14 +57,14 @@ pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { )) } -fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { +pub(crate) fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match native_attr(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } } -fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { +pub(crate) fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { match error { FreshnessError::Observation(message) => { raised(py, NativeAttr::SourceFreshnessObservationError, message) @@ -90,20 +76,7 @@ fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { } } -/// Hash the stable source freshness identity used as a graph input. -#[pyfunction(name = "source_freshness_data_version_hash")] -#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] -pub(crate) fn version_hash( - source_name: &str, - strategy: &str, - value_kind: &str, - data_version: &str, -) -> PyResult { - data_version_hash((source_name, strategy), value_kind, data_version) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) -} - -fn observation_error(py: Python<'_>, message: &str) -> PyErr { +pub(crate) fn observation_error(py: Python<'_>, message: &str) -> PyErr { raised( py, NativeAttr::SourceFreshnessObservationError, @@ -111,62 +84,10 @@ fn observation_error(py: Python<'_>, message: &str) -> PyErr { ) } -/// A source freshness value normalized for stable state storage. -#[pyfunction(name = "normalize_source_freshness_data_version")] -#[pyo3(signature = (*, value, value_kind))] -pub(crate) fn normalize<'py>( - value: &Bound<'py, PyAny>, - value_kind: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = value.py(); - if value_kind.eq(TIMESTAMP_KIND)? { - if !date_time_typed(value)? { - return Err(observation_error( - py, - "Timestamp source freshness values must be datetime values", - )); - } - let aware: bool = !native_field(value, "tzinfo")?.is_none(); - let local: LocalDateTime = if aware { - local_date_time(&in_utc(value)?)? - } else { - local_date_time(value)? - }; - let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); - return Ok(PyString::new(py, &text).into_any()); - } - if value_kind.eq(INTEGER_KIND)? { - if value.is_instance_of::() || !value.is_instance_of::() { - return Err(observation_error( - py, - "Integer source freshness values must be integer values", - )); - } - return Ok(value.str()?.into_any()); - } - if value_kind.eq(STRING_KIND)? { - if !value.is_instance_of::() { - return Err(observation_error( - py, - "String source freshness values must be string values", - )); - } - return Ok(value.clone()); - } - Err(raised( - py, - NativeAttr::SourceFreshnessObservationError, - format!( - "Unsupported source freshness value kind: {}", - value_kind.str()? - ), - )) -} - /// Stored timestamp text parsed with `datetime.fromisoformat`, remembering a Python failure. -struct StateTimestamps<'py> { - py: Python<'py>, - failure: RefCell>, +pub(crate) struct StateTimestamps<'py> { + pub(crate) py: Python<'py>, + pub(crate) failure: RefCell>, } impl StateTimestamps<'_> { @@ -200,7 +121,7 @@ impl TimestampParser for StateTimestamps<'_> { } } -fn comparable(record: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn comparable(record: &Bound<'_, PyAny>) -> PyResult { Ok(ComparableRecord { data_version_hash: native_field(record, "data_version_hash")?.extract()?, value_kind: native_field(record, "value_kind")?.extract()?, @@ -208,31 +129,8 @@ fn comparable(record: &Bound<'_, PyAny>) -> PyResult { }) } -/// Whether two source freshness records are equivalent for skip decisions. -#[pyfunction(name = "source_freshness_records_equivalent")] -#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] -pub(crate) fn equivalent( - previous_record: &Bound<'_, PyAny>, - current_record: &Bound<'_, PyAny>, - lag_tolerance: Option<&str>, -) -> PyResult { - let py: Python<'_> = previous_record.py(); - let parser = StateTimestamps { - py, - failure: RefCell::new(None), - }; - let records = (&comparable(previous_record)?, &comparable(current_record)?); - records_equivalent(records, lag_tolerance, &parser).map_err(|error| { - parser - .failure - .borrow_mut() - .take() - .unwrap_or_else(|| python_error(py, error)) - }) -} - /// A `datetime` required to be aware, in UTC. -fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { +pub(crate) fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { if !offset_aware(value)? { return Err(observation_error( value.py(), @@ -241,45 +139,3 @@ fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult( - policy: &Bound<'py, PyAny>, - data_version: &Bound<'py, PyAny>, - observed_at: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = policy.py(); - if policy.is_none() { - return Ok(None); - } - let statuses: Bound<'py, PyAny> = record_class(py, RecordId::SourceFreshnessAgeStatus)?; - if !date_time_typed(data_version)? { - return native_field(&statuses, "UNKNOWN").map(Some); - } - let version: LocalDateTime = aware_utc(data_version, "data_version")?; - let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; - let error_after: Option = native_field(policy, "error_after")?.extract()?; - let warn_after: Option = native_field(policy, "warn_after")?.extract()?; - let status: AgeStatus = age_status( - (&version, &observed), - error_after.as_deref(), - warn_after.as_deref(), - ) - .map_err(|error| python_error(py, error))?; - let member: &str = match status { - AgeStatus::Pass => "PASS", - AgeStatus::Warn => "WARN", - AgeStatus::Error => "ERROR", - }; - enum_member(py, RecordId::SourceFreshnessAgeStatus, member).map(Some) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(version_hash, module)?)?; - module.add_function(wrap_pyfunction!(normalize, module)?)?; - module.add_function(wrap_pyfunction!(equivalent, module)?)?; - module.add_function(wrap_pyfunction!(age_policy, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs index fd6367a2fd..6dfde51710 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs @@ -1,3 +1,2 @@ pub(crate) mod events; pub(crate) mod freshness; -pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs deleted file mode 100644 index 645abff126..0000000000 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Python `datetime` values normalized to UTC wall time. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_cursor::algebra::models::LocalDateTime; - -use crate::cursor::constants::DATETIME_MODULE; -use crate::cursor::main::values::local_date_time::local_date_time; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// Whether `value` is a `datetime`. -pub(crate) fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { - value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) -} - -/// Whether `value` has a `tzinfo` that reports an offset. -pub(crate) fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { - Ok(!native_field(value, "tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) -} - -/// `value.astimezone(UTC)`. -pub(crate) fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - value.call_method1( - "astimezone", - (imported(value.py(), DATETIME_MODULE, "UTC")?,), - ) -} - -/// The UTC wall time of `value`, reading a naive or offset-less value as UTC. -pub(crate) fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { - if offset_aware(value)? { - return local_date_time(&in_utc(value)?); - } - local_date_time(value) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs index 94b847285c..7d8b673ad1 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs @@ -1,41 +1,28 @@ //! Read and append migration state events through the adapter's execute callable. +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; +use crate::state_rows::main::migrations::layout::layout; +use crate::state_rows::main::migrations::read_events::read_events; use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::recorded_renames::recorded_renames; use sqlbuild_state::events::models::RecordedRenameKey; use crate::state_rows::_helpers::event_rows::events::relation as event_relation; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; -use crate::state_rows::_helpers::migrations::migration_objects::event_order; -use crate::state_rows::_helpers::migrations::migration_objects::layout; -use crate::state_rows::_helpers::migrations::migration_objects::state_error; -use crate::state_rows::_helpers::migrations::migration_projections::old_name_views; -use crate::state_rows::_helpers::migrations::migration_rows::decoded_events; -use crate::state_rows::_helpers::migrations::migration_rows::read_statement; +use crate::state_rows::_helpers::migrations::migration_objects::{event_order, state_error}; use crate::state_rows::_helpers::migrations::migration_rows::write_statements; -use crate::state_rows::constants::COLUMN_KIND; -use crate::state_rows::constants::LOGGING_MODULE; -use crate::state_rows::constants::MIGRATION_LOGGER; -use crate::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; -use crate::state_rows::constants::MIGRATION_WRITE_RETRY_BASE_SECONDS; -use crate::state_rows::constants::MIGRATION_WRITE_RETRY_MESSAGE; -use crate::state_rows::constants::MODEL_KIND; -use crate::state_rows::constants::OLD_NAME_KIND; -use crate::state_rows::constants::TIME_MODULE; -use crate::state_rows::models::EventTable; -use crate::state_rows::models::EventTableLayout; -use crate::state_rows::models::StateAccess; +use crate::state_rows::constants::{ + COLUMN_KIND, LOGGING_MODULE, MIGRATION_LOGGER, MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, MIGRATION_WRITE_RETRY_MESSAGE, TIME_MODULE, +}; +use crate::state_rows::models::{EventTable, EventTableLayout, StateAccess}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; -fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { +pub(crate) fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { let py: Python<'py> = access.execute.py(); call_with( &access.execute, @@ -46,7 +33,11 @@ fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult, error: PyErr) -> PyErr { +pub(crate) fn unreadable( + layout: &EventTableLayout, + schema: &Bound<'_, PyAny>, + error: PyErr, +) -> PyErr { let py: Python<'_> = schema.py(); let describe = |value: &Bound<'_, PyAny>| -> String { value.str().map(|text| text.to_string()).unwrap_or_default() @@ -62,56 +53,6 @@ fn unreadable(layout: &EventTableLayout, schema: &Bound<'_, PyAny>, error: PyErr wrapped } -/// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -pub(crate) fn read_events<'py>( - access: &StateAccess<'py>, - layout: &EventTableLayout, - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), - stored: Option<&[String]>, -) -> PyResult> { - let py: Python<'py> = schema.py(); - let (sql, columns): (String, Vec) = read_statement( - layout, - &access.render_qualified_name, - (database, schema.clone()), - stored, - )?; - let rows: Bound<'py, PyAny> = - match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { - Ok(rows) => rows, - Err(error) if error.is_instance_of::(py) => { - return Err(unreadable(layout, &schema, error)); - } - Err(error) => return Err(error), - }; - decoded_events(layout.table, &rows, &columns) -} - -/// The column names of one schema's old-name view state table, as the adapter lists them. -pub(crate) fn stored_old_name_columns<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = adapter.py(); - let columns: Bound<'py, PyAny> = call_with( - &native_field(adapter, "get_columns")?, - &[ - ("connection", connection.clone()), - ("database", database), - ("schema", schema), - ( - "name", - PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), - ), - ], - )?; - columns - .try_iter()? - .map(|column| native_field(&column?, "name")?.extract()) - .collect() -} - fn subject(table: EventTable, event: &Bound<'_, PyAny>) -> PyResult { let text = |value: Bound<'_, PyAny>| -> PyResult { Ok(value.str()?.to_string()) }; Ok(match table { @@ -203,7 +144,7 @@ pub(crate) fn access<'py>(request: &Request<'_, 'py>) -> PyResult>) -> PyResult>> { +pub(crate) fn stored_names(stored: Option>) -> PyResult>> { match stored { Some(stored) if !stored.is_none() => stored .try_iter()? @@ -217,7 +158,9 @@ fn stored_names(stored: Option>) -> PyResult /// Every event of one migration state table this version reads from one schema. #[pyfunction(name = "read_migration_state_events")] #[pyo3(signature = (**request))] -fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_state_events<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_migration_state_events", request)?; let stored: Option> = stored_names(request.optional("stored_columns")?)?; read_events( @@ -228,32 +171,12 @@ fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("read_old_name_view_histories", request)?; - let access: StateAccess<'py> = access(&request)?; - let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok((request.get("database")?, request.get("schema")?)) - }; - let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; - let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; - let facts: Bound<'py, PyTuple> = read_events( - &access, - &layout(OLD_NAME_KIND)?, - location()?, - stored.as_deref(), - )?; - old_name_views(moves.as_any(), facts.as_any()) -} - /// Renames recorded on the relation at or after `since`, oldest first. #[pyfunction(name = "read_recorded_column_renames")] #[pyo3(signature = (**request))] -fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_renames<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_recorded_column_renames", request)?; let py: Python<'py> = request.py(); let relation: Bound<'py, PyAny> = request.get("relation")?; @@ -289,25 +212,10 @@ fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - database: Bound<'py, PyAny>, - schema: Bound<'py, PyAny>, -) -> PyResult> { - PyTuple::new( - adapter.py(), - stored_old_name_columns(adapter, connection, (database, schema))?, - ) -} - /// Create the state table from adapter DDL when given, then append the event unless recorded. #[pyfunction(name = "write_migration_state_event")] #[pyo3(signature = (**request))] -fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { +pub(crate) fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { let request = Request::new("write_migration_state_event", request)?; let create_table_sql: Option = request.get("create_table_sql")?.extract()?; let attempts: i64 = match request.optional("attempts")? { @@ -324,12 +232,3 @@ fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { attempts, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(read_state_events, module)?)?; - module.add_function(wrap_pyfunction!(read_histories, module)?)?; - module.add_function(wrap_pyfunction!(read_renames, module)?)?; - module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; - module.add_function(wrap_pyfunction!(write_state_event, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs index 64fdb804f0..e6616cbe36 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs @@ -1,63 +1,16 @@ //! Migration state table layouts and the Python objects stored in them. -use pyo3::exceptions::PyValueError; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; -use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, -}; use sqlbuild_state::events::models::EventOrder; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; -use crate::state_rows::constants::COLUMN_KIND; -use crate::state_rows::constants::MODEL_KIND; -use crate::state_rows::constants::OLD_NAME_KIND; -use crate::state_rows::models::EventTable; use crate::state_rows::models::EventTableLayout; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -/// The layout of the state table `kind` names: `model`, `column` or `old_name`. -pub(crate) fn layout(kind: &str) -> PyResult { - match kind { - MODEL_KIND => Ok(EventTableLayout { - table: EventTable::Model, - name: MIGRATION_TABLE_NAME, - columns: &MIGRATION_COLUMNS, - relation_attribute: "destination", - state_message: "model migration state requires a target schema", - event_message: "model migration events require a destination schema", - read_subject: "model migration events", - }), - COLUMN_KIND => Ok(EventTableLayout { - table: EventTable::Column, - name: COLUMN_MIGRATION_TABLE_NAME, - columns: &COLUMN_MIGRATION_COLUMNS, - relation_attribute: "relation", - state_message: "column migration state requires a target schema", - event_message: "column migration events require a relation schema", - read_subject: "column migration events", - }), - OLD_NAME_KIND => Ok(EventTableLayout { - table: EventTable::OldName, - name: OLD_NAME_VIEW_TABLE_NAME, - columns: &OLD_NAME_VIEW_COLUMNS, - relation_attribute: "new", - state_message: "old-name view state requires a target schema", - event_message: "old-name view facts require a destination schema", - read_subject: "old-name view facts", - }), - _ => Err(PyValueError::new_err(format!( - "unknown migration state table kind: {kind}" - ))), - } -} - /// `MigrationStateError(message)`. pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { new_plain_error(py, ErrorId::MigrationState, message) diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs index e2e0dc9087..f9a1c6b56a 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs @@ -1,28 +1,21 @@ //! Current-state projections over recorded migration events. +use crate::state_rows::main::migrations::newest_column::newest_column; +use crate::state_rows::main::migrations::newest_move::newest_move; +use crate::state_rows::main::migrations::old_name_views::old_name_views; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::events::main::newest_column_event::newest_column_event; -use sqlbuild_state::events::main::newest_move_event::newest_move_event; -use sqlbuild_state::events::main::old_name_histories::old_name_histories; -use sqlbuild_state::events::models::{ - ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_state::events::models::OldNameStep; -use crate::state_rows::_helpers::event_rows::events::relation as event_relation; -use crate::state_rows::_helpers::migrations::migration_objects::event_order; -use crate::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { events.try_iter()?.collect() } -fn chosen<'py>( +pub(crate) fn chosen<'py>( py: Python<'py>, events: &[Bound<'py, PyAny>], index: Option, @@ -32,51 +25,7 @@ fn chosen<'py>( .unwrap_or_else(|| py.None().into_bound(py)) } -/// The latest recorded move naming `relation` as origin or destination, or `None`. -pub(crate) fn newest_move<'py>( - events: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, -) -> PyResult> { - let events: Vec> = collected(events)?; - let keys: Vec = events - .iter() - .map(|event| { - Ok(MoveEventKey { - order: event_order(event)?, - origin: event_relation(&native_field(event, "origin")?)?, - destination: event_relation(&native_field(event, "destination")?)?, - }) - }) - .collect::>()?; - let index: Option = newest_move_event(&keys, &event_relation(relation)?); - Ok(chosen(relation.py(), &events, index)) -} - -/// The latest recorded rename on `relation` naming either column, or `None`. -pub(crate) fn newest_column<'py>( - events: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, - (origin_column, destination_column): (&str, &str), -) -> PyResult> { - let events: Vec> = collected(events)?; - let keys: Vec = events - .iter() - .map(|event| { - Ok(ColumnEventKey { - order: event_order(event)?, - relation: event_relation(&native_field(event, "relation")?)?, - origin_column: native_field(event, "origin_column")?.extract()?, - destination_column: native_field(event, "destination_column")?.extract()?, - }) - }) - .collect::>()?; - let target: EventRelation = event_relation(relation)?; - let index: Option = - newest_column_event(&keys, &target, (origin_column, destination_column)); - Ok(chosen(relation.py(), &events, index)) -} - -fn step(fact: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn step(fact: &Bound<'_, PyAny>) -> PyResult { let value: String = native_field(&native_field(fact, "event_type")?, "value")?.extract()?; match value.as_str() { "required" => Ok(OldNameStep::Required), @@ -89,64 +38,9 @@ fn step(fact: &Bound<'_, PyAny>) -> PyResult { } } -/// One history per recorded move whose old name needs a compatibility view, in move order. -pub(crate) fn old_name_views<'py>( - moves: &Bound<'py, PyAny>, - facts: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = moves.py(); - let moves: Vec> = collected(moves)?; - let facts: Vec> = collected(facts)?; - let move_orders: Vec = moves.iter().map(event_order).collect::>()?; - let fact_keys: Vec = facts - .iter() - .map(|fact| { - Ok(OldNameFactKey { - order: event_order(fact)?, - migration_event_id: native_field(fact, "migration_event_id")?.extract()?, - step: step(fact)?, - }) - }) - .collect::>()?; - let fact_at = |index: Option| chosen(py, &facts, index); - let histories: Vec> = old_name_histories(&move_orders, &fact_keys) - .into_iter() - .map(|indexes| { - new_record_with( - py, - RecordId::OldNameViewHistory, - &[ - ("move", moves[indexes.moved].clone()), - ("required", facts[indexes.required].clone()), - ("archived", fact_at(indexes.archived)), - ("created", fact_at(indexes.created)), - ("dropped", fact_at(indexes.dropped)), - ], - ) - }) - .collect::>()?; - PyTuple::new(py, histories) -} - -/// The model's old-name view retention, the project's when unset, or `None` when disabled. -pub(crate) fn old_name_retention<'py>( - config_values: &Bound<'py, PyAny>, - project_retention: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = config_values.py(); - if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { - return Ok(project_retention.clone()); - } - let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; - if value.is(PyBool::new(py, false)) { - return Ok(py.None().into_bound(py)); - } - value.str()?.call_method0("strip") -} - #[pyfunction(name = "newest_migration_event_mentioning")] #[pyo3(signature = (*, events, relation))] -fn newest_move_binding<'py>( +pub(crate) fn newest_move_binding<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { @@ -155,7 +49,7 @@ fn newest_move_binding<'py>( #[pyfunction(name = "newest_column_migration_event_mentioning")] #[pyo3(signature = (*, events, relation, origin_column, destination_column))] -fn newest_column_binding<'py>( +pub(crate) fn newest_column_binding<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, origin_column: &str, @@ -166,16 +60,9 @@ fn newest_column_binding<'py>( #[pyfunction(name = "project_old_name_views")] #[pyo3(signature = (*, moves, facts))] -fn old_name_views_binding<'py>( +pub(crate) fn old_name_views_binding<'py>( moves: &Bound<'py, PyAny>, facts: &Bound<'py, PyAny>, ) -> PyResult> { old_name_views(moves, facts) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; - module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; - module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs index 77326fddcd..76132113ad 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs @@ -1,11 +1,11 @@ //! Create, read and append SQL of migration state tables, and stored rows decoded to events. +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::migrations::layout::layout; use pyo3::exceptions::{PyKeyError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_state::events::constants::TIMESTAMP_COLUMNS; use sqlbuild_state::events::main::column_aliases_json::column_aliases_json; use sqlbuild_state::events::main::decoded_column_aliases::decoded_column_aliases; @@ -17,13 +17,9 @@ use sqlbuild_state::events::main::readable_columns::readable_columns; use crate::cursor::constants::DATETIME_MODULE; use crate::state_rows::_helpers::event_rows::events::typed_insert_sql; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::migrations::migration_objects::layout; -use crate::state_rows::_helpers::migrations::migration_objects::qualified_table; -use crate::state_rows::_helpers::migrations::migration_objects::state_error; +use crate::state_rows::_helpers::migrations::migration_objects::{qualified_table, state_error}; use crate::state_rows::_helpers::tables::state_tables::state_table_create_sql; -use crate::state_rows::models::EventTable; -use crate::state_rows::models::EventTableLayout; +use crate::state_rows::models::{EventTable, EventTableLayout}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -35,7 +31,7 @@ use sqlbuild_python_base::records::types::RecordId; /// DDL that creates one migration state table when it is missing. #[pyfunction(name = "render_migration_state_create_sql")] #[pyo3(signature = (**request))] -fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { let request = Request::new("render_migration_state_create_sql", request)?; let py: Python<'py> = request.py(); let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; @@ -414,8 +410,3 @@ pub(crate) fn decoded_events<'py>( } PyTuple::new(py, events) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(create_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs index a5f914e949..31f669810e 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs @@ -1,11 +1,9 @@ //! Append-only state table DDL and the source freshness insert, rendered through adapter callbacks. use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ @@ -71,7 +69,7 @@ pub(crate) fn state_table_create_sql<'py>( /// Render create-if-missing DDL for string, integer, and text-timestamp state columns. #[pyfunction(name = "render_state_table_create_sql")] #[pyo3(signature = (**request))] -fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("render_state_table_create_sql", request)?; state_table_create_sql( &request.get("qualified_name")?.extract::()?, @@ -99,7 +97,7 @@ fn adapter_user_error(py: Python<'_>, message: &str) -> PyErr { /// Render DML that appends source freshness records. #[pyfunction] #[pyo3(signature = (*, database, schema, records, render_qualified_name))] -fn render_insert_source_freshness_records_sql<'py>( +pub(crate) fn render_insert_source_freshness_records_sql<'py>( database: &Bound<'py, PyAny>, schema: &Bound<'py, PyAny>, records: &Bound<'py, PyAny>, @@ -143,11 +141,3 @@ fn render_insert_source_freshness_records_sql<'py>( &rows, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; - module.add_function(wrap_pyfunction!( - render_insert_source_freshness_records_sql, - module - )?) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs index 994837c021..b751740557 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs @@ -1,16 +1,48 @@ //! Optional timestamp age policy for one source freshness observation. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::freshness::aware_utc; +use crate::state_rows::_helpers::event_rows::freshness::python_error; +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_state::freshness::main::age_status::age_status; +use sqlbuild_state::freshness::models::AgeStatus; /// Optional timestamp age policy for one source freshness observation. -#[inline] +#[pyfunction(name = "evaluate_source_freshness_age_policy")] +#[pyo3(signature = (*, policy, data_version, observed_at))] pub fn age_policy<'py>( policy: &Bound<'py, PyAny>, data_version: &Bound<'py, PyAny>, observed_at: &Bound<'py, PyAny>, ) -> PyResult>> { - freshness::age_policy(policy, data_version, observed_at) + let py: Python<'py> = policy.py(); + if policy.is_none() { + return Ok(None); + } + let statuses: Bound<'py, PyAny> = record_class(py, RecordId::SourceFreshnessAgeStatus)?; + if !date_time_typed(data_version)? { + return native_field(&statuses, "UNKNOWN").map(Some); + } + let version: LocalDateTime = aware_utc(data_version, "data_version")?; + let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; + let error_after: Option = native_field(policy, "error_after")?.extract()?; + let warn_after: Option = native_field(policy, "warn_after")?.extract()?; + let status: AgeStatus = age_status( + (&version, &observed), + error_after.as_deref(), + warn_after.as_deref(), + ) + .map_err(|error| python_error(py, error))?; + let member: &str = match status { + AgeStatus::Pass => "PASS", + AgeStatus::Warn => "WARN", + AgeStatus::Error => "ERROR", + }; + enum_member(py, RecordId::SourceFreshnessAgeStatus, member).map(Some) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs index 7215ae0fd5..f209a3cddb 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs @@ -1,12 +1,10 @@ //! Whether `value` is a `datetime`. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::constants::DATETIME_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::imported::imported; /// Whether `value` is a `datetime`. -#[inline] pub fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::date_time_typed(value) + value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs index cb8bd180b5..5c3da9029d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs @@ -1,16 +1,32 @@ //! Whether two source freshness records are equivalent for skip decisions. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::freshness::StateTimestamps; +use crate::state_rows::_helpers::event_rows::freshness::comparable; +use crate::state_rows::_helpers::event_rows::freshness::python_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; +use std::cell::RefCell; /// Whether two source freshness records are equivalent for skip decisions. -#[inline] +#[pyfunction(name = "source_freshness_records_equivalent")] +#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] pub fn equivalent( previous_record: &Bound<'_, PyAny>, current_record: &Bound<'_, PyAny>, lag_tolerance: Option<&str>, ) -> PyResult { - freshness::equivalent(previous_record, current_record, lag_tolerance) + let py: Python<'_> = previous_record.py(); + let parser = StateTimestamps { + py, + failure: RefCell::new(None), + }; + let records = (&comparable(previous_record)?, &comparable(current_record)?); + records_equivalent(records, lag_tolerance, &parser).map_err(|error| { + parser + .failure + .borrow_mut() + .take() + .unwrap_or_else(|| python_error(py, error)) + }) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs index 8d28e7e853..208245a749 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs @@ -1,12 +1,13 @@ //! `value.astimezone(UTC)`. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::constants::DATETIME_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::imported::imported; /// `value.astimezone(UTC)`. -#[inline] pub fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - timestamps::in_utc(value) + value.call_method1( + "astimezone", + (imported(value.py(), DATETIME_MODULE, "UTC")?,), + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs index ec853a1552..794807f328 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs @@ -1,15 +1,70 @@ //! A source freshness value normalized for stable state storage. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::state_rows::_helpers::event_rows::freshness::INTEGER_KIND; +use crate::state_rows::_helpers::event_rows::freshness::STRING_KIND; +use crate::state_rows::_helpers::event_rows::freshness::TIMESTAMP_KIND; +use crate::state_rows::_helpers::event_rows::freshness::observation_error; +use crate::state_rows::_helpers::event_rows::freshness::raised; +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::event_rows::in_utc::in_utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyInt, PyString}; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::NativeAttr; +use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; +use sqlbuild_state::freshness::models::DataVersion; /// A source freshness value normalized for stable state storage. -#[inline] +#[pyfunction(name = "normalize_source_freshness_data_version")] +#[pyo3(signature = (*, value, value_kind))] pub fn normalize<'py>( value: &Bound<'py, PyAny>, value_kind: &Bound<'py, PyAny>, ) -> PyResult> { - freshness::normalize(value, value_kind) + let py: Python<'py> = value.py(); + if value_kind.eq(TIMESTAMP_KIND)? { + if !date_time_typed(value)? { + return Err(observation_error( + py, + "Timestamp source freshness values must be datetime values", + )); + } + let aware: bool = !native_field(value, "tzinfo")?.is_none(); + let local: LocalDateTime = if aware { + local_date_time(&in_utc(value)?)? + } else { + local_date_time(value)? + }; + let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); + return Ok(PyString::new(py, &text).into_any()); + } + if value_kind.eq(INTEGER_KIND)? { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(observation_error( + py, + "Integer source freshness values must be integer values", + )); + } + return Ok(value.str()?.into_any()); + } + if value_kind.eq(STRING_KIND)? { + if !value.is_instance_of::() { + return Err(observation_error( + py, + "String source freshness values must be string values", + )); + } + return Ok(value.clone()); + } + Err(raised( + py, + NativeAttr::SourceFreshnessObservationError, + format!( + "Unsupported source freshness value kind: {}", + value_kind.str()? + ), + )) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs index 88ffd3edd5..e07cb3ac56 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs @@ -1,12 +1,9 @@ //! Whether `value` has a `tzinfo` that reports an offset. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Whether `value` has a `tzinfo` that reports an offset. -#[inline] pub fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::offset_aware(value) + Ok(!native_field(value, "tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs index 9d8d99beed..064da5dade 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs @@ -1,13 +1,15 @@ //! The UTC wall time of `value`, reading a naive or offset-less value as UTC. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::state_rows::main::event_rows::in_utc::in_utc; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_cursor::algebra::models::LocalDateTime; /// The UTC wall time of `value`, reading a naive or offset-less value as UTC. -#[inline] pub fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::presumed_utc(value) + if offset_aware(value)? { + return local_date_time(&in_utc(value)?); + } + local_date_time(value) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs index 530de6df36..9c193887ad 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs @@ -1,12 +1,15 @@ //! Register the events functions and classes. -use crate::state_rows::_helpers::event_rows::events; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::column_id; +use crate::state_rows::_helpers::event_rows::events::migration_id; +use crate::state_rows::_helpers::event_rows::events::old_name_id; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the events functions and classes. -#[inline] pub fn register_events(module: &Bound<'_, PyModule>) -> PyResult<()> { - events::register(module) + module.add_function(wrap_pyfunction!(migration_id, module)?)?; + module.add_function(wrap_pyfunction!(column_id, module)?)?; + module.add_function(wrap_pyfunction!(old_name_id, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs index 61581e44e6..0d8791b56b 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs @@ -1,12 +1,17 @@ //! Register the freshness functions and classes. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::main::event_rows::age_policy::age_policy; +use crate::state_rows::main::event_rows::equivalent::equivalent; +use crate::state_rows::main::event_rows::normalize::normalize; +use crate::state_rows::main::event_rows::version_hash::version_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the freshness functions and classes. -#[inline] pub fn register_freshness(module: &Bound<'_, PyModule>) -> PyResult<()> { - freshness::register(module) + module.add_function(wrap_pyfunction!(version_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize, module)?)?; + module.add_function(wrap_pyfunction!(equivalent, module)?)?; + module.add_function(wrap_pyfunction!(age_policy, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs index 544e6287c0..b3a371135c 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs @@ -1,15 +1,18 @@ //! Hash the stable source freshness identity used as a graph input. -use crate::state_rows::_helpers::event_rows::freshness; use pyo3::prelude::PyResult; +use pyo3::pyfunction; +use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; /// Hash the stable source freshness identity used as a graph input. -#[inline] +#[pyfunction(name = "source_freshness_data_version_hash")] +#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] pub fn version_hash( source_name: &str, strategy: &str, value_kind: &str, data_version: &str, ) -> PyResult { - freshness::version_hash(source_name, strategy, value_kind, data_version) + data_version_hash((source_name, strategy), value_kind, data_version) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs index caae386fb9..0a3a1fdc96 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs @@ -1,11 +1,48 @@ //! The layout of the state table `kind` names: `model`, `column` or `old_name`. -use crate::state_rows::_helpers::migrations::migration_objects; -use crate::state_rows::models::EventTableLayout; +use crate::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use crate::state_rows::models::{EventTable, EventTableLayout}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::PyResult; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, +}; /// The layout of the state table `kind` names: `model`, `column` or `old_name`. -#[inline] pub fn layout(kind: &str) -> PyResult { - migration_objects::layout(kind) + match kind { + MODEL_KIND => Ok(EventTableLayout { + table: EventTable::Model, + name: MIGRATION_TABLE_NAME, + columns: &MIGRATION_COLUMNS, + relation_attribute: "destination", + state_message: "model migration state requires a target schema", + event_message: "model migration events require a destination schema", + read_subject: "model migration events", + }), + COLUMN_KIND => Ok(EventTableLayout { + table: EventTable::Column, + name: COLUMN_MIGRATION_TABLE_NAME, + columns: &COLUMN_MIGRATION_COLUMNS, + relation_attribute: "relation", + state_message: "column migration state requires a target schema", + event_message: "column migration events require a relation schema", + read_subject: "column migration events", + }), + OLD_NAME_KIND => Ok(EventTableLayout { + table: EventTable::OldName, + name: OLD_NAME_VIEW_TABLE_NAME, + columns: &OLD_NAME_VIEW_COLUMNS, + relation_attribute: "new", + state_message: "old-name view state requires a target schema", + event_message: "old-name view facts require a destination schema", + read_subject: "old-name view facts", + }), + _ => Err(PyValueError::new_err(format!( + "unknown migration state table kind: {kind}" + ))), + } } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs index aa92be6ad4..87bcdc7aa0 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs @@ -1,16 +1,34 @@ //! The latest recorded rename on `relation` naming either column, or `None`. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::events::main::newest_column_event::newest_column_event; +use sqlbuild_state::events::models::{ColumnEventKey, EventRelation}; /// The latest recorded rename on `relation` naming either column, or `None`. -#[inline] pub fn newest_column<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, (origin_column, destination_column): (&str, &str), ) -> PyResult> { - migration_projections::newest_column(events, relation, (origin_column, destination_column)) + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(ColumnEventKey { + order: event_order(event)?, + relation: event_relation(&native_field(event, "relation")?)?, + origin_column: native_field(event, "origin_column")?.extract()?, + destination_column: native_field(event, "destination_column")?.extract()?, + }) + }) + .collect::>()?; + let target: EventRelation = event_relation(relation)?; + let index: Option = + newest_column_event(&keys, &target, (origin_column, destination_column)); + Ok(chosen(relation.py(), &events, index)) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs index c24e453093..b65f4ec24d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs @@ -1,15 +1,30 @@ //! The latest recorded move naming `relation` as origin or destination, or `None`. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::events::main::newest_move_event::newest_move_event; +use sqlbuild_state::events::models::MoveEventKey; /// The latest recorded move naming `relation` as origin or destination, or `None`. -#[inline] pub fn newest_move<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::newest_move(events, relation) + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(MoveEventKey { + order: event_order(event)?, + origin: event_relation(&native_field(event, "origin")?)?, + destination: event_relation(&native_field(event, "destination")?)?, + }) + }) + .collect::>()?; + let index: Option = newest_move_event(&keys, &event_relation(relation)?); + Ok(chosen(relation.py(), &events, index)) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs index 6396c515f7..0291efb114 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs @@ -1,15 +1,21 @@ //! The model's old-name view retention, the project's when unset, or `None` when disabled. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; /// The model's old-name view retention, the project's when unset, or `None` when disabled. -#[inline] pub fn old_name_retention<'py>( config_values: &Bound<'py, PyAny>, project_retention: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::old_name_retention(config_values, project_retention) + let py: Python<'py> = config_values.py(); + if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { + return Ok(project_retention.clone()); + } + let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; + if value.is(PyBool::new(py, false)) { + return Ok(py.None().into_bound(py)); + } + value.str()?.call_method0("strip") } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs index 92d839abeb..c8504209b9 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs @@ -1,16 +1,52 @@ //! One history per recorded move whose old name needs a compatibility view, in move order. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use crate::state_rows::_helpers::migrations::migration_projections::step; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_state::events::main::old_name_histories::old_name_histories; +use sqlbuild_state::events::models::{EventOrder, OldNameFactKey}; /// One history per recorded move whose old name needs a compatibility view, in move order. -#[inline] pub fn old_name_views<'py>( moves: &Bound<'py, PyAny>, facts: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::old_name_views(moves, facts) + let py: Python<'py> = moves.py(); + let moves: Vec> = collected(moves)?; + let facts: Vec> = collected(facts)?; + let move_orders: Vec = moves.iter().map(event_order).collect::>()?; + let fact_keys: Vec = facts + .iter() + .map(|fact| { + Ok(OldNameFactKey { + order: event_order(fact)?, + migration_event_id: native_field(fact, "migration_event_id")?.extract()?, + step: step(fact)?, + }) + }) + .collect::>()?; + let fact_at = |index: Option| chosen(py, &facts, index); + let histories: Vec> = old_name_histories(&move_orders, &fact_keys) + .into_iter() + .map(|indexes| { + new_record_with( + py, + RecordId::OldNameViewHistory, + &[ + ("move", moves[indexes.moved].clone()), + ("required", facts[indexes.required].clone()), + ("archived", fact_at(indexes.archived)), + ("created", fact_at(indexes.created)), + ("dropped", fact_at(indexes.dropped)), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, histories) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs index 44beac2afc..55a96e322e 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs @@ -1,20 +1,34 @@ //! Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -use crate::state_rows::_helpers::migrations::migration_io; -use crate::state_rows::models::EventTableLayout; -use crate::state_rows::models::StateAccess; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_io::execute; +use crate::state_rows::_helpers::migrations::migration_io::unreadable; +use crate::state_rows::_helpers::migrations::migration_rows::{decoded_events, read_statement}; +use crate::state_rows::models::{EventTableLayout, StateAccess}; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; /// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -#[inline] pub fn read_events<'py>( access: &StateAccess<'py>, layout: &EventTableLayout, (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), stored: Option<&[String]>, ) -> PyResult> { - migration_io::read_events(access, layout, (database, schema), stored) + let py: Python<'py> = schema.py(); + let (sql, columns): (String, Vec) = read_statement( + layout, + &access.render_qualified_name, + (database, schema.clone()), + stored, + )?; + let rows: Bound<'py, PyAny> = + match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { + Ok(rows) => rows, + Err(error) if error.is_instance_of::(py) => { + return Err(unreadable(layout, &schema, error)); + } + Err(error) => return Err(error), + }; + decoded_events(layout.table, &rows, &columns) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs index 1c946a2d48..b8c80a71d8 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs @@ -1,13 +1,33 @@ //! The old-name view history of every recorded move in a schema that holds both state tables. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::state_rows::_helpers::migrations::migration_io::access; +use crate::state_rows::_helpers::migrations::migration_io::stored_names; +use crate::state_rows::constants::{MODEL_KIND, OLD_NAME_KIND}; +use crate::state_rows::main::migrations::layout::layout; +use crate::state_rows::main::migrations::old_name_views::old_name_views; +use crate::state_rows::main::migrations::read_events::read_events; +use crate::state_rows::models::StateAccess; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; /// The old-name view history of every recorded move in a schema that holds both state tables. -#[inline] +#[pyfunction(name = "read_old_name_view_histories")] +#[pyo3(signature = (**request))] pub fn read_histories<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { - migration_io::read_histories(request) + let request = Request::new("read_old_name_view_histories", request)?; + let access: StateAccess<'py> = access(&request)?; + let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok((request.get("database")?, request.get("schema")?)) + }; + let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; + let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; + let facts: Bound<'py, PyTuple> = read_events( + &access, + &layout(OLD_NAME_KIND)?, + location()?, + stored.as_deref(), + )?; + old_name_views(moves.as_any(), facts.as_any()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs index 90b03f6d18..c3167fcd2d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs @@ -1,12 +1,19 @@ //! Register the migration io functions and classes. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_io::read_renames; +use crate::state_rows::_helpers::migrations::migration_io::read_state_events; +use crate::state_rows::_helpers::migrations::migration_io::write_state_event; +use crate::state_rows::main::migrations::read_histories::read_histories; +use crate::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration io functions and classes. -#[inline] pub fn register_migration_io(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_io::register(module) + module.add_function(wrap_pyfunction!(read_state_events, module)?)?; + module.add_function(wrap_pyfunction!(read_histories, module)?)?; + module.add_function(wrap_pyfunction!(read_renames, module)?)?; + module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; + module.add_function(wrap_pyfunction!(write_state_event, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs index 65e9d95e8d..5f07e4919f 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs @@ -1,12 +1,15 @@ //! Register the migration projections functions and classes. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_projections::newest_column_binding; +use crate::state_rows::_helpers::migrations::migration_projections::newest_move_binding; +use crate::state_rows::_helpers::migrations::migration_projections::old_name_views_binding; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration projections functions and classes. -#[inline] pub fn register_migration_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_projections::register(module) + module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; + module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; + module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs index 8a095b7b1e..dce4494d52 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs @@ -1,12 +1,11 @@ //! Register the migration rows functions and classes. -use crate::state_rows::_helpers::migrations::migration_rows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_rows::create_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration rows functions and classes. -#[inline] pub fn register_migration_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_rows::register(module) + module.add_function(wrap_pyfunction!(create_sql, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs index dfd775e2ba..d4c0606771 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs @@ -1,18 +1,21 @@ //! The column names of one schema's old-name view state table. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// The column names of one schema's old-name view state table. -#[inline] +#[pyfunction(name = "stored_old_name_view_columns")] +#[pyo3(signature = (*, adapter, connection, database, schema))] pub fn stored_columns_binding<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, database: Bound<'py, PyAny>, schema: Bound<'py, PyAny>, ) -> PyResult> { - migration_io::stored_columns_binding(adapter, connection, database, schema) + PyTuple::new( + adapter.py(), + stored_old_name_columns(adapter, connection, (database, schema))?, + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs index 6c29967353..cf9cc38b80 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs @@ -1,16 +1,32 @@ //! The column names of one schema's old-name view state table, as the adapter lists them. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The column names of one schema's old-name view state table, as the adapter lists them. -#[inline] pub fn stored_old_name_columns<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { - migration_io::stored_old_name_columns(adapter, connection, (database, schema)) + let py: Python<'py> = adapter.py(); + let columns: Bound<'py, PyAny> = call_with( + &native_field(adapter, "get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database), + ("schema", schema), + ( + "name", + PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), + ), + ], + )?; + columns + .try_iter()? + .map(|column| native_field(&column?, "name")?.extract()) + .collect() } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs index db283b6c98..374cfdc103 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs @@ -1,12 +1,15 @@ //! Register the state tables functions and classes. -use crate::state_rows::_helpers::tables::state_tables; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::tables::state_tables::render_create_sql; +use crate::state_rows::_helpers::tables::state_tables::render_insert_source_freshness_records_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the state tables functions and classes. -#[inline] pub fn register_state_tables(module: &Bound<'_, PyModule>) -> PyResult<()> { - state_tables::register(module) + module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_insert_source_freshness_records_sql, + module + )?) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs index 20b153bb59..3657bb3367 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs @@ -1,9 +1,7 @@ //! Create-if-missing DDL whose column types the adapter renders. use crate::state_rows::_helpers::tables::state_tables; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Create-if-missing DDL whose column types the adapter renders. #[inline] diff --git a/crates/sqlbuild-python-front/src/state_rows/models.rs b/crates/sqlbuild-python-front/src/state_rows/models.rs index 542ffd658c..1a2a8e7146 100644 --- a/crates/sqlbuild-python-front/src/state_rows/models.rs +++ b/crates/sqlbuild-python-front/src/state_rows/models.rs @@ -1,7 +1,6 @@ //! Public records of state rows. -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; /// Which migration state table a request reads or writes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs index d9d12ad91e..39d46a71d0 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs @@ -4,11 +4,9 @@ use crate::analysis::constants::NATIVE_DIALECT_ALIASES; use crate::analysis::models::BindingCatalog; use crate::analysis::models::CatalogKey; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; @@ -18,37 +16,15 @@ use sqlbuild_python_base::types::IntoPositional; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; pub(crate) const QUOTED_IDENTIFIER_DELIMITER: char = '"'; -const SNOWFLAKE_DIALECT: &str = "snowflake"; -const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; -const SESSION_PARAMETERS: &str = "session_parameters"; +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; +pub(crate) const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; +pub(crate) const SESSION_PARAMETERS: &str = "session_parameters"; const GENERIC_DIALECT: &str = "generic"; -/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. -pub(crate) fn ignores_quoted_case( - connection: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { - return Ok(false); - } - let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; - let Ok(parameters) = parameters.downcast::() else { - return Ok(false); - }; - for (key, value) in parameters.iter() { - let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; - if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { - return Ok(true); - } - } - Ok(false) -} - /// The native dialect a request's adapter dialect validates as. fn native_dialect(dialect: &Bound<'_, PyAny>) -> PyResult { let name: String = if dialect.is_truthy()? { @@ -144,9 +120,3 @@ pub(crate) fn schema_validations<'py>( } PyTuple::new(py, results) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(schema_validations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs index 3ea5e12ed0..a7abc19801 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs @@ -6,9 +6,8 @@ use crate::analysis::_helpers::conversions::columns; use crate::analysis::_helpers::conversions::relations; use crate::analysis::_helpers::native_classes::CatalogInput; use crate::analysis::_helpers::normalization_results::normalization_results; -use crate::analysis::models::CompactAnalysisJob; use crate::analysis::models::Positional; -use crate::analysis::models::ProjectCatalog; +use crate::analysis::models::{CompactAnalysisJob, ProjectCatalog}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs index 960787e244..066785deb1 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs @@ -1,7 +1,7 @@ //! Normalize SQL for analysis and recover binding diagnostics. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; use crate::analysis::_helpers::normalization_results::normalization_results; use crate::analysis::models::Positional; @@ -10,20 +10,7 @@ use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; #[pyfunction] -pub(crate) fn normalize_analysis_sql( - py: Python<'_>, - request: crate::analysis::models::NormalizationInput, -) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( - request.into(), - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { +pub(crate) fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql( sql, dialect, @@ -34,7 +21,7 @@ fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, dialect: &str, requests: Vec, @@ -50,7 +37,7 @@ fn normalize_analysis_sqls( } #[pyfunction] -fn binding_diagnostics( +pub(crate) fn binding_diagnostics( py: Python<'_>, sql: &str, dialect: &str, @@ -65,11 +52,3 @@ fn binding_diagnostics( .map(Positional::wrap_all) .map_err(value_error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; - module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; - module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; - module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs index b53de697dc..42d5c7533f 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs @@ -2,41 +2,20 @@ use crate::analysis::types::RelationRow; use pyo3::PyErr; -use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; -use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; -use sqlbuild_analysis::relation_names::models::{ExtractedRelations, RelationName}; +use sqlbuild_analysis::relation_names::models::RelationName; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::types::CompilerDetach; const RELATION_NAMES_CONTEXT: &str = "native relation name extraction"; -/// `(relations, temporary names)` named in `sql`, or None if unparseable. -pub(crate) fn extract_sql_relation_names( - py: Python<'_>, - sql: &str, - dialect: Option<&str>, -) -> PyResult, Vec)>> { - let extracted = py - .compiler_detach(|| Ok(extract_relation_names(sql, dialect))) - .map_err(compiler_error)? - .map_err(relation_names_error)?; - Ok(extracted.map( - |ExtractedRelations { - relations, - temporary, - }| { (relations.into_iter().map(relation_row).collect(), temporary) }, - )) -} - -fn relation_row(relation: RelationName) -> RelationRow { +pub(crate) fn relation_row(relation: RelationName) -> RelationRow { (relation.name, relation.schema, relation.database) } /// An unknown dialect as the wheel's `ValueError`; anything else as an internal failure. -fn relation_names_error(failure: RelationNamesFailure) -> PyErr { +pub(crate) fn relation_names_error(failure: RelationNamesFailure) -> PyErr { match failure { RelationNamesFailure::UnknownDialect(message) => compiler_error(message), RelationNamesFailure::Internal(reason) => { diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs index 3899eab30c..b306f205d8 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs @@ -1,59 +1,3 @@ //! Build the stable native schema-validation request of one model query. -use crate::analysis::models::SchemaRequestParts; -use std::collections::HashMap; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDict; -use sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql; -use sqlbuild_analysis::semantic_validation::models::NormalizationInput; - -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::types::CompilerDetach; - -const GENERIC_DIALECT: &str = "generic"; - -/// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. -pub(crate) fn schema_validation_request<'py>( - py: Python<'py>, - parts: SchemaRequestParts<'py>, -) -> PyResult> { - let dialect: String = if parts.dialect.is_truthy()? { - parts.dialect.extract()? - } else { - GENERIC_DIALECT.to_owned() - }; - let placeholders: HashMap = if parts.placeholders.is_truthy()? { - parts.placeholders.extract()? - } else { - HashMap::new() - }; - let input: NormalizationInput = NormalizationInput { - sql: parts.query_sql, - dialect, - stubs: HashMap::new(), - placeholders, - }; - let sql: String = py - .compiler_detach(|| normalize_analysis_sql(input)) - .map_err(value_error)?; - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("sql", sql)?; - arguments.set_item("dialect", parts.dialect)?; - arguments.set_item("schema", parts.schema)?; - arguments.set_item("known_functions", parts.known_functions)?; - arguments.set_item("known_types", parts.known_types)?; - arguments.set_item( - "quoted_identifiers_ignore_case", - parts.quoted_identifiers_ignore_case, - )?; - arguments.set_item("catalog", parts.catalog)?; - new_record( - py, - RecordId::SqlSchemaValidationRequest, - (), - Some(&arguments), - ) -} +pub(crate) const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs index 19d2f4a596..9b2e06b121 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs @@ -1,16 +1,28 @@ //! `(relations, temporary names)` named in `sql`, or None if unparseable. -use crate::analysis::_helpers::relation_names; +use crate::analysis::_helpers::relation_names::relation_names_error; +use crate::analysis::_helpers::relation_names::relation_row; use crate::analysis::types::RelationRow; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; +use sqlbuild_analysis::relation_names::models::ExtractedRelations; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; /// `(relations, temporary names)` named in `sql`, or None if unparseable. -#[inline] pub fn extract_sql_relation_names( py: Python<'_>, sql: &str, dialect: Option<&str>, ) -> PyResult, Vec)>> { - relation_names::extract_sql_relation_names(py, sql, dialect) + let extracted = py + .compiler_detach(|| Ok(extract_relation_names(sql, dialect))) + .map_err(compiler_error)? + .map_err(relation_names_error)?; + Ok(extracted.map( + |ExtractedRelations { + relations, + temporary, + }| { (relations.into_iter().map(relation_row).collect(), temporary) }, + )) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs index 89f1e7795e..9995efd954 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs @@ -1,15 +1,29 @@ //! `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. -use crate::analysis::_helpers::binding_catalog; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis::_helpers::binding_catalog::QUOTED_CASE_PARAMETER; +use crate::analysis::_helpers::binding_catalog::SESSION_PARAMETERS; +use crate::analysis::_helpers::binding_catalog::SNOWFLAKE_DIALECT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyDictMethods, PyResult}; +use pyo3::types::{PyBool, PyDict}; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; /// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. -#[inline] pub fn ignores_quoted_case( connection: &Bound<'_, PyAny>, dialect: &Bound<'_, PyAny>, ) -> PyResult { - binding_catalog::ignores_quoted_case(connection, dialect) + if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { + return Ok(false); + } + let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; + let Ok(parameters) = parameters.downcast::() else { + return Ok(false); + }; + for (key, value) in parameters.iter() { + let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; + if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { + return Ok(true); + } + } + Ok(false) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs index 14458feef5..37407c64b2 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs @@ -1,14 +1,20 @@ //! Normalize analysis sql. -use crate::analysis::_helpers::normalization; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; /// Normalize analysis sql. -#[inline] +#[pyfunction] pub fn normalize_analysis_sql( py: Python<'_>, request: crate::analysis::models::NormalizationInput, ) -> PyResult { - normalization::normalize_analysis_sql(py, request) + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( + request.into(), + ) + }) + .map_err(value_error) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs index 4568154300..71b586b3f3 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs @@ -1,12 +1,13 @@ //! Register the binding catalog functions and classes. -use crate::analysis::_helpers::binding_catalog; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis::_helpers::binding_catalog::schema_validations; +use crate::analysis::models::BindingCatalog; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the binding catalog functions and classes. -#[inline] pub fn register_binding_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { - binding_catalog::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(schema_validations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs index c6f68e563a..46e2e59955 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs @@ -1,12 +1,17 @@ //! Register the normalization functions and classes. -use crate::analysis::_helpers::normalization; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis::_helpers::normalization::binding_diagnostics; +use crate::analysis::_helpers::normalization::normalize_analysis_sqls; +use crate::analysis::_helpers::normalization::normalize_dialect_sql; +use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the normalization functions and classes. -#[inline] pub fn register_normalization(module: &Bound<'_, PyModule>) -> PyResult<()> { - normalization::register(module) + module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; + module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs index 5c1e2e7a89..88718e56c7 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs @@ -1,17 +1,56 @@ //! A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. -use crate::analysis::_helpers::schema_request; +use crate::analysis::_helpers::schema_request::GENERIC_DIALECT; use crate::analysis::models::SchemaRequestParts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql; +use sqlbuild_analysis::semantic_validation::models::NormalizationInput; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use std::collections::HashMap; /// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. -#[inline] pub fn schema_validation_request<'py>( py: Python<'py>, parts: SchemaRequestParts<'py>, ) -> PyResult> { - schema_request::schema_validation_request(py, parts) + let dialect: String = if parts.dialect.is_truthy()? { + parts.dialect.extract()? + } else { + GENERIC_DIALECT.to_owned() + }; + let placeholders: HashMap = if parts.placeholders.is_truthy()? { + parts.placeholders.extract()? + } else { + HashMap::new() + }; + let input: NormalizationInput = NormalizationInput { + sql: parts.query_sql, + dialect, + stubs: HashMap::new(), + placeholders, + }; + let sql: String = py + .compiler_detach(|| normalize_analysis_sql(input)) + .map_err(value_error)?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", sql)?; + arguments.set_item("dialect", parts.dialect)?; + arguments.set_item("schema", parts.schema)?; + arguments.set_item("known_functions", parts.known_functions)?; + arguments.set_item("known_types", parts.known_types)?; + arguments.set_item( + "quoted_identifiers_ignore_case", + parts.quoted_identifiers_ignore_case, + )?; + arguments.set_item("catalog", parts.catalog)?; + new_record( + py, + RecordId::SqlSchemaValidationRequest, + (), + Some(&arguments), + ) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs index f7badf212c..95873c6a8b 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs @@ -1,10 +1,7 @@ //! Binding diagnostics for a batch of complete-schema validation requests, in request order. use crate::analysis::_helpers::binding_catalog; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; /// Binding diagnostics for a batch of complete-schema validation requests, in request order. diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs index 62330a8029..ae8acc15c0 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs @@ -2,10 +2,10 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; use sqlbuild_analysis::assembly::analysis_session::models::ProjectionFacts; use sqlbuild_core::panics::main::native_failure::native_failure; @@ -90,7 +90,7 @@ pub(crate) fn infer_fixture_column_facts<'py>( /// Fixture column inferences computed in this process, optionally clearing the count first. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_inference_count(py: Python<'_>, reset: bool) -> PyResult { +pub(crate) fn fixture_inference_count(py: Python<'_>, reset: bool) -> PyResult { let count = inference_count(py)?; let mut count = count.borrow_mut(); if reset { @@ -106,8 +106,3 @@ fn inference_count(py: Python<'_>) -> PyResult> .bind(py) .clone()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?)?; - module.add_function(wrap_pyfunction!(fixture_inference_count, module)?) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs index 11424c4f1a..3c5b046325 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs @@ -1,25 +1,19 @@ //! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. -use crate::analysis_session::constants::ADAPTER_RULE_ID; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_host::entries::models::AdapterHost; -use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; use crate::analysis::constants::NATIVE_DIALECT_ALIASES; -use crate::analysis::models::BindingCatalog; -use crate::analysis::models::CatalogKey; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::record_class::record_class; +use crate::analysis::models::{BindingCatalog, CatalogKey}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; -const GENERIC_DIALECT: &str = "generic"; +pub(crate) const GENERIC_DIALECT: &str = "generic"; /// The nullability rules the native analysis knows, with their native rule ids. -const KNOWN_RULES: [(AdapterHost, &str); 3] = [ +pub(crate) const KNOWN_RULES: [(AdapterHost, &str); 3] = [ (AdapterHost::FirstArgNullability, "first_arg"), ( AdapterHost::ConditionalResultNullability, @@ -31,56 +25,6 @@ const KNOWN_RULES: [(AdapterHost, &str); 3] = [ /// The adapter's own nullability rules and `InferredNullability`, which native analysis calls. pub(crate) type AdapterRules = Option<(Py, Py)>; -/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. -pub(crate) fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { - let py: Python<'_> = profile.py(); - let known: Bound<'_, PyDict> = PyDict::new(py); - for (rule, id) in KNOWN_RULES { - known.set_item(host_entry(py, rule)?, id)?; - } - let adapter: Bound<'_, PyDict> = PyDict::new(py); - let mut rows: Pairs = Vec::new(); - for item in native_field(profile, "function_nullability_rules")? - .call_method0("items")? - .try_iter()? - { - let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - let id: String = match known.get_item(&rule)? { - Some(id) => id.extract()?, - None => { - adapter.set_item(&name, &rule)?; - ADAPTER_RULE_ID.to_owned() - } - }; - rows.push((name.extract()?, id)); - } - if adapter.is_empty() { - return Ok((rows, None)); - } - let nullability: Bound<'_, PyAny> = record_class(py, RecordId::InferredNullability)?; - Ok((rows, Some((adapter.unbind(), nullability.unbind())))) -} - -/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. -pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { - let mut rows: Shapes = Vec::new(); - for item in mapping_items(shapes)?.try_iter()? { - let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; - rows.push((name, text_pairs(&shape)?)); - } - Ok(rows) -} - -/// `{column: value}` as ordered pairs, each value as `str(value)`. -pub(crate) fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { - let mut pairs: Pairs = Vec::new(); - for column in mapping_items(shape)?.try_iter()? { - let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; - pairs.push((column, value.str()?.extract()?)); - } - Ok(pairs) -} - /// A `{key: value}` text mapping's items in order. pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { mapping_items(mapping)? @@ -89,19 +33,6 @@ pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { .collect() } -/// `profile.sql_analysis_dialect or "generic"`. -pub(crate) fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { - dialect_or_generic(&native_field(profile, "sql_analysis_dialect")?) -} - -/// `dialect or "generic"`. -pub(crate) fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { - if dialect.is_truthy()? { - return dialect.extract(); - } - Ok(GENERIC_DIALECT.to_owned()) -} - /// Python's `NATIVE_DIALECT_ALIASES.get(dialect, dialect)`. pub(crate) fn aliased_dialect(_py: Python<'_>, dialect: String) -> PyResult { Ok(NATIVE_DIALECT_ALIASES @@ -110,21 +41,6 @@ pub(crate) fn aliased_dialect(_py: Python<'_>, dialect: String) -> PyResult, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - if native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()? { - return Ok(false); - } - if dialect.is_none() { - return Ok(false); - } - let dialect: String = dialect.extract()?; - Ok(CASE_SENSITIVE_BINDING_DIALECTS.contains(&dialect.as_str())) -} - /// `profile.binding_catalog or create_binding_catalog(...)` with no functions, types or relations. pub(crate) fn profile_catalog<'py>( profile: &Bound<'py, PyAny>, @@ -148,8 +64,3 @@ pub(crate) fn profile_catalog<'py>( )?; Ok(Bound::new(py, catalog)?.into_any()) } - -/// `profile.function_return_types` items in order. -pub(crate) fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { - pairs(&native_field(profile, "function_return_types")?) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs index 903c5ad29a..2ed7350994 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs @@ -1,10 +1,10 @@ //! Python's per-query column analysis outside the session, run natively. +use crate::analysis_session::main::query_columns_error::query_columns_error; use crate::analysis_session::types::ColumnsRow; -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::prelude::{Py, PyAny, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::analysis_session::errors::QueryColumnsFailure; use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; use sqlbuild_analysis::assembly::analysis_session::main::query_columns::query_columns; use sqlbuild_analysis::assembly::analysis_session::models::{ @@ -16,13 +16,13 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::positional_record; use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::nullability_rules::RuleFailure; -use crate::analysis_session::_helpers::nullability_rules::nullability_callback; -use crate::analysis_session::_helpers::nullability_rules::raised; +use crate::analysis_session::_helpers::nullability_rules::{ + RuleFailure, nullability_callback, raised, +}; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; -const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; +pub(crate) const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; positional_record! { @@ -103,16 +103,6 @@ pub(crate) fn infer_query_columns( .collect()) } -/// A normalization rejection as Python's `ValueError`; anything else as an internal failure. -pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { - match failure { - QueryColumnsFailure::Normalization(message) => compiler_error(message), - QueryColumnsFailure::Internal(reason) => { - compiler_error(native_failure(QUERY_COLUMNS_CONTEXT, &reason)) - } - } -} - fn query_input(row: QueryRow) -> Result { let QueryRow { sql, @@ -151,7 +141,7 @@ fn columns_row(columns: QueryColumns) -> ColumnsRow { /// Python's `_simple_projection_facts`: `(types, NULL literal names, quoted names)`, casefolded. #[pyfunction] -fn fixture_projection_facts( +pub(crate) fn fixture_projection_facts( py: Python<'_>, sql: &str, dialect: Option<&str>, @@ -162,11 +152,6 @@ fn fixture_projection_facts( Ok((facts.types, facts.null_literal_names, facts.quoted_names)) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; - Ok(()) -} - fn column_row(column: ColumnFact) -> (String, Option, String) { (column.name, column.data_type, column.nullability) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs index d5d180b943..bfe801e6b7 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs @@ -1,25 +1,18 @@ //! The native model analysis session and expression-source shapes. -use crate::analysis_session::models::CacheStatsRow; use crate::analysis_session::models::FamilyRow; use crate::analysis_session::models::ModelRow; -use crate::analysis_session::models::NativeModelAnalysisSession; -use crate::analysis_session::models::PivotRequestRow; use crate::analysis_session::models::ProofRow; use crate::analysis_session::models::RequestRow; -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::prelude::{Py, PyAny, PyErr, PyRef, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict; -use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::model_reference_facts::model_reference_facts; use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; -use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; -use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use sqlbuild_analysis::assembly::analysis_session::models::{ - AnalysisSession, ColumnFact, DynamicFamily, ExpressionShape, ExpressionShapeRequest, - LineageFacts, LineageRow, ModelOutcome, ModelRequest, PivotBatchRequest, PivotModel, - PivotOutcome, PivotTables, SessionRequest, SessionStep, + ColumnFact, DynamicFamily, ExpressionShape, ExpressionShapeRequest, LineageFacts, LineageRow, + ModelOutcome, ModelRequest, PivotModel, PivotOutcome, SessionRequest, SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; @@ -29,15 +22,13 @@ use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; -use std::path::PathBuf; use crate::analysis::models::Positional; use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::nullability_rules::RuleFailure; -use crate::analysis_session::_helpers::nullability_rules::nullability_callback; -use crate::analysis_session::_helpers::nullability_rules::raised; +use crate::analysis_session::_helpers::nullability_rules::{ + RuleFailure, nullability_callback, raised, +}; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::cache::main::open_store::open_store; use sqlbuild_python_base::types::CompilerDetach; /// `(name, type, nullability)`. @@ -97,7 +88,7 @@ pub(crate) type FinishRow = (Shapes, Vec, Vec); const SESSION_CONTEXT: &str = "native model analysis"; const SHAPES_CONTEXT: &str = "native expression-source shapes"; /// The native store kind holding finished model analyses. -const MODEL_ANALYSIS_STORE_KIND: &str = "model-analyses"; +pub(crate) const MODEL_ANALYSIS_STORE_KIND: &str = "model-analyses"; /// An internal native failure of the session as `NativeCompilerError`. pub(crate) fn internal_error(reason: &str) -> PyErr { @@ -107,59 +98,6 @@ pub(crate) fn internal_error(reason: &str) -> PyErr { compiler_error(native_failure(SESSION_CONTEXT, reason)) } -/// Start a session on `catalog`. -pub(crate) fn start_model_analysis_session( - catalog: PyRef<'_, ProjectCatalog>, - request: RequestRow, - cache: Option<(PathBuf, String)>, - adapter_rules: Option<(Py, Py)>, -) -> PyResult { - let py: Python<'_> = catalog.py(); - let mut request: SessionRequest = - sqlbuild_python_base::compile_context::main::delta::inject_relations::inject_relations( - session_request(request), - )?; - let rule_failure: RuleFailure = RuleFailure::default(); - let dialect: String = request.dialect.clone(); - request.nullability_callback = adapter_rules - .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); - let catalog = &catalog.inner; - let mut session: AnalysisSession = py - .compiler_detach(|| start_analysis_session(request, catalog)) - .map_err(|reason| internal_error(&reason))?; - let cache: Option<(PathBuf, String)> = cache.filter(|_| { - sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() - }); - let mut cache_path: Option = None; - let mut cache_stats: Option = None; - if let Some((path, environment)) = cache { - match open_store(py, &path, MODEL_ANALYSIS_STORE_KIND, &environment) { - Ok(store) => { - attach_analysis_cache(&mut session, store); - cache_path = Some(path); - } - Err(error) => { - cache_stats = Some(CacheStatsRow { - hits: 0, - misses: 0, - stored: 0, - failure: Some(error.to_string()), - }); - } - } - } - Ok(NativeModelAnalysisSession { - inner: Some(session), - finished: None, - outcomes: Vec::new(), - sharing: (0, 0), - cache_path, - cache_stats, - rule_failure, - dialect, - }) -} - /// Each expression's shape, `None` where it has none; adapter rules run as in the session. #[pyfunction] #[pyo3(signature = (catalog, request, adapter_rules=None))] @@ -202,7 +140,7 @@ pub(crate) fn infer_expression_source_shapes( .collect()) } -fn session_request(request: RequestRow) -> SessionRequest { +pub(crate) fn session_request(request: RequestRow) -> SessionRequest { let RequestRow { dialect, case_sensitive_shapes, @@ -357,36 +295,6 @@ pub(crate) fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> Outcome } } -/// Each model's dynamic pivot proof outside any session. -#[pyfunction] -pub(crate) fn prove_dynamic_column_contracts( - py: Python<'_>, - request: PivotRequestRow, -) -> PyResult> { - let PivotRequestRow { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: by_table, - models, - } = request; - let request = PivotBatchRequest { - tables: PivotTables { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: families_by_table(by_table), - }, - models: models.into_iter().map(pivot_model).collect(), - }; - let outcomes: Vec = py - .compiler_detach(|| prove_dynamic_contracts(&request)) - .map_err(|reason| internal_error(&reason))?; - Ok(outcomes.into_iter().map(contract_row).collect()) -} - pub(crate) fn pivot_model(row: PivotModelRow) -> PivotModel { let (sql, families) = row; PivotModel { @@ -394,8 +302,3 @@ pub(crate) fn pivot_model(row: PivotModelRow) -> PivotModel { families: families.into_iter().map(dynamic_family).collect(), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs index 8d8e4a1013..927e17c6b8 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs @@ -1,43 +1,34 @@ //! Query columns, expression-source shapes and dynamic pivot proofs read from Python objects. +use crate::analysis_session::main::case_sensitive_shapes::case_sensitive_shapes; +use crate::analysis_session::main::dialect_or_generic::dialect_or_generic; +use crate::analysis_session::main::nullability_rules::nullability_rules; +use crate::analysis_session::main::profile_dialect::profile_dialect; +use crate::analysis_session::main::prove_dynamic_column_contracts::prove_dynamic_column_contracts; +use crate::analysis_session::main::return_types::return_types; +use crate::analysis_session::main::shape_rows::shape_rows; use pyo3::IntoPyObjectExt; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::profile_inputs::aliased_dialect; -use crate::analysis_session::_helpers::profile_inputs::case_sensitive_shapes; -use crate::analysis_session::_helpers::profile_inputs::dialect_or_generic; -use crate::analysis_session::_helpers::profile_inputs::nullability_rules; -use crate::analysis_session::_helpers::profile_inputs::profile_catalog; -use crate::analysis_session::_helpers::profile_inputs::profile_dialect; -use crate::analysis_session::_helpers::profile_inputs::return_types; -use crate::analysis_session::_helpers::profile_inputs::shape_rows; -use crate::analysis_session::_helpers::query_columns::QueryRequestRow; -use crate::analysis_session::_helpers::query_columns::QueryRow; -use crate::analysis_session::_helpers::query_columns::infer_query_columns; -use crate::analysis_session::_helpers::session::ContractRow; -use crate::analysis_session::_helpers::session::ExpressionShapesRow; -use crate::analysis_session::_helpers::session::PivotModelRow; -use crate::analysis_session::_helpers::session::infer_expression_source_shapes; -use crate::analysis_session::_helpers::session::prove_dynamic_column_contracts; -use crate::analysis_session::models::FamilyRow; -use crate::analysis_session::models::NativeModelAnalysisSession; -use crate::analysis_session::models::PivotRequestRow; +use crate::analysis_session::_helpers::profile_inputs::{aliased_dialect, profile_catalog}; +use crate::analysis_session::_helpers::query_columns::{ + QueryRequestRow, QueryRow, infer_query_columns, +}; +use crate::analysis_session::_helpers::session::{ + ContractRow, ExpressionShapesRow, PivotModelRow, infer_expression_source_shapes, +}; +use crate::analysis_session::models::{FamilyRow, NativeModelAnalysisSession, PivotRequestRow}; use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPES_INFERRED; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; const BATCH_MODE: &str = "batch"; @@ -54,7 +45,7 @@ const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ /// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. #[pyfunction] #[pyo3(signature = (**request))] -fn infer_native_query_column_rows<'py>( +pub(crate) fn infer_native_query_column_rows<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("infer_native_query_column_rows", request)?; @@ -68,71 +59,6 @@ fn infer_native_query_column_rows<'py>( ) } -/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. -#[pyfunction] -#[pyo3(signature = (**request))] -pub(crate) fn infer_native_query_columns<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("infer_native_query_columns", request)?; - let py: Python<'py> = request.py(); - let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); - let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; - let nullability: Bound<'py, PyAny> = - or_empty(request.optional("column_nullability_by_table")?, &empty)?; - let rows: Vec = query_column_rows( - &request.get("queries")?, - &request.get("profile")?, - (&types, &nullability), - )?; - let mut analyses: Vec> = Vec::with_capacity(rows.len()); - for (succeeded, columns, has_star) in rows { - let columns: Bound<'py, PyAny> = match columns { - None => py.None().into_bound(py), - Some(columns) => { - let mut built: Vec> = Vec::with_capacity(columns.len()); - for (name, data_type, nullability) in columns { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("name", name)?; - arguments.set_item("type", data_type)?; - arguments.set_item( - "nullability", - enum_from_value(py, RecordId::InferredNullability, nullability)?, - )?; - built.push(new_record( - py, - RecordId::InferredColumn, - (), - Some(&arguments), - )?); - } - PyTuple::new(py, built)?.into_any() - } - }; - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("succeeded", succeeded)?; - arguments.set_item("columns", columns)?; - arguments.set_item("has_star", has_star)?; - analyses.push(new_record( - py, - RecordId::NativeQueryColumns, - (), - Some(&arguments), - )?); - } - PyTuple::new(py, analyses) -} - -/// One shape per expression, cached on the profile's binding catalog; a failure raises. -#[pyfunction] -#[pyo3(signature = (**request))] -pub(crate) fn infer_native_expression_source_shapes<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("infer_native_expression_source_shapes", request)?; - expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) -} - /// One shape per expression, cached on `profile`'s binding catalog; a failure raises. pub(crate) fn expression_source_shapes<'py>( expressions: &Bound<'py, PyAny>, @@ -199,7 +125,7 @@ pub(crate) fn expression_source_shapes<'py>( /// The proof row per `(sql, families)` model from the session or `tables`, else None. #[pyfunction] #[pyo3(signature = (**request))] -fn prove_native_dynamic_contracts<'py>( +pub(crate) fn prove_native_dynamic_contracts<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("prove_native_dynamic_contracts", request)?; @@ -268,7 +194,7 @@ pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult( +pub(crate) fn or_empty<'py>( value: Option>, empty: &Bound<'py, PyAny>, ) -> PyResult> { @@ -387,14 +313,3 @@ impl OrderedDict for Pairs { Ok(dict.into_any()) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; - module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; - module.add_function(wrap_pyfunction!( - infer_native_expression_source_shapes, - module - )?)?; - module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs index ebae1e5056..ae616cee72 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs @@ -1,15 +1,20 @@ //! Python's `inferred_binding_shape` test for keeping authored identifier quoting. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Python's `inferred_binding_shape` test for keeping authored identifier quoting. -#[inline] pub fn case_sensitive_shapes( profile: &Bound<'_, PyAny>, dialect: &Bound<'_, PyAny>, ) -> PyResult { - profile_inputs::case_sensitive_shapes(profile, dialect) + if native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()? { + return Ok(false); + } + if dialect.is_none() { + return Ok(false); + } + let dialect: String = dialect.extract()?; + Ok(CASE_SENSITIVE_BINDING_DIALECTS.contains(&dialect.as_str())) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs index 867182823e..a9cd9ce776 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs @@ -1,12 +1,12 @@ //! `dialect or "generic"`. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::profile_inputs::GENERIC_DIALECT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `dialect or "generic"`. -#[inline] pub fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { - profile_inputs::dialect_or_generic(dialect) + if dialect.is_truthy()? { + return dialect.extract(); + } + Ok(GENERIC_DIALECT.to_owned()) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs index 7c4d3fc00c..eb5af66909 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs @@ -1,9 +1,7 @@ //! One shape per expression, cached on `profile`'s binding catalog; a failure raises. use crate::analysis_session::_helpers::stage_functions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; /// One shape per expression, cached on `profile`'s binding catalog; a failure raises. diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs index c4cd1c4327..203ec0e8f8 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs @@ -1,15 +1,17 @@ //! One shape per expression, cached on the profile's binding catalog; a failure raises. -use crate::analysis_session::_helpers::stage_functions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::analysis_session::_helpers::stage_functions::expression_source_shapes; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; /// One shape per expression, cached on the profile's binding catalog; a failure raises. -#[inline] +#[pyfunction] +#[pyo3(signature = (**request))] pub fn infer_native_expression_source_shapes<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - stage_functions::infer_native_expression_source_shapes(request) + let request = Request::new("infer_native_expression_source_shapes", request)?; + expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs index 67d5659bc2..42e356a67a 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs @@ -1,15 +1,67 @@ //! Each query's columns as `NativeQueryColumns`; a native internal failure raises. -use crate::analysis_session::_helpers::stage_functions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::analysis_session::_helpers::stage_functions::or_empty; +use crate::analysis_session::_helpers::stage_functions::query_column_rows; +use crate::analysis_session::types::ColumnsRow; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// Each query's columns as `NativeQueryColumns`; a native internal failure raises. -#[inline] +#[pyfunction] +#[pyo3(signature = (**request))] pub fn infer_native_query_columns<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - stage_functions::infer_native_query_columns(request) + let request = Request::new("infer_native_query_columns", request)?; + let py: Python<'py> = request.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + let rows: Vec = query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&types, &nullability), + )?; + let mut analyses: Vec> = Vec::with_capacity(rows.len()); + for (succeeded, columns, has_star) in rows { + let columns: Bound<'py, PyAny> = match columns { + None => py.None().into_bound(py), + Some(columns) => { + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item( + "nullability", + enum_from_value(py, RecordId::InferredNullability, nullability)?, + )?; + built.push(new_record( + py, + RecordId::InferredColumn, + (), + Some(&arguments), + )?); + } + PyTuple::new(py, built)?.into_any() + } + }; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("succeeded", succeeded)?; + arguments.set_item("columns", columns)?; + arguments.set_item("has_star", has_star)?; + analyses.push(new_record( + py, + RecordId::NativeQueryColumns, + (), + Some(&arguments), + )?); + } + PyTuple::new(py, analyses) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs index 2f1b088234..439f65fec8 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs @@ -1,14 +1,42 @@ //! The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. -use crate::analysis_session::_helpers::profile_inputs; use crate::analysis_session::_helpers::profile_inputs::AdapterRules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::profile_inputs::KNOWN_RULES; +use crate::analysis_session::constants::ADAPTER_RULE_ID; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. -#[inline] pub fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { - profile_inputs::nullability_rules(profile) + let py: Python<'_> = profile.py(); + let known: Bound<'_, PyDict> = PyDict::new(py); + for (rule, id) in KNOWN_RULES { + known.set_item(host_entry(py, rule)?, id)?; + } + let adapter: Bound<'_, PyDict> = PyDict::new(py); + let mut rows: Pairs = Vec::new(); + for item in native_field(profile, "function_nullability_rules")? + .call_method0("items")? + .try_iter()? + { + let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let id: String = match known.get_item(&rule)? { + Some(id) => id.extract()?, + None => { + adapter.set_item(&name, &rule)?; + ADAPTER_RULE_ID.to_owned() + } + }; + rows.push((name.extract()?, id)); + } + if adapter.is_empty() { + return Ok((rows, None)); + } + let nullability: Bound<'_, PyAny> = record_class(py, RecordId::InferredNullability)?; + Ok((rows, Some((adapter.unbind(), nullability.unbind())))) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs index 02ae235604..0004287fe3 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs @@ -1,12 +1,10 @@ //! `profile.sql_analysis_dialect or "generic"`. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis_session::main::dialect_or_generic::dialect_or_generic; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `profile.sql_analysis_dialect or "generic"`. -#[inline] pub fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { - profile_inputs::profile_dialect(profile) + dialect_or_generic(&native_field(profile, "sql_analysis_dialect")?) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs index a5736753f6..434f9c01e9 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs @@ -1,16 +1,45 @@ //! Each model's dynamic pivot proof outside any session. -use crate::analysis_session::_helpers::session; use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::contract_row; +use crate::analysis_session::_helpers::session::families_by_table; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::pivot_model; use crate::analysis_session::models::PivotRequestRow; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::models::{ + PivotBatchRequest, PivotOutcome, PivotTables, +}; +use sqlbuild_python_base::types::CompilerDetach; /// Each model's dynamic pivot proof outside any session. -#[inline] +#[pyfunction] pub fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { - session::prove_dynamic_column_contracts(py, request) + let PivotRequestRow { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: by_table, + models, + } = request; + let request = PivotBatchRequest { + tables: PivotTables { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: families_by_table(by_table), + }, + models: models.into_iter().map(pivot_model).collect(), + }; + let outcomes: Vec = py + .compiler_detach(|| prove_dynamic_contracts(&request)) + .map_err(|reason| internal_error(&reason))?; + Ok(outcomes.into_iter().map(contract_row).collect()) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs index 92b0605f52..7695575ae0 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs @@ -2,9 +2,7 @@ use crate::analysis_session::_helpers::stage_functions; use crate::analysis_session::types::ColumnsRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Query column rows. #[inline] diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs index 7f5390de5a..830d37bf47 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs @@ -1,11 +1,17 @@ //! A normalization rejection as Python's `ValueError`; anything else as an internal failure. -use crate::analysis_session::_helpers::query_columns; +use crate::analysis_session::_helpers::query_columns::QUERY_COLUMNS_CONTEXT; use pyo3::PyErr; use sqlbuild_analysis::assembly::analysis_session::errors::QueryColumnsFailure; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; /// A normalization rejection as Python's `ValueError`; anything else as an internal failure. -#[inline] pub fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { - query_columns::query_columns_error(failure) + match failure { + QueryColumnsFailure::Normalization(message) => compiler_error(message), + QueryColumnsFailure::Internal(reason) => { + compiler_error(native_failure(QUERY_COLUMNS_CONTEXT, &reason)) + } + } } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs index 4c0001ab40..e80dec0138 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs @@ -1,12 +1,12 @@ //! Register the fixture columns functions and classes. -use crate::analysis_session::_helpers::fixture_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::fixture_columns::fixture_inference_count; +use crate::analysis_session::_helpers::fixture_columns::infer_fixture_column_facts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fixture columns functions and classes. -#[inline] pub fn register_fixture_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { - fixture_columns::register(module) + module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?)?; + module.add_function(wrap_pyfunction!(fixture_inference_count, module)?) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs index 27e3a80089..b6aee381b3 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs @@ -1,12 +1,11 @@ //! Register the query columns functions and classes. -use crate::analysis_session::_helpers::query_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::query_columns::fixture_projection_facts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the query columns functions and classes. -#[inline] pub fn register_query_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { - query_columns::register(module) + module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs index 7f615b3e62..175ff571d5 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs @@ -1,12 +1,10 @@ //! Register the session functions and classes. -use crate::analysis_session::_helpers::session; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis_session::models::NativeModelAnalysisSession; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the session functions and classes. -#[inline] pub fn register_session(module: &Bound<'_, PyModule>) -> PyResult<()> { - session::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs index 12cff3a33a..8ccb4adce7 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs @@ -1,12 +1,20 @@ //! Register the stage functions functions and classes. -use crate::analysis_session::_helpers::stage_functions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::stage_functions::infer_native_query_column_rows; +use crate::analysis_session::_helpers::stage_functions::prove_native_dynamic_contracts; +use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; +use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the stage functions functions and classes. -#[inline] pub fn register_stage_functions(module: &Bound<'_, PyModule>) -> PyResult<()> { - stage_functions::register(module) + module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; + module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; + module.add_function(wrap_pyfunction!( + infer_native_expression_source_shapes, + module + )?)?; + module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs index 2be87656fd..2f78eb422a 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs @@ -1,13 +1,11 @@ //! `profile.function_return_types` items in order. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis_session::_helpers::profile_inputs::pairs; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `profile.function_return_types` items in order. -#[inline] pub fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { - profile_inputs::return_types(profile) + pairs(&native_field(profile, "function_return_types")?) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs index c337119079..3772c68e18 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs @@ -1,13 +1,16 @@ //! `{relation: {column: value}}` as ordered rows, each value as `str(value)`. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::analysis_session::main::text_pairs::text_pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. -#[inline] pub fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { - profile_inputs::shape_rows(shapes) + let mut rows: Shapes = Vec::new(); + for item in mapping_items(shapes)?.try_iter()? { + let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; + rows.push((name, text_pairs(&shape)?)); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs index 078f42bb9d..7340e2cd0a 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs @@ -1,23 +1,71 @@ //! Start a session on `catalog`. use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::session; +use crate::analysis_session::_helpers::nullability_rules::{RuleFailure, nullability_callback}; +use crate::analysis_session::_helpers::session::MODEL_ANALYSIS_STORE_KIND; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::session_request; +use crate::analysis_session::models::CacheStatsRow; use crate::analysis_session::models::NativeModelAnalysisSession; use crate::analysis_session::models::RequestRow; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyRef; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Py, PyAny, PyRef, PyResult, Python}; use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; +use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::models::{AnalysisSession, SessionRequest}; +use sqlbuild_python_base::cache::main::open_store::open_store; +use sqlbuild_python_base::types::CompilerDetach; use std::path::PathBuf; /// Start a session on `catalog`. -#[inline] pub fn start_model_analysis_session( catalog: PyRef<'_, ProjectCatalog>, request: RequestRow, cache: Option<(PathBuf, String)>, adapter_rules: Option<(Py, Py)>, ) -> PyResult { - session::start_model_analysis_session(catalog, request, cache, adapter_rules) + let py: Python<'_> = catalog.py(); + let mut request: SessionRequest = + sqlbuild_python_base::compile_context::main::delta::inject_relations::inject_relations( + session_request(request), + )?; + let rule_failure: RuleFailure = RuleFailure::default(); + let dialect: String = request.dialect.clone(); + request.nullability_callback = adapter_rules + .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); + let catalog = &catalog.inner; + let mut session: AnalysisSession = py + .compiler_detach(|| start_analysis_session(request, catalog)) + .map_err(|reason| internal_error(&reason))?; + let cache: Option<(PathBuf, String)> = cache.filter(|_| { + sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() + }); + let mut cache_path: Option = None; + let mut cache_stats: Option = None; + if let Some((path, environment)) = cache { + match open_store(py, &path, MODEL_ANALYSIS_STORE_KIND, &environment) { + Ok(store) => { + attach_analysis_cache(&mut session, store); + cache_path = Some(path); + } + Err(error) => { + cache_stats = Some(CacheStatsRow { + hits: 0, + misses: 0, + stored: 0, + failure: Some(error.to_string()), + }); + } + } + } + Ok(NativeModelAnalysisSession { + inner: Some(session), + finished: None, + outcomes: Vec::new(), + sharing: (0, 0), + cache_path, + cache_stats, + rule_failure, + dialect, + }) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs index b5a37782c9..2965be6b94 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs @@ -1,13 +1,15 @@ //! `{column: value}` as ordered pairs, each value as `str(value)`. -use crate::analysis_session::_helpers::profile_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// `{column: value}` as ordered pairs, each value as `str(value)`. -#[inline] pub fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { - profile_inputs::text_pairs(shape) + let mut pairs: Pairs = Vec::new(); + for column in mapping_items(shape)?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + Ok(pairs) } diff --git a/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs index 35a959547e..830e89296f 100644 --- a/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs +++ b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs @@ -1,79 +1,31 @@ //! Audit SQL with its reference calls replaced by relation names or overrides. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use crate::references::main::render_source_read::render_source_read; -use crate::references::main::render_source_relation::render_source_relation; -use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -const REF_FUNCTION: &str = "__ref"; -const SEED_FUNCTION: &str = "__seed"; -const SOURCE_FUNCTION: &str = "__source"; +pub(crate) const REF_FUNCTION: &str = "__ref"; +pub(crate) const SEED_FUNCTION: &str = "__seed"; +pub(crate) const SOURCE_FUNCTION: &str = "__source"; /// `mapping.get(name)`, or `None` when absent or `None`. -fn lookup<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>> { +pub(crate) fn lookup<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { let value: Bound<'py, PyAny> = mapping_get(mapping, name)?; Ok(Some(value).filter(|value| !value.is_none())) } /// The qualified name of the relation location `mapping` holds for `name`, if any. -fn qualified<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>> { +pub(crate) fn qualified<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { let Some(location) = lookup(mapping, name)? else { return Ok(None); }; let qualified: Bound<'py, PyAny> = native_field(&location, "qualified_name")?; Ok(Some(qualified).filter(|qualified| !qualified.is_none())) } - -/// Render audit SQL from unresolved markers, applying relation overrides first. -#[pyfunction] -#[pyo3(signature = (**request))] -pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { - let request = Request::new("render_audit_sql", request)?; - let sql: String = request.get("unresolved_sql")?.extract()?; - let models: Bound<'_, PyAny> = request.get("model_locations")?; - let seeds: Bound<'_, PyAny> = request.get("seed_locations")?; - let sources: Bound<'_, PyAny> = request.get("source_map")?; - let adapter: Option> = request - .optional("adapter")? - .filter(|adapter| !adapter.is_none()); - let overrides: Option> = request - .optional("relation_overrides")? - .filter(|overrides| !overrides.is_none()); - let resolved: String = replace_reference_calls(&sql, REF_FUNCTION, |name| { - if let Some(overrides) = &overrides - && let Some(relation) = lookup(overrides, name)? - { - return relation.extract().map(Some); - } - let location: Option> = match lookup(&models, name)? { - Some(_) => qualified(&models, name)?, - None => qualified(&seeds, name)?, - }; - location.map(|name| name.extract()).transpose() - })?; - let resolved: String = replace_reference_calls(&resolved, SEED_FUNCTION, |name| { - qualified(&seeds, name)? - .map(|name| name.extract()) - .transpose() - })?; - replace_reference_calls(&resolved, SOURCE_FUNCTION, |name| { - let Some(entry) = lookup(&sources, name)? else { - return Ok(None); - }; - match adapter.as_ref() { - Some(adapter) => render_source_read(&entry, adapter).map(Some), - None => render_source_relation(&entry, None).map(Some), - } - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_audit_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/auditing/main/register_render.rs b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs index 4e007b92b6..96a1f2dc26 100644 --- a/crates/sqlbuild-python-project/src/auditing/main/register_render.rs +++ b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs @@ -1,12 +1,11 @@ //! Register the render functions and classes. -use crate::auditing::_helpers::render; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::auditing::main::render_audit_sql::render_audit_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the render functions and classes. -#[inline] pub fn register_render(module: &Bound<'_, PyModule>) -> PyResult<()> { - render::register(module) + module.add_function(wrap_pyfunction!(render_audit_sql, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs index 27a5ccd785..fda991b3d4 100644 --- a/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs +++ b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs @@ -1,12 +1,57 @@ //! Render audit SQL from unresolved markers, applying relation overrides first. -use crate::auditing::_helpers::render; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; +use crate::auditing::_helpers::render::REF_FUNCTION; +use crate::auditing::_helpers::render::SEED_FUNCTION; +use crate::auditing::_helpers::render::SOURCE_FUNCTION; +use crate::auditing::_helpers::render::lookup; +use crate::auditing::_helpers::render::qualified; +use crate::references::main::render_source_read::render_source_read; +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; +use sqlbuild_python_base::boundary::models::Request; /// Render audit SQL from unresolved markers, applying relation overrides first. -#[inline] +#[pyfunction] +#[pyo3(signature = (**request))] pub fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { - render::render_audit_sql(request) + let request = Request::new("render_audit_sql", request)?; + let sql: String = request.get("unresolved_sql")?.extract()?; + let models: Bound<'_, PyAny> = request.get("model_locations")?; + let seeds: Bound<'_, PyAny> = request.get("seed_locations")?; + let sources: Bound<'_, PyAny> = request.get("source_map")?; + let adapter: Option> = request + .optional("adapter")? + .filter(|adapter| !adapter.is_none()); + let overrides: Option> = request + .optional("relation_overrides")? + .filter(|overrides| !overrides.is_none()); + let resolved: String = replace_reference_calls(&sql, REF_FUNCTION, |name| { + if let Some(overrides) = &overrides + && let Some(relation) = lookup(overrides, name)? + { + return relation.extract().map(Some); + } + let location: Option> = match lookup(&models, name)? { + Some(_) => qualified(&models, name)?, + None => qualified(&seeds, name)?, + }; + location.map(|name| name.extract()).transpose() + })?; + let resolved: String = replace_reference_calls(&resolved, SEED_FUNCTION, |name| { + qualified(&seeds, name)? + .map(|name| name.extract()) + .transpose() + })?; + replace_reference_calls(&resolved, SOURCE_FUNCTION, |name| { + let Some(entry) = lookup(&sources, name)? else { + return Ok(None); + }; + match adapter.as_ref() { + Some(adapter) => render_source_read(&entry, adapter).map(Some), + None => render_source_relation(&entry, None).map(Some), + } + }) } diff --git a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs index 3f42923150..e0a418a5c9 100644 --- a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs @@ -1,24 +1,16 @@ //! A compiled model's authored cursor inputs resolved into filter and watermark roles. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyfunction; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use crate::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; -use sqlbuild_project::models::ReferenceRow; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; - -const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; +pub(crate) const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; const FILTER_ROLE: &str = "filter"; const WATERMARK_ROLE: &str = "watermark"; -const WATERMARK_STRATEGY: &str = "watermark"; -const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; +pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; +pub(crate) const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; /// Watermark-strategy cursor inputs: each relation's column under the roles it declares. -fn role_blocks<'py>( +pub(crate) fn role_blocks<'py>( authored: &Bound<'py, PyDict>, (filter, watermark): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), ) -> PyResult<()> { @@ -51,7 +43,7 @@ fn role_blocks<'py>( } /// The string-to-string entries of an authored mapping. -fn string_map<'py>(authored: &Bound<'py, PyDict>) -> PyResult> { +pub(crate) fn string_map<'py>(authored: &Bound<'py, PyDict>) -> PyResult> { let result: Bound<'py, PyDict> = PyDict::new(authored.py()); for (key, item) in authored.iter() { if key.is_instance_of::() && item.is_instance_of::() { @@ -60,55 +52,3 @@ fn string_map<'py>(authored: &Bound<'py, PyDict>) -> PyResult } Ok(result) } - -/// Resolve authored cursor input fields into explicit effective roles. -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn resolve_cursor_input_roles<'py>( - model: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let values: Bound<'py, PyAny> = native_field(model, "config")?.getattr("values")?; - let authored: Bound<'py, PyAny> = mapping_get(&values, CURSOR_INPUTS_KEY)?; - let strategy: Bound<'py, PyAny> = mapping_get(&values, "microbatch_strategy")?; - let rolling: bool = strategy.eq(ROLLING_WINDOW_STRATEGY)?; - let mut filter: Bound<'py, PyDict> = PyDict::new(py); - let mut watermark: Bound<'py, PyDict> = PyDict::new(py); - match authored.downcast::() { - Ok(authored) if strategy.eq(WATERMARK_STRATEGY)? => { - role_blocks(authored, (&filter, &watermark))?; - } - Ok(authored) => { - filter = string_map(authored)?; - if !rolling { - watermark = filter.clone(); - } - } - Err(_) => { - let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; - if cursor.is_instance_of::() { - for ReferenceRow { name, .. } in input_reference_rows(model)? { - filter.set_item(name, &cursor)?; - } - } - if !rolling { - watermark = filter.clone(); - } - } - } - new_record_with( - py, - RecordId::CursorInputRoles, - &[ - ("filter_inputs", filter.into_any()), - ("watermark_inputs", watermark.into_any()), - ], - ) -} - -/// `resolve_cursor_input_roles` as a function of `module`. -pub(crate) fn resolve_cursor_input_roles_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(resolve_cursor_input_roles, module) -} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs index 4bed4dd2c0..61edc9df4e 100644 --- a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs @@ -1,12 +1,59 @@ //! Resolve authored cursor input fields into explicit effective roles. -use crate::compile_attachment::_helpers::cursor_roles; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::cursor_roles::CURSOR_INPUTS_KEY; +use crate::compile_attachment::_helpers::cursor_roles::ROLLING_WINDOW_STRATEGY; +use crate::compile_attachment::_helpers::cursor_roles::WATERMARK_STRATEGY; +use crate::compile_attachment::_helpers::cursor_roles::role_blocks; +use crate::compile_attachment::_helpers::cursor_roles::string_map; +use crate::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; /// Resolve authored cursor input fields into explicit effective roles. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model))] pub fn resolve_cursor_input_roles<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - cursor_roles::resolve_cursor_input_roles(model) + let py: Python<'py> = model.py(); + let values: Bound<'py, PyAny> = native_field(model, "config")?.getattr("values")?; + let authored: Bound<'py, PyAny> = mapping_get(&values, CURSOR_INPUTS_KEY)?; + let strategy: Bound<'py, PyAny> = mapping_get(&values, "microbatch_strategy")?; + let rolling: bool = strategy.eq(ROLLING_WINDOW_STRATEGY)?; + let mut filter: Bound<'py, PyDict> = PyDict::new(py); + let mut watermark: Bound<'py, PyDict> = PyDict::new(py); + match authored.downcast::() { + Ok(authored) if strategy.eq(WATERMARK_STRATEGY)? => { + role_blocks(authored, (&filter, &watermark))?; + } + Ok(authored) => { + filter = string_map(authored)?; + if !rolling { + watermark = filter.clone(); + } + } + Err(_) => { + let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; + if cursor.is_instance_of::() { + for ReferenceRow { name, .. } in input_reference_rows(model)? { + filter.set_item(name, &cursor)?; + } + } + if !rolling { + watermark = filter.clone(); + } + } + } + new_record_with( + py, + RecordId::CursorInputRoles, + &[ + ("filter_inputs", filter.into_any()), + ("watermark_inputs", watermark.into_any()), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs index 7eecc98d17..d2eddceeb5 100644 --- a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs @@ -1,11 +1,10 @@ //! `resolve_cursor_input_roles` as a function of `module`. -use crate::compile_attachment::_helpers::cursor_roles; +use crate::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; /// `resolve_cursor_input_roles` as a function of `module`. -#[inline] pub fn resolve_cursor_input_roles_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - cursor_roles::resolve_cursor_input_roles_function(module) + pyo3::wrap_pyfunction!(resolve_cursor_input_roles, module) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs index 8f9a7a6353..2792e52147 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs @@ -25,7 +25,7 @@ use crate::resource_views::models::NativeSourceInput; const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; -static INSTALLED: OnceLock = OnceLock::new(); +pub(crate) static INSTALLED: OnceLock = OnceLock::new(); /// Attaches the project's sources, given their rows and settings. pub(crate) type AttachSources = fn( @@ -64,11 +64,6 @@ pub(crate) type AttachAudits = AuditSettings, ) -> PyResult<(Vec, Vec>)>; -/// Install the compile's attachment runs once, at registration. -pub(crate) fn install(attachment: ResourceAttachment) { - let _ = INSTALLED.get_or_init(|| attachment); -} - /// The installed attachment runs. pub(crate) fn resource_attachment() -> PyResult<&'static ResourceAttachment> { INSTALLED diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 570483cbdb..9d7502e05a 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -1,6 +1,5 @@ //! The command target's jobs, loaded when a selector names one or `--changed` needs triggered jobs. -use std::borrow::Cow; use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; @@ -8,17 +7,14 @@ use std::path::PathBuf; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; -use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGraph}; +use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; -use crate::compiled_project::constants::JOB_SELECTOR_KINDS; -use crate::compiled_project::constants::SELECTOR_INTERSECTION; -use crate::compiled_project::constants::SENSOR_RUN_ARGUMENT; +use crate::compiled_project::constants::{ + JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, +}; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. @@ -53,72 +49,8 @@ pub(crate) fn with_job_scope( result } -/// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -pub(crate) fn graph_with_jobs<'g>( - py: Python<'_>, - graph: &'g ProjectGraph, - selectors: &[String], - also_known: &[&str], -) -> PyResult> { - Ok(match needed_jobs(py, graph, selectors, also_known)? { - Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), - None => Cow::Borrowed(graph), - }) -} - -/// The command's jobs when `selectors` name one and `graph` has none attached yet. -pub(crate) fn needed_jobs( - py: Python<'_>, - graph: &ProjectGraph, - selectors: &[String], - also_known: &[&str], -) -> PyResult>> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(None); - } - Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) -} - -/// The command's job and job group names, loaded on first use; empty outside a command. -pub(crate) fn job_names(py: Python<'_>) -> PyResult> { - Ok(job_catalog(py)? - .map(|(catalog, _)| catalog.into_keys().collect()) - .unwrap_or_default()) -} - -/// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -pub(crate) fn triggered_jobs( - py: Python<'_>, - graph: &ProjectGraph, - selectors: &[String], -) -> PyResult)>> { - let Some((catalog, triggered)) = job_catalog(py)? else { - return Ok(Vec::new()); - }; - if triggered.is_empty() { - return Ok(Vec::new()); - } - let running: BTreeSet = selected_jobs(selectors, &catalog); - let mut graph: ProjectGraph = attach_jobs(graph.clone(), catalog); - let mut resolved = graph.jobs.take().unwrap_or_default(); - let mut running_nodes: BTreeSet = BTreeSet::new(); - for name in &running { - if let Some(Ok(keys)) = resolved.get(name) { - running_nodes.extend(keys.iter().cloned()); - } - } - Ok(triggered - .into_iter() - .filter(|name| !running.contains(name)) - .filter_map(|name| match resolved.remove(&name) { - Some(Ok(keys)) => Some((name, &keys - &running_nodes)), - _ => None, - }) - .collect()) -} - /// The jobs `selectors` name, as `job:` or as a bare job name. -fn selected_jobs( +pub(crate) fn selected_jobs( selectors: &[String], catalog: &BTreeMap, ) -> BTreeSet { @@ -138,7 +70,7 @@ fn selected_jobs( jobs } -fn job_catalog(py: Python<'_>) -> PyResult> { +pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { let Some(scope) = JOB_SCOPE.with(|scope| { scope.borrow().as_ref().map(|scope| match &scope.catalog { Some(catalog) => Ok(catalog.clone()), diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs index 0a33885dcf..bf19c152a0 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod attachment_inputs; pub(crate) mod job_catalog; pub(crate) mod project; -pub(crate) mod rows; pub(crate) mod schema_facts; pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs index 0014567ecb..1b8f938370 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs @@ -1,41 +1,21 @@ //! The compile's natively owned project, which producers record into and consumers read. -use crate::compiled_project::constants::CONTRACT_KEY; -use crate::compiled_project::constants::INCREMENTAL_MODE_KEY; use crate::compiled_project::models::DeclarationPy; -use crate::compiled_project::models::ModelInputPy; -use crate::compiled_project::models::NativeCompiledProject; use sqlbuild_python_base::positional_record; use std::collections::BTreeMap; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::prelude::PyResult; use serde_json::Value; -use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, DeclarationFact, ModelConfigFacts, ModelReferenceFact, -}; +use sqlbuild_analysis::compiled_project::models::DeclarationFact; -use crate::compiled_project::_helpers::values::object_value; use crate::compiled_project::_helpers::values::plain_value; -use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeInputsView; -use crate::compiled_views::models::NativeModelInput; -use crate::compiled_views::models::NativeProjectView; -use crate::resource_views::models::NativeAuditInput; -use crate::resource_views::models::NativeCompiledAudit; -use crate::resource_views::models::NativeCompiledFunction; -use crate::resource_views::models::NativeCompiledSeed; -use crate::resource_views::models::NativeCompiledSource; -use crate::resource_views::models::NativeFunctionInput; -use crate::resource_views::models::NativeSeedInput; -use crate::resource_views::models::NativeSourceInput; pub(crate) const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; pub(crate) const ATTACHMENT_LOOP_UNSET: &str = "no model loop was kept to attach this project's resources"; -const MATERIALIZED_KEY: &str = "materialized"; -const TABLE_MATERIALIZATION: &str = "table"; +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const TABLE_MATERIALIZATION: &str = "table"; /// `(name, type)`. pub(crate) type NamedType = (String, Option); @@ -56,84 +36,6 @@ positional_record! { /// Encoded config values and the encoder's error, if any. pub(crate) type ConfigValues = (BTreeMap, Option); -pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { - let ModelInputPy( - ModelTextsPy { - name, - relative_path, - query_sql, - authored_query_sql, - authored_sql, - }, - (values, header_keys, layer_schema), - references, - enum_columns, - (enum_declarations, constant_declarations), - ) = model; - let config_values: ConfigValues = config_values(&values); - Ok(CompiledModelFacts { - name, - relative_path, - query_sql: query_sql.into(), - authored_query_sql: authored_query_sql.into(), - authored_sql: authored_sql.into(), - config: ModelConfigFacts { - values: config_values.0, - encoding_error: config_values.1, - header_keys, - layer_schema, - contract: text_value(&values, CONTRACT_KEY)?, - materialized: materialization(&values)?, - incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, - }, - references: references - .into_iter() - .map(|(kind, name, package)| ModelReferenceFact { - kind, - name, - package, - }) - .collect(), - schema: None, - enum_columns, - enum_declarations: declarations(enum_declarations)?, - constant_declarations: declarations(constant_declarations)?, - analysis: None, - typed_calls: Vec::new(), - expansion_passes: Vec::new(), - sql_analysis_skipped: false, - }) -} - -/// The config values, or the encoder's error, kept for the consumer that reads them. -pub(crate) fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { - match object_value(values) { - Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), - Ok(_) => (BTreeMap::new(), None), - Err(error) => (BTreeMap::new(), Some(error.to_string())), - } -} - -/// The `materialized` config value when it is text, `table` when it is absent. -pub(crate) fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { - if values.get_item(MATERIALIZED_KEY)?.is_none() { - return Ok(Some(TABLE_MATERIALIZATION.to_owned())); - } - text_value(values, MATERIALIZED_KEY) -} - -/// A config value when it is text. -pub(crate) fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match values.get_item(key)? { - Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), - _ => Ok(None), - } -} - -pub(crate) fn declarations(rows: Vec>) -> PyResult> { - rows.into_iter().map(declaration).collect() -} - pub(crate) fn declaration(row: DeclarationPy<'_>) -> PyResult { let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; Ok(DeclarationFact { @@ -148,19 +50,3 @@ pub(crate) fn declaration(row: DeclarationPy<'_>) -> PyResult { render_as, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs deleted file mode 100644 index 8de140100b..0000000000 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/rows.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Positional rows shared by the compiled views and the input loops that fill them. - -use crate::compiled_project::models::ReferenceTuple; -use sqlbuild_project::models::ReferenceRow; - -/// A reference as the tuple Python reads. -pub(crate) fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { - let ReferenceRow { - kind, - name, - package, - call_argument_count, - } = row; - ReferenceTuple { - kind, - name, - package, - call_argument_count, - } -} diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs index d22d801145..4bb65d4779 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs @@ -1,122 +1,31 @@ //! Native facts of declared schema columns and families, read once from their Python specs. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::compiled_project::models::{ - DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, -}; -use sqlbuild_core::text::main::python_casefold::python_casefold; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::record_attribute_opt::record_attribute_opt; -const NAME_FIELD: &str = "name"; -const TYPE_FIELD: &str = "type"; -const NULLABLE_FIELD: &str = "nullable"; -const DESCRIPTION_FIELD: &str = "description"; -const MIGRATE_FROM_FIELD: &str = "migrate_from"; -const AUDITS_FIELD: &str = "audits"; -const LOCATION_FIELD: &str = "location"; -const PATH_FIELD: &str = "path"; -const DEFINITION_NAME_FIELD: &str = "definition_name"; -const COLUMNS_FIELD: &str = "columns"; -const DYNAMIC_COLUMNS_FIELD: &str = "dynamic_columns"; -const TYPE_ENFORCEMENT_FIELD: &str = "type_enforcement"; -const MODEL_SCHEMA_FIELD: &str = "model_schema"; -const PIVOT_COLUMN_FIELD: &str = "pivot_column"; -const VALUE_COLUMN_FIELD: &str = "value_column"; -const AGGREGATE_FIELD: &str = "aggregate"; -const NAME_PATTERN_FIELD: &str = "name_pattern"; - -/// A Python column spec's fact; its location is kept when it differs from `model_path`. -pub(crate) fn python_column_fact( - column: &Bound<'_, PyAny>, - model_path: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let mut audit_definitions: Vec = Vec::new(); - if let Some(audits) = present(column, AUDITS_FIELD)? { - for audit in audits.try_iter()? { - audit_definitions.push(native_field(&audit?, DEFINITION_NAME_FIELD)?.extract()?); - } - } - let location_path: Option = match present(column, LOCATION_FIELD)? { - Some(location) => { - let path: Bound<'_, PyAny> = native_field(&location, PATH_FIELD)?; - let outside: bool = match model_path { - Some(model_path) => !path.is(model_path) && path.ne(model_path)?, - None => true, - }; - outside - .then(|| path.str().map(|text| text.to_string())) - .transpose()? - } - None => None, - }; - Ok(SchemaColumnFact { - name: native_field(column, NAME_FIELD)?.str()?.extract()?, - data_type: optional(column, TYPE_FIELD)?, - nullable: optional(column, NULLABLE_FIELD)?, - described: present(column, DESCRIPTION_FIELD)?.is_some(), - migrate_from: optional(column, MIGRATE_FROM_FIELD)?, - audit_definitions, - location_path, - }) -} - -/// The facts of every column spec in `columns`. -pub(crate) fn python_column_facts( - columns: &Bound<'_, PyAny>, - model_path: Option<&Bound<'_, PyAny>>, -) -> PyResult> { - columns - .try_iter()? - .map(|column| python_column_fact(&column?, model_path)) - .collect() -} - -/// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. -pub(crate) fn entry_schema_fact( - entry: &Bound<'_, PyAny>, - model_path: &Bound<'_, PyAny>, -) -> PyResult> { - if entry.is_none() { - return Ok(None); - } - let mut dynamic_columns: Vec = Vec::new(); - for family in native_field(entry, DYNAMIC_COLUMNS_FIELD)?.try_iter()? { - let family: Bound<'_, PyAny> = family?; - dynamic_columns.push(family_fact(&DynamicFamily { - name: native_field(&family, NAME_FIELD)?.extract()?, - pivot_column: native_field(&family, PIVOT_COLUMN_FIELD)?.extract()?, - value_column: native_field(&family, VALUE_COLUMN_FIELD)?.extract()?, - aggregate: native_field(&family, AGGREGATE_FIELD)?.extract()?, - data_type: native_field(&family, TYPE_FIELD)?.extract()?, - name_pattern: native_field(&family, NAME_PATTERN_FIELD)?.extract()?, - })); - } - Ok(Some(ModelSchemaFact { - audit_count: u32::try_from(native_field(entry, AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), - columns: python_column_facts(&native_field(entry, COLUMNS_FIELD)?, Some(model_path))?, - dynamic_columns, - type_enforcement: native_field(entry, TYPE_ENFORCEMENT_FIELD)?.is_truthy()?, - named_schema: !native_field(entry, MODEL_SCHEMA_FIELD)?.is_none(), - })) -} - -/// A parsed dynamic column family's fact, keyed by its casefolded name. -pub(crate) fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { - DynamicFamilyFact { - key: python_casefold(&family.name), - name: family.name.clone(), - pivot_column: family.pivot_column.clone(), - value_column: family.value_column.clone(), - aggregate: family.aggregate.clone(), - data_type: family.data_type.clone(), - name_pattern: family.name_pattern.clone(), - } -} +pub(crate) const NAME_FIELD: &str = "name"; +pub(crate) const TYPE_FIELD: &str = "type"; +pub(crate) const NULLABLE_FIELD: &str = "nullable"; +pub(crate) const DESCRIPTION_FIELD: &str = "description"; +pub(crate) const MIGRATE_FROM_FIELD: &str = "migrate_from"; +pub(crate) const AUDITS_FIELD: &str = "audits"; +pub(crate) const LOCATION_FIELD: &str = "location"; +pub(crate) const PATH_FIELD: &str = "path"; +pub(crate) const DEFINITION_NAME_FIELD: &str = "definition_name"; +pub(crate) const COLUMNS_FIELD: &str = "columns"; +pub(crate) const DYNAMIC_COLUMNS_FIELD: &str = "dynamic_columns"; +pub(crate) const TYPE_ENFORCEMENT_FIELD: &str = "type_enforcement"; +pub(crate) const MODEL_SCHEMA_FIELD: &str = "model_schema"; +pub(crate) const PIVOT_COLUMN_FIELD: &str = "pivot_column"; +pub(crate) const VALUE_COLUMN_FIELD: &str = "value_column"; +pub(crate) const AGGREGATE_FIELD: &str = "aggregate"; +pub(crate) const NAME_PATTERN_FIELD: &str = "name_pattern"; /// An attribute's value, `None` when it is missing or `None`. -fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>> { +pub(crate) fn present<'py>( + spec: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { let value: Option> = match record_attribute_opt(spec, name) { Some(value) => value?, None => spec.getattr_opt(name)?, @@ -124,7 +33,7 @@ fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>( +pub(crate) fn optional<'py, T: pyo3::FromPyObject<'py>>( spec: &Bound<'py, PyAny>, name: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs b/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs index 2574a1758a..7ea19c4621 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs @@ -1,12 +1,17 @@ //! The config values, or the encoder's error, kept for the consumer that reads them. -use crate::compiled_project::_helpers::project; use crate::compiled_project::_helpers::project::ConfigValues; +use crate::compiled_project::_helpers::values::object_value; use pyo3::prelude::Bound; use pyo3::types::PyDict; +use serde_json::Value; +use std::collections::BTreeMap; /// The config values, or the encoder's error, kept for the consumer that reads them. -#[inline] pub fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { - project::config_values(values) + match object_value(values) { + Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), + Ok(_) => (BTreeMap::new(), None), + Err(error) => (BTreeMap::new(), Some(error.to_string())), + } } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs index dcccc7d386..9e50d50c3a 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs @@ -1,12 +1,11 @@ //! Declarations. -use crate::compiled_project::_helpers::project; +use crate::compiled_project::_helpers::project::declaration; use crate::compiled_project::models::DeclarationPy; use pyo3::prelude::PyResult; use sqlbuild_analysis::compiled_project::models::DeclarationFact; /// Declarations. -#[inline] pub fn declarations(rows: Vec>) -> PyResult> { - project::declarations(rows) + rows.into_iter().map(declaration).collect() } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs index feeedc6cf6..6be3f0d8e0 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs @@ -1,16 +1,48 @@ //! The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. -use crate::compiled_project::_helpers::schema_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; +use crate::compiled_project::_helpers::schema_facts::AGGREGATE_FIELD; +use crate::compiled_project::_helpers::schema_facts::AUDITS_FIELD; +use crate::compiled_project::_helpers::schema_facts::COLUMNS_FIELD; +use crate::compiled_project::_helpers::schema_facts::DYNAMIC_COLUMNS_FIELD; +use crate::compiled_project::_helpers::schema_facts::MODEL_SCHEMA_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_PATTERN_FIELD; +use crate::compiled_project::_helpers::schema_facts::PIVOT_COLUMN_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_ENFORCEMENT_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_FIELD; +use crate::compiled_project::_helpers::schema_facts::VALUE_COLUMN_FIELD; +use crate::compiled_project::main::family_fact::family_fact; +use crate::compiled_project::main::python_column_facts::python_column_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::compiled_project::models::{DynamicFamilyFact, ModelSchemaFact}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. -#[inline] pub fn entry_schema_fact( entry: &Bound<'_, PyAny>, model_path: &Bound<'_, PyAny>, ) -> PyResult> { - schema_facts::entry_schema_fact(entry, model_path) + if entry.is_none() { + return Ok(None); + } + let mut dynamic_columns: Vec = Vec::new(); + for family in native_field(entry, DYNAMIC_COLUMNS_FIELD)?.try_iter()? { + let family: Bound<'_, PyAny> = family?; + dynamic_columns.push(family_fact(&DynamicFamily { + name: native_field(&family, NAME_FIELD)?.extract()?, + pivot_column: native_field(&family, PIVOT_COLUMN_FIELD)?.extract()?, + value_column: native_field(&family, VALUE_COLUMN_FIELD)?.extract()?, + aggregate: native_field(&family, AGGREGATE_FIELD)?.extract()?, + data_type: native_field(&family, TYPE_FIELD)?.extract()?, + name_pattern: native_field(&family, NAME_PATTERN_FIELD)?.extract()?, + })); + } + Ok(Some(ModelSchemaFact { + audit_count: u32::try_from(native_field(entry, AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), + columns: python_column_facts(&native_field(entry, COLUMNS_FIELD)?, Some(model_path))?, + dynamic_columns, + type_enforcement: native_field(entry, TYPE_ENFORCEMENT_FIELD)?.is_truthy()?, + named_schema: !native_field(entry, MODEL_SCHEMA_FIELD)?.is_none(), + })) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs index 16da422dc4..532f7edda9 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs @@ -1,11 +1,18 @@ //! A parsed dynamic column family's fact, keyed by its casefolded name. -use crate::compiled_project::_helpers::schema_facts; use sqlbuild_analysis::compiled_project::models::DynamicFamilyFact; +use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_model_config::header_metadata::models::DynamicFamily; /// A parsed dynamic column family's fact, keyed by its casefolded name. -#[inline] pub fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { - schema_facts::family_fact(family) + DynamicFamilyFact { + key: python_casefold(&family.name), + name: family.name.clone(), + pivot_column: family.pivot_column.clone(), + value_column: family.value_column.clone(), + aggregate: family.aggregate.clone(), + data_type: family.data_type.clone(), + name_pattern: family.name_pattern.clone(), + } } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs index 15f2a698f6..e8fda5ea27 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs @@ -1,18 +1,20 @@ //! `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -use crate::compiled_project::_helpers::job_catalog; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compiled_project::main::needed_jobs::needed_jobs; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::models::ProjectGraph; use std::borrow::Cow; /// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -#[inline] pub fn graph_with_jobs<'g>( py: Python<'_>, graph: &'g ProjectGraph, selectors: &[String], also_known: &[&str], ) -> PyResult> { - job_catalog::graph_with_jobs(py, graph, selectors, also_known) + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + None => Cow::Borrowed(graph), + }) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/install.rs b/crates/sqlbuild-python-project/src/compiled_project/main/install.rs index 1892d7130b..ee2e1117e2 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/install.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/install.rs @@ -1,10 +1,9 @@ //! Install the compile's attachment runs once, at registration. -use crate::compiled_project::_helpers::attachment_inputs; +use crate::compiled_project::_helpers::attachment_inputs::INSTALLED; use crate::compiled_project::models::ResourceAttachment; /// Install the compile's attachment runs once, at registration. -#[inline] pub fn install(attachment: ResourceAttachment) { - attachment_inputs::install(attachment) + let _ = INSTALLED.get_or_init(|| attachment); } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs index 9781461c25..bd3875b222 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs @@ -1,11 +1,11 @@ //! The command's job and job group names, loaded on first use; empty outside a command. -use crate::compiled_project::_helpers::job_catalog; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use pyo3::prelude::{PyResult, Python}; /// The command's job and job group names, loaded on first use; empty outside a command. -#[inline] pub fn job_names(py: Python<'_>) -> PyResult> { - job_catalog::job_names(py) + Ok(job_catalog(py)? + .map(|(catalog, _)| catalog.into_keys().collect()) + .unwrap_or_default()) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs b/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs index 8b4d7256f3..ca4573eee6 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs @@ -1,12 +1,15 @@ //! The `materialized` config value when it is text, `table` when it is absent. -use crate::compiled_project::_helpers::project; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::compiled_project::_helpers::project::MATERIALIZED_KEY; +use crate::compiled_project::_helpers::project::TABLE_MATERIALIZATION; +use crate::compiled_project::main::text_value::text_value; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; /// The `materialized` config value when it is text, `table` when it is absent. -#[inline] pub fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { - project::materialization(values) + if values.get_item(MATERIALIZED_KEY)?.is_none() { + return Ok(Some(TABLE_MATERIALIZATION.to_owned())); + } + text_value(values, MATERIALIZED_KEY) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs index 850e339ddc..ce21e47ad3 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs @@ -1,12 +1,65 @@ //! Model facts. -use crate::compiled_project::_helpers::project; +use crate::compiled_project::_helpers::project::ConfigValues; +use crate::compiled_project::_helpers::project::ModelTextsPy; +use crate::compiled_project::constants::CONTRACT_KEY; +use crate::compiled_project::constants::INCREMENTAL_MODE_KEY; +use crate::compiled_project::main::config_values::config_values; +use crate::compiled_project::main::declarations::declarations; +use crate::compiled_project::main::materialization::materialization; +use crate::compiled_project::main::text_value::text_value; use crate::compiled_project::models::ModelInputPy; use pyo3::prelude::PyResult; -use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, ModelConfigFacts, ModelReferenceFact, +}; /// Model facts. -#[inline] pub fn model_facts(model: ModelInputPy<'_>) -> PyResult { - project::model_facts(model) + let ModelInputPy( + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, + (values, header_keys, layer_schema), + references, + enum_columns, + (enum_declarations, constant_declarations), + ) = model; + let config_values: ConfigValues = config_values(&values); + Ok(CompiledModelFacts { + name, + relative_path, + query_sql: query_sql.into(), + authored_query_sql: authored_query_sql.into(), + authored_sql: authored_sql.into(), + config: ModelConfigFacts { + values: config_values.0, + encoding_error: config_values.1, + header_keys, + layer_schema, + contract: text_value(&values, CONTRACT_KEY)?, + materialized: materialization(&values)?, + incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, + }, + references: references + .into_iter() + .map(|(kind, name, package)| ModelReferenceFact { + kind, + name, + package, + }) + .collect(), + schema: None, + enum_columns, + enum_declarations: declarations(enum_declarations)?, + constant_declarations: declarations(constant_declarations)?, + analysis: None, + typed_calls: Vec::new(), + expansion_passes: Vec::new(), + sql_analysis_skipped: false, + }) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs index ebeb109627..949d525f66 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs @@ -1,19 +1,20 @@ //! The command's jobs when `selectors` name one and `graph` has none attached yet. -use crate::compiled_project::_helpers::job_catalog; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_analysis::graph::models::JobSelection; -use sqlbuild_analysis::graph::models::ProjectGraph; +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; +use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; use std::collections::BTreeMap; /// The command's jobs when `selectors` name one and `graph` has none attached yet. -#[inline] pub fn needed_jobs( py: Python<'_>, graph: &ProjectGraph, selectors: &[String], also_known: &[&str], ) -> PyResult>> { - job_catalog::needed_jobs(py, graph, selectors, also_known) + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs index 73944e7583..3103672450 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs @@ -1,9 +1,7 @@ //! orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. use crate::compiled_project::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use serde_json::Value; /// orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs index 19cdddf230..ad733482c6 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs @@ -1,16 +1,51 @@ //! A Python column spec's fact; its location is kept when it differs from `model_path`. -use crate::compiled_project::_helpers::schema_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::_helpers::schema_facts::AUDITS_FIELD; +use crate::compiled_project::_helpers::schema_facts::DEFINITION_NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::DESCRIPTION_FIELD; +use crate::compiled_project::_helpers::schema_facts::LOCATION_FIELD; +use crate::compiled_project::_helpers::schema_facts::MIGRATE_FROM_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::NULLABLE_FIELD; +use crate::compiled_project::_helpers::schema_facts::PATH_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_FIELD; +use crate::compiled_project::_helpers::schema_facts::optional; +use crate::compiled_project::_helpers::schema_facts::present; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A Python column spec's fact; its location is kept when it differs from `model_path`. -#[inline] pub fn python_column_fact( column: &Bound<'_, PyAny>, model_path: Option<&Bound<'_, PyAny>>, ) -> PyResult { - schema_facts::python_column_fact(column, model_path) + let mut audit_definitions: Vec = Vec::new(); + if let Some(audits) = present(column, AUDITS_FIELD)? { + for audit in audits.try_iter()? { + audit_definitions.push(native_field(&audit?, DEFINITION_NAME_FIELD)?.extract()?); + } + } + let location_path: Option = match present(column, LOCATION_FIELD)? { + Some(location) => { + let path: Bound<'_, PyAny> = native_field(&location, PATH_FIELD)?; + let outside: bool = match model_path { + Some(model_path) => !path.is(model_path) && path.ne(model_path)?, + None => true, + }; + outside + .then(|| path.str().map(|text| text.to_string())) + .transpose()? + } + None => None, + }; + Ok(SchemaColumnFact { + name: native_field(column, NAME_FIELD)?.str()?.extract()?, + data_type: optional(column, TYPE_FIELD)?, + nullable: optional(column, NULLABLE_FIELD)?, + described: present(column, DESCRIPTION_FIELD)?.is_some(), + migrate_from: optional(column, MIGRATE_FROM_FIELD)?, + audit_definitions, + location_path, + }) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs index 9f2a1d9d72..d0096876f3 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs @@ -1,16 +1,16 @@ //! The facts of every column spec in `columns`. -use crate::compiled_project::_helpers::schema_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::main::python_column_fact::python_column_fact; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; /// The facts of every column spec in `columns`. -#[inline] pub fn python_column_facts( columns: &Bound<'_, PyAny>, model_path: Option<&Bound<'_, PyAny>>, ) -> PyResult> { - schema_facts::python_column_facts(columns, model_path) + columns + .try_iter()? + .map(|column| python_column_fact(&column?, model_path)) + .collect() } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs index 43465df81d..252124f7ec 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs @@ -1,11 +1,20 @@ //! A reference as the tuple Python reads. -use crate::compiled_project::_helpers::rows; use crate::compiled_project::models::ReferenceTuple; use sqlbuild_project::models::ReferenceRow; /// A reference as the tuple Python reads. -#[inline] pub fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { - rows::reference_row_tuple(row) + let ReferenceRow { + kind, + name, + package, + call_argument_count, + } = row; + ReferenceTuple { + kind, + name, + package, + call_argument_count, + } } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs b/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs index d3070cd2e3..033b3ff829 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs @@ -1,12 +1,29 @@ //! Register the project functions and classes. -use crate::compiled_project::_helpers::project; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::resource_views::models::{NativeAuditInput, NativeCompiledAudit}; +use crate::resource_views::models::{NativeCompiledFunction, NativeFunctionInput}; +use crate::resource_views::models::{NativeCompiledSeed, NativeSeedInput}; +use crate::resource_views::models::{NativeCompiledSource, NativeSourceInput}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the project functions and classes. -#[inline] pub fn register_project(module: &Bound<'_, PyModule>) -> PyResult<()> { - project::register(module) + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs index 7f4fe85c39..6003013b76 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs @@ -1,12 +1,12 @@ //! A config value when it is text. -use crate::compiled_project::_helpers::project; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; /// A config value when it is text. -#[inline] pub fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - project::text_value(values, key) + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs index 39e3153282..741f09e695 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs @@ -1,18 +1,40 @@ //! Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -use crate::compiled_project::_helpers::job_catalog; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::compiled_project::_helpers::job_catalog::selected_jobs; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::models::ProjectGraph; use std::collections::BTreeSet; /// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -#[inline] pub fn triggered_jobs( py: Python<'_>, graph: &ProjectGraph, selectors: &[String], ) -> PyResult)>> { - job_catalog::triggered_jobs(py, graph, selectors) + let Some((catalog, triggered)) = job_catalog(py)? else { + return Ok(Vec::new()); + }; + if triggered.is_empty() { + return Ok(Vec::new()); + } + let running: BTreeSet = selected_jobs(selectors, &catalog); + let mut graph: ProjectGraph = attach_jobs(graph.clone(), catalog); + let mut resolved = graph.jobs.take().unwrap_or_default(); + let mut running_nodes: BTreeSet = BTreeSet::new(); + for name in &running { + if let Some(Ok(keys)) = resolved.get(name) { + running_nodes.extend(keys.iter().cloned()); + } + } + Ok(triggered + .into_iter() + .filter(|name| !running.contains(name)) + .filter_map(|name| match resolved.remove(&name) { + Some(Ok(keys)) => Some((name, &keys - &running_nodes)), + _ => None, + }) + .collect()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs index 250b283471..dca81a3464 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs @@ -1,8 +1,6 @@ //! A compiled model, read from the native compiled project that owns it. use crate::compiled_views::models::CompiledProjections; -use crate::compiled_views::models::CompiledViewClasses; -use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::types::LineageInput; use sqlbuild_project::models as project_models; use sqlbuild_project::models::DestinationFact; @@ -17,25 +15,23 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; -use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisRecord; -use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::ProofArgument; -use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::SessionArgument; -use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::analysis_record; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::{ + AnalysisRecord, ProofArgument, SessionArgument, analysis_record, +}; use crate::compiled_views::models::PyProjectHost; use crate::compiled_views::types::ModelInputRecord; -use crate::graph::models::ResourceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::models::HeaderColumns; -const DATABASE_KEY: &str = "database"; -const SCHEMA_KEY: &str = "schema"; +pub(crate) const DATABASE_KEY: &str = "database"; +pub(crate) const SCHEMA_KEY: &str = "schema"; const ALIAS_KEY: &str = "alias"; -const VALUES_ATTRIBUTE: &str = "values"; -const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; -const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; +pub(crate) const VALUES_ATTRIBUTE: &str = "values"; +pub(crate) const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; +pub(crate) const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; pub(crate) const MODEL_RESOURCE: &str = "model"; pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; -const TAGS_KEY: &str = "tags"; +pub(crate) const TAGS_KEY: &str = "tags"; pub(crate) const NULLABLE: &str = "nullable"; /// A schema's entry audits, column specs and each audited column's index and name. @@ -139,30 +135,6 @@ fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> } } -/// The destination with adapter defaults filled in; `render` renders its qualified name. -pub(crate) fn defaulted_destination( - current: &DestinationFact, - defaults: (Option, Option), - render: &Bound<'_, PyAny>, -) -> PyResult { - let (default_schema, default_database) = defaults; - let schema: Option = current.schema.clone().or(default_schema); - let database: Option = current.database.clone().or(default_database); - let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); - keywords.set_item(DATABASE_KEY, &database)?; - keywords.set_item(SCHEMA_KEY, &schema)?; - keywords.set_item("name", ¤t.name)?; - let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; - Ok(DestinationFact { - database, - schema, - name: current.name.clone(), - qualified_name, - logical_schema: current.logical_schema.clone(), - logical_database: current.logical_database.clone(), - }) -} - /// A lineage row's output column with its `(resource name, column)` sources. pub(crate) fn lineage_input(row: &LineageRow) -> LineageInput { let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); @@ -190,59 +162,6 @@ pub(crate) fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) } -/// The model's project graph row: its key, dependencies, tags and folder. -pub(crate) fn model_graph_row( - model: &NativeCompiledModel, - py: Python<'_>, -) -> PyResult { - model.read(py, |record, py| { - let compiled: &CompiledRecord = compiled_of(record)?; - let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; - let mut tags: Vec = Vec::new(); - if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? - && (value.is_instance_of::() - || value.is_instance_of::()) - { - for tag in value.try_iter()? { - tags.push(tag?.str()?.extract()?); - } - } - let folder: String = record.host.model_file.get().folder(py)?; - Ok(ResourceRow { - key: (MODEL_RESOURCE.to_owned(), model_name(record)), - deps: compiled.deps.clone(), - tags, - folder: Some(folder), - }) - }) -} - pub(crate) fn model_name(record: &ModelInputRecord) -> String { record.name.clone() } - -/// The `CompiledRelationLocation` of a destination. -pub(crate) fn relation_location( - classes: &CompiledViewClasses<'_>, - destination: &DestinationFact, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); - fields.set_item(DATABASE_KEY, &destination.database)?; - fields.set_item(SCHEMA_KEY, &destination.schema)?; - fields.set_item("name", &destination.name)?; - fields.set_item("qualified_name", &destination.qualified_name)?; - fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; - fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; - Ok(classes.relation_location.call((), Some(&fields))?.unbind()) -} - -pub(crate) fn object_key( - classes: &CompiledViewClasses<'_>, - kind: &str, - name: &str, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); - fields.set_item("resource_type", kind)?; - fields.set_item("name", name)?; - Ok(classes.object_key.call((), Some(&fields))?.unbind()) -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs index 5124a36da9..9524176ce3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs @@ -1,15 +1,10 @@ //! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. -use crate::compiled_views::models::LineageFactRow; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::PyResult; -use crate::compiled_views::main::compiled_models::native_model::native_model; -use crate::compiled_views::types::DeclaredRow; -use crate::compiled_views::types::LineageInput; use sqlbuild_project::types::InferredRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -const TRANSFORM_VALUES: [&str; 6] = [ +pub(crate) const TRANSFORM_VALUES: [&str; 6] = [ "direct", "cast", "expression", @@ -17,288 +12,15 @@ const TRANSFORM_VALUES: [&str; 6] = [ "star", "constant", ]; -const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; +pub(crate) const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; -/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. -pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.inferred_rows(model.py()); - } - let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; - if rows.is_none() { - return Ok(None); - } - let mut inferred: Vec = Vec::new(); - for row in rows.try_iter()? { - let (name, column_type, nullability): (String, Option, Bound<'_, PyAny>) = - row?.extract()?; - inferred.push((name, column_type, nullability.str()?.extract()?)); - } - Ok(Some(inferred)) -} - -/// The names of the inferred output columns, `None` when the shape is unknown. -pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { - Ok(inferred_rows(model)?.map(column_names)) -} - -fn column_names(rows: Vec) -> Vec { +pub(crate) fn column_names(rows: Vec) -> Vec { rows.into_iter().map(|(name, _, _)| name).collect() } -/// Each declared column of the attached schema as `(name, type)`, `None` without a schema. -pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.schema_column_rows(model.py()); - } - native_field(model, "schema_column_rows")?.extract() -} - -/// Whether the attached schema declares dynamic column families. -pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.dynamic_columns_declared(model.py()); - } - native_field(model, "has_dynamic_columns")?.is_truthy() -} - -/// `(has star, star resolved)` of the model's analysed query. -pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { - if let Some(view) = native_model(model)? { - return Ok(( - view.fast_lineage_has_star(model.py())?, - view.fast_lineage_star_resolved(model.py())?, - )); - } - Ok(( - native_field(model, "fast_lineage_has_star")?.is_truthy()?, - native_field(model, "fast_lineage_star_resolved")?.is_truthy()?, - )) -} - -/// Output columns whose types semantic recovery forgot, in any order. -pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.unchecked_columns().to_vec()); - } - native_field(model, "unchecked_output_columns")? - .try_iter()? - .map(|column| column?.extract()) - .collect() -} - -/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. -pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); - } - let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; - if !columns.is_truthy()? { - return Ok(Vec::new()); - } - let mut lineage: Vec = Vec::new(); - for output in columns.try_iter()? { - let output: Bound<'_, PyAny> = output?; - let mut upstream: Vec<(String, String)> = Vec::new(); - for source in native_field(&output, "upstream_columns")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - upstream.push(( - native_field(&source, "resource_name")?.extract()?, - native_field(&source, "column_name")?.extract()?, - )); - } - lineage.push((native_field(&output, "output_column")?.extract()?, upstream)); - } - Ok(lineage) -} - -/// Whether the native session validated the model's bindings. -pub(crate) fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.binding_validated(model.py()); - } - native_field(model, "binding_validated")?.is_truthy() -} - -/// Whether the model's dynamic pivot proof proved its output columns. -pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.output_proven(model.py()); - } - let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; - Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) -} - -/// Each lineage output with its transform kind and confidence values, `None` without lineage. -pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - let Some(rows) = view.lineage_rows(model.py())? else { - return Ok(None); - }; - let mut facts: Vec = Vec::with_capacity(rows.len()); - for row in rows { - facts.push(LineageFactRow { - output_column: row.output_column, - transform: coded(&TRANSFORM_VALUES, row.transform_code)?, - confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, - sources: row.sources, - }); - } - return Ok(Some(facts)); - } - let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; - if columns.is_none() { - return Ok(None); - } - let mut facts: Vec = Vec::new(); - for output in columns.try_iter()? { - let output: Bound<'_, PyAny> = output?; - let mut sources: Vec<(String, String, String)> = Vec::new(); - for source in native_field(&output, "upstream_columns")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - sources.push(( - native_field(&source, "resource_type")?.extract()?, - native_field(&source, "resource_name")?.extract()?, - native_field(&source, "column_name")?.extract()?, - )); - } - facts.push(LineageFactRow { - output_column: native_field(&output, "output_column")?.extract()?, - transform: native_field(&output, "transform_kind")?.str()?.extract()?, - confidence: native_field(&output, "confidence")?.str()?.extract()?, - sources, - }); - } - Ok(Some(facts)) -} - -fn coded(values: &[&str], code: u8) -> PyResult { +pub(crate) fn coded(values: &[&str], code: u8) -> PyResult { values .get(usize::from(code)) .map(|value| (*value).to_owned()) .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("tuple index out of range")) } - -/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. -pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.proven_inputs(model.py()); - } - let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; - if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { - return Ok(None); - } - native_field(&proof, "input_relations")?.extract().map(Some) -} - -/// The model's name. -pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.name(model.py()); - } - native_field(model, "name")?.extract() -} - -/// The model's compiled query SQL. -pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.query_sql(model.py())?.into_any()); - } - native_field(model, "query_sql") -} - -/// The model's `CompiledObjectKey`. -pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.key(model.py())?.into_bound(model.py())); - } - native_field(model, "key") -} - -/// The model's destination `CompiledRelationLocation`. -pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.destination(model.py())?.into_bound(model.py())); - } - native_field(model, "destination") -} - -/// The model's name as a Python string. -pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - let name: String = view.name(model.py())?; - return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); - } - native_field(model, "name") -} - -/// The model's attached `SchemaModelEntry`, or `None`. -pub(crate) fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.schema_entry(py)?.into_bound(py)); - } - native_field(model, "schema_entry") -} - -/// The model's `CompileModelConfig`. -pub(crate) fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.config(py)?.into_bound(py)); - } - native_field(model, "config") -} - -/// The model's `CompileSqlReference` tuple. -pub(crate) fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.references(py)?.into_bound(py).into_any()); - } - native_field(model, "references") -} - -/// The model file's relative path. -pub(crate) fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.relative_path(py)?.into_bound(py)); - } - native_field(model, "relative_path") -} - -/// The model's `InferredColumn` tuple, or `None`. -pub(crate) fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.inferred_columns(py)?.into_bound(py)); - } - native_field(model, "inferred_columns") -} - -/// `(resource type, name)` of each dependency. -pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return view.dep_rows(model.py()); - } - native_field(model, "dep_rows")?.extract() -} - -/// The model's dependency `CompiledObjectKey` tuple. -pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.deps(py)?.into_bound(py).into_any()); - } - native_field(model, "deps") -} - -/// The model's enum columns by name. -pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.enum_columns(py)?.into_bound(py).into_any()); - } - native_field(model, "enum_columns") -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs index 885491e727..b12a75d8cd 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs @@ -1,37 +1,28 @@ //! Declared unique keys of project relations, derived natively from their key facts. +use crate::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; +use crate::compiled_views::main::compiled_models::relation_keys::relation_keys; use crate::compiled_views::models::RelationKeyRow; use crate::compiled_views::types::RelationKeys; -use std::collections::BTreeMap; -use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; -use sqlbuild_discovery::project_validation::models::CsvSettings; -use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; -use crate::compiled_project::main::python_column_facts::python_column_facts; -use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; -use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeModelInput; use crate::resource_views::main::source_views::read_source_view::read_source_view; use crate::resource_views::models::SourceKeyFacts; use crate::resource_views::types::SourceViews; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; const MATERIALIZED_CONFIG: &str = "materialized"; const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; const CURSOR_CONFIG: &str = "cursor"; const UNIQUE_KEY_CONFIG: &str = "unique_key"; -const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; -const RELATION_KIND_SEED: &str = "seed"; -const RELATION_KIND_SOURCE: &str = "source"; +pub(crate) const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; +pub(crate) const RELATION_KIND_SEED: &str = "seed"; +pub(crate) const RELATION_KIND_SOURCE: &str = "source"; /// Native declared columns and model audits used to derive relation keys. type DeclaredKeyFacts<'a> = (&'a [SchemaColumnFact], Vec<(String, Option)>); @@ -70,25 +61,6 @@ pub(crate) fn relation_key_row( }) } -/// Each audit spec's `(definition name, column argument when it is text)`. -pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { - let mut rows: Vec<(String, Option)> = Vec::new(); - for audit in audits.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let argument: Option> = native_field(&audit, "arguments")? - .downcast::()? - .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; - rows.push(( - native_field(&audit, "definition_name")?.extract()?, - argument - .filter(pyo3::types::PyAnyMethods::is_instance_of::) - .map(|value| value.extract()) - .transpose()?, - )); - } - Ok(rows) -} - /// A source's key row from its native view's declared facts, or `None` for any other object. pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option> { read_source_view(view, |source, py| { @@ -117,14 +89,6 @@ pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option(source: &Bound<'py, PyAny>) -> PyResult> { - if read_source_view(source, |_, _| Ok(())).is_some() { - return Ok(source.clone()); - } - native_field(source, "source_entry") -} - /// `str(values.get(key, ""))`. fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { match values.get_item(key)? { @@ -155,91 +119,17 @@ pub(crate) fn unique_key(values: &Bound<'_, PyDict>) -> PyResult, - entry: &Bound<'_, PyAny>, -) -> PyResult { - if entry.is_none() { - return relation_key_row(kind, name, values, None); - } - let columns: Vec = - python_column_facts(&declared_specs(entry, "columns")?, None)?; - let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; - relation_key_row( - kind, - name, - values, - Some((&columns, audit_key_rows(&audits)?)), - ) -} - /// `getattr(entry, name, None) or ()`. -fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { +pub(crate) fn declared_specs<'py>( + entry: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { match entry.getattr_opt(name)? { Some(specs) if specs.is_truthy()? => Ok(specs), _ => Ok(PyTuple::empty(entry.py()).into_any()), } } -/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. -pub(crate) fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { - let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); - for seed in seeds { - let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; - candidates.push(( - native_field(&entry, "name")?.extract()?, - native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, - seed_csv_settings(&entry)?, - )); - } - let py: Python<'_> = match seeds.first() { - Some(seed) => seed.py(), - None => return Ok(Vec::new()), - }; - Ok(py.detach(|| { - candidates - .into_iter() - .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) - .map(|(name, _, _)| name) - .collect() - })) -} - -/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. -pub(crate) fn relation_keys<'py>( - py: Python<'py>, - mut rows: Vec, - (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - one_row_seeds: &[String], -) -> PyResult { - for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { - for entry in entries.try_iter()? { - let entry: Bound<'py, PyAny> = entry?; - if let Some(row) = source_view_key_row(&entry) { - rows.push(row?); - continue; - } - let name: String = native_field(&entry, "name")?.extract()?; - rows.push(entry_relation_key_row( - kind, - name, - &PyDict::new(py), - &entry, - )?); - } - } - let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); - for relation in &mut relations { - relation.at_most_one_row = - relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); - } - let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); - Ok(keys.into_iter().collect()) -} - /// `keys` as a `{identity: ((column, ...), ...)}` dict. fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult> { let found: Bound<'py, PyDict> = PyDict::new(py); @@ -256,7 +146,7 @@ fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult( +pub(crate) fn declared_relation_keys<'py>( py: Python<'py>, model_rows: Vec, seeds: &Bound<'py, PyAny>, @@ -265,43 +155,15 @@ fn declared_relation_keys<'py>( keys_dict(py, relation_keys(py, model_rows, (seeds, sources), &[])?) } -/// Declared keys of a compiled project's models, seeds and sources. -pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = project.py(); - let mut rows: Vec = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - rows.push(model_key_row(&model?)?); - } - let compiled_seeds: Vec> = native_field(project, "seeds")? - .try_iter()? - .collect::>()?; - let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; - let mut seeds: Vec> = Vec::new(); - for seed in &compiled_seeds { - seeds.push(native_field(seed, "schema_entry")?); - } - let mut sources: Vec> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { - sources.push(source_key_input(&source?)?); - } - relation_keys( - py, - rows, - ( - PyTuple::new(py, seeds)?.as_any(), - PyTuple::new(py, sources)?.as_any(), - ), - &one_row_seeds, - ) -} - /// Declared keys of a compiled project's models, seeds and sources. #[pyfunction] -fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn compiled_relation_keys<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { keys_dict(project.py(), project_relation_keys(project)?) } -fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { +pub(crate) fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { let RelationKeyRow { kind, name, @@ -333,22 +195,3 @@ fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { at_most_one_row: false, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; - module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) -} - -/// The declared key facts of a model input or compiled model. -pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = model.py(); - if let Ok(input) = model.downcast::() { - return input.get().read_with_owner(py, |owner, record| { - model_relation_key_row(owner, record, py) - }); - } - if let Ok(compiled) = model.downcast::() { - return compiled.get().relation_key_row(py); - } - native_field(model, "relation_key_row")?.extract() -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs index 70eb8f8fd3..0046c3ab7b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs @@ -1,21 +1,15 @@ //! A model's compile input, read from the native compiled project that owns it. -use crate::compiled_views::models::AttachedSchema; +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; use crate::compiled_views::models::InputViewClasses; -use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::types::ModelInputRecord; use sqlbuild_project::models::ExpansionRecord; use std::sync::Arc; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; -use crate::compiled_project::main::python_column_facts::python_column_facts; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::_helpers::compiled_models::relation_keys::relation_key_row; use crate::compiled_views::models::RelationKeyRow; @@ -23,7 +17,6 @@ use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::records::projected_text::projected_text; use sqlbuild_python_front::discovery::models::NativeModelFile; -use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; use sqlbuild_python_front::render::main::calls::span_tuple::span_tuple; use sqlbuild_python_front::render::types::SpanRow; @@ -51,14 +44,6 @@ pub(crate) fn read_input<'py, T>( read(owner.input(index)?, py) } -/// The classes the owner's views project attributes with. -pub(crate) fn view_classes<'py>( - owner: &Py, - py: Python<'py>, -) -> PyResult> { - owner.bind(py).borrow().input_classes(py)?.extract() -} - /// The compiled query as the Python string every reader shares; the authored query's when equal. pub(crate) fn query_sql_text<'py>( py: Python<'py>, @@ -116,35 +101,6 @@ pub(crate) fn references( ) } -/// `CompileSqlReference` records of `rows`, built once into `cache`. -pub(crate) fn reference_tuple( - classes: &InputViewClasses<'_>, - rows: &[ReferenceRow], - cache: &PyOnceLock>, -) -> PyResult> { - let py: Python<'_> = classes.sql_reference.py(); - cache - .get_or_try_init(py, || { - let mut references: Vec> = Vec::with_capacity(rows.len()); - for ReferenceRow { - kind, - name, - package, - call_argument_count: count, - } in rows - { - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; - fields.set_item("ref_name", name)?; - fields.set_item("ref_package", package)?; - fields.set_item("call_argument_count", count)?; - references.push(classes.sql_reference.call((), Some(&fields))?); - } - Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) - }) - .map(|references| references.clone_ref(py)) -} - /// The model's declared key facts: its config and attached schema columns and audits. pub(crate) fn model_relation_key_row( owner: &NativeCompiledProject, @@ -224,106 +180,8 @@ pub(crate) fn expansion_object( Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } -/// The attached schema of `model_input`; a native input answers without building its entry. -pub(crate) fn attached_schema<'py>( - model_input: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native.get().read(py, |record, py| { - let Some(schema) = &record.schema else { - return Ok(None); - }; - Ok(Some(AttachedSchema { - name: record.name.clone(), - dynamic_columns: schema.host.dynamic_columns.read(py)?, - type_enforcement: schema.type_enforcement, - })) - }); - } - let entry: Bound<'py, PyAny> = native_field(model_input, "schema_entry")?; - if entry.is_none() { - return Ok(None); - } - Ok(Some(AttachedSchema { - name: native_field(&entry, "name")?.extract()?, - dynamic_columns: family_specs(&native_field(&entry, "dynamic_columns")?)?, - type_enforcement: native_field(&entry, "type_enforcement")?.is_truthy()?, - })) -} - -/// The declared column facts of `model_input`'s attached schema, empty without one. -pub(crate) fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native.get().read_with_owner(py, |owner, record| { - Ok(owner - .input_schema_fact(record) - .map(|fact| fact.columns.clone()) - .unwrap_or_default()) - }); - } - let entry: Bound<'_, PyAny> = native_field(model_input, "schema_entry")?; - if entry.is_none() { - return Ok(Vec::new()); - } - python_column_facts(&native_field(&entry, "columns")?, None) -} - -/// The name of `model_input`, its file stem; a native input answers from its record. -pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { - if let Ok(native) = model_input.cast::() { - return native - .get() - .read(model_input.py(), |record, _| Ok(record.name.clone())); - } - native_field(model_input, "model_file")? - .getattr("file_path")? - .getattr("stem")? - .extract() -} - -/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. -pub(crate) fn input_sql_macro_checks( - model_input: &Bound<'_, PyAny>, -) -> PyResult> { - if let Ok(native) = model_input.cast::() { - return native.get().read(model_input.py(), |record, _| { - Ok(record.sql_macro_checks.clone()) - }); - } - Ok(Vec::new()) -} - -/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -pub(crate) fn input_analysis_sql( - model_input: &Bound<'_, PyAny>, - dialect: &str, -) -> PyResult { - if let Ok(native) = model_input.cast::() { - return native.get().read(model_input.py(), |record, _| { - Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) - }); - } - native_field(model_input, "query_sql")?.extract() -} - -/// Each reference of `model_input` as `(kind, name, package, call argument count)`. -pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native - .get() - .read(py, |record, _| Ok(record.references.clone())); - } - native_field(model_input, "references")? - .try_iter()? - .map(|reference| reference_row_of(&reference?)) - .collect() -} - /// One `CompileSqlReference` record as its row. -fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { Ok(ReferenceRow { kind: native_field(reference, "ref_kind")?.str()?.extract()?, name: native_field(reference, "ref_name")?.extract()?, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs index 769e1eebee..51ed97700f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs @@ -7,14 +7,10 @@ use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; -use crate::resource_views::models::NativeAuditInput; -use crate::resource_views::models::NativeCompiledAudit; -use crate::resource_views::models::NativeCompiledFunction; -use crate::resource_views::models::NativeCompiledSeed; -use crate::resource_views::models::NativeCompiledSource; -use crate::resource_views::models::NativeFunctionInput; -use crate::resource_views::models::NativeSeedInput; -use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::models::{NativeAuditInput, NativeCompiledAudit}; +use crate::resource_views::models::{NativeCompiledFunction, NativeFunctionInput}; +use crate::resource_views::models::{NativeCompiledSeed, NativeSeedInput}; +use crate::resource_views::models::{NativeCompiledSource, NativeSourceInput}; pub(crate) fn read_native_audit_input<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs index e59b937909..269a8bc275 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs @@ -4,8 +4,7 @@ use std::fmt::{Debug, Formatter, Result as FmtResult}; use crate::compiled_views::models::AnalysisProjections; use crate::compiled_views::models::CompiledProjections; -use crate::compiled_views::models::ModelInputHandles; -use crate::compiled_views::models::SchemaHandles; +use crate::compiled_views::models::{ModelInputHandles, SchemaHandles}; use crate::resource_views::models::AuditProjections; use crate::resource_views::models::FunctionProjections; use crate::resource_views::models::SeedHandles; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs index 4d92f81fe6..d9f4098d68 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs @@ -1,93 +1,25 @@ //! A compiled project's resource collections, read from its native view. -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use crate::compiled_views::models::NativeProjectView; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -const MODELS_FIELD: &str = "models"; -const SOURCES_FIELD: &str = "sources"; -const SEEDS_FIELD: &str = "seeds"; -const FUNCTIONS_FIELD: &str = "functions"; -const AUDITS_FIELD: &str = "audits"; +pub(crate) const MODELS_FIELD: &str = "models"; +pub(crate) const SOURCES_FIELD: &str = "sources"; +pub(crate) const SEEDS_FIELD: &str = "seeds"; +pub(crate) const FUNCTIONS_FIELD: &str = "functions"; +pub(crate) const AUDITS_FIELD: &str = "audits"; /// The native view behind a project object; any other object is a Python-built record, counted. #[inline] -fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn native_project<'a>( + project: &'a Bound<'_, PyAny>, +) -> PyResult> { if let Ok(view) = project.cast::() { return Ok(Some(view.get())); } count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?; Ok(None) } - -/// The project's compiled models, in compile order. -pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.models(py).into_bound(py).into_any()); - } - native_field(project, MODELS_FIELD) -} - -/// The project's compiled sources. -pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.sources(py).into_bound(py)); - } - native_field(project, SOURCES_FIELD) -} - -/// The project's compiled functions. -pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.functions(py).into_bound(py)); - } - native_field(project, FUNCTIONS_FIELD) -} - -/// The project's compiled seeds. -pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.seeds(py).into_bound(py)); - } - native_field(project, SEEDS_FIELD) -} - -/// The project's compiled audits. -pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.audits(py).into_bound(py)); - } - native_field(project, AUDITS_FIELD) -} - -/// The project's resource collection named `field`; any other field is read by attribute. -pub(crate) fn collection<'py>( - project: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - match field { - MODELS_FIELD => models(project), - SOURCES_FIELD => sources(project), - SEEDS_FIELD => seeds(project), - FUNCTIONS_FIELD => functions(project), - AUDITS_FIELD => audits(project), - _ => native_field(project, field), - } -} - -/// The project's compiled SQL tests. -pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.sql_tests(py).into_bound(py)); - } - native_field(project, "sql_tests") -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs index 2481309065..ff33af9d6f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs @@ -1,51 +1,35 @@ //! Compiled records and views: projecting native views as records, and recording later stages. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple, PyType, -}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use crate::compiled_views::main::project::declaration_rows::declaration_rows; +use crate::compiled_views::main::project::plain_typed_value::plain_typed_value; +use crate::compiled_views::main::project::record_model_analyses::record_model_analyses; +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple, PyType}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use crate::compiled_project::main::entry_schema_fact::entry_schema_fact; use crate::compiled_project::main::model_facts::model_facts; -use crate::compiled_project::models::AnalysisPy; use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_project::models::ProofPy; -use crate::compiled_views::models::NativeCompiledModel; -use crate::compiled_views::models::NativeInputsView; -use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::record_spec::record_spec; use sqlbuild_python_base::records::types::RecordId; -const DATACLASSES_MODULE: &str = "dataclasses"; -const DECIMAL_KIND: &str = "decimal"; -const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; -const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; +pub(crate) const DATACLASSES_MODULE: &str = "dataclasses"; +pub(crate) const DECIMAL_KIND: &str = "decimal"; +pub(crate) const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +pub(crate) const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; -/// `dataclasses.replace(value, **changes)`. -pub(crate) fn replace<'py>( +/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. +pub(crate) fn record<'py>( value: &Bound<'py, PyAny>, - changes: &[(&str, Bound<'py, PyAny>)], + class: RecordId, ) -> PyResult> { - let py: Python<'py> = value.py(); - let keywords: Bound<'py, PyDict> = PyDict::new(py); - for (name, change) in changes { - keywords.set_item(*name, change)?; - } - imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) -} - -/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. -fn record<'py>(value: &Bound<'py, PyAny>, class: RecordId) -> PyResult> { let py: Python<'py> = value.py(); let class: Bound<'py, PyAny> = record_class(py, class)?; if value.is_instance(&class)? { @@ -58,40 +42,6 @@ fn record<'py>(value: &Bound<'py, PyAny>, class: RecordId) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { - record(model, RecordId::CompiledModel) -} - -/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - record(audit, RecordId::CompiledAudit) -} - -/// The function as a `CompiledFunction`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_function_record<'py>( - function: &Bound<'py, PyAny>, -) -> PyResult> { - record(function, RecordId::CompiledFunction) -} - -/// The source as a `CompiledSource`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_source_record<'py>( - source: &Bound<'py, PyAny>, -) -> PyResult> { - record(source, RecordId::CompiledSource) -} - -/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - record(seed, RecordId::CompiledSeed) -} - /// Each item of `fields[name]` as a `class` record. fn records_of<'py>(fields: &Bound<'py, PyDict>, (name, class): (&str, RecordId)) -> PyResult<()> { let py: Python<'py> = fields.py(); @@ -133,91 +83,8 @@ pub(crate) fn compiled_project_record<'py>( new_record(py, RecordId::CompiledProject, (), Some(&fields)) } -/// The model with these final binding diagnostics. -#[pyfunction] -#[pyo3(signature = (*, model, diagnostics))] -pub(crate) fn with_binding_diagnostics<'py>( - model: &Bound<'py, PyAny>, - diagnostics: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - if model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { - return replace(model, &[("binding_diagnostics", diagnostics.clone())]); - } - model.call_method1("with_binding_diagnostics", (diagnostics,)) -} - -/// The model with its recovered bindings and its poisoned outputs' types forgotten. -#[pyfunction] -#[pyo3(signature = (*, model, diagnostics, poisoned))] -pub(crate) fn with_type_recovery<'py>( - model: &Bound<'py, PyAny>, - diagnostics: &Bound<'py, PyAny>, - poisoned: &Bound<'py, PyFrozenSet>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let name: String = native_field(model, "name")?.extract()?; - let mut unchecked: Vec = Vec::new(); - for pair in poisoned.iter() { - let (model_name, column): (String, String) = pair.extract()?; - if model_name == name { - unchecked.push(column); - } - } - if !model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { - return model.call_method1("with_type_recovery", (diagnostics, unchecked)); - } - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; - let columns: Bound<'py, PyAny> = if inferred.is_none() { - inferred - } else { - let mut columns: Vec> = Vec::new(); - for column in inferred.try_iter()? { - let column: Bound<'py, PyAny> = column?; - let key = (name.as_str(), native_field(&column, "name")?); - columns.push(if poisoned.contains(key)? { - replace(&column, &[("type", py.None().into_bound(py))])? - } else { - column - }); - } - PyTuple::new(py, columns)?.into_any() - }; - replace( - model, - &[ - ("binding_diagnostics", diagnostics.clone()), - ("inferred_columns", columns), - ( - "unchecked_output_columns", - PyFrozenSet::new(py, &unchecked)?.into_any(), - ), - ], - ) -} - -/// The project with `changes`; a native view shares its owner's models. -#[pyfunction] -#[pyo3(signature = (*, project, **changes))] -pub(crate) fn project_with<'py>( - project: &Bound<'py, PyAny>, - changes: Option>, -) -> PyResult> { - with_changes(project, RecordId::CompiledProject, changes) -} - -/// The compile inputs with `changes`; a native view shares its owner's model inputs. -#[pyfunction] -#[pyo3(signature = (*, inputs, **changes))] -pub(crate) fn inputs_with<'py>( - inputs: &Bound<'py, PyAny>, - changes: Option>, -) -> PyResult> { - with_changes(inputs, RecordId::CompileProjectInputs, changes) -} - /// `replace(value, **changes)` for a record, else the native view's `with_changes(**changes)`. -fn with_changes<'py>( +pub(crate) fn with_changes<'py>( value: &Bound<'py, PyAny>, class: RecordId, changes: Option>, @@ -230,62 +97,8 @@ fn with_changes<'py>( value.call_method("with_changes", (), Some(&changes)) } -/// The native owner's inputs view when it holds every model input; else a record. -#[pyfunction] -#[pyo3(signature = (*, fields, native_project))] -pub(crate) fn compile_inputs_view<'py>( - fields: &Bound<'py, PyDict>, - native_project: Option>, -) -> PyResult> { - let py: Python<'py> = fields.py(); - if let Some(project) = &native_project { - let mut native: bool = true; - for model in fields.as_any().get_item("model_inputs")?.try_iter()? { - if !model?.is_instance_of::() { - native = false; - break; - } - } - if native { - let view: NativeInputsView = - NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; - return view.into_bound_py_any(py); - } - } - let keywords: Bound<'py, PyDict> = fields.copy()?; - keywords.set_item("native_project", native_project)?; - new_record(py, RecordId::CompileProjectInputs, (), Some(&keywords)) -} - -/// A typed SQL value as the plain value native consumers read. -#[pyfunction] -pub(crate) fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - let kind: String = native_field(value, "kind")?.getattr("value")?.extract()?; - let inner: Bound<'py, PyAny> = native_field(value, "value")?; - if kind == DECIMAL_KIND { - return Ok(inner.str()?.into_any()); - } - if SCALAR_KINDS.contains(&kind.as_str()) { - return Ok(inner); - } - if COLLECTION_KINDS.contains(&kind.as_str()) { - let items: Vec> = inner - .try_iter()? - .map(|item| plain_typed_value(&item?)) - .collect::>()?; - return Ok(PyList::new(py, items)?.into_any()); - } - let mapping: Bound<'py, PyDict> = PyDict::new(py); - for pair in inner.try_iter()? { - let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; - mapping.set_item(key, plain_typed_value(&item)?)?; - } - Ok(mapping.into_any()) -} - /// An enum declaration as the native project row. -fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = declaration.py(); let members: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = native_field(declaration, "members")? @@ -312,7 +125,7 @@ fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult(declaration: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = declaration.py(); let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; PyTuple::new( @@ -332,27 +145,8 @@ fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult( - enums: &Bound<'py, PyAny>, - constants: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { - let py: Python<'py> = enums.py(); - let enum_rows: Vec> = enums - .try_iter()? - .map(|declaration| enum_row(&declaration?)) - .collect::>()?; - let constant_rows: Vec> = constants - .try_iter()? - .map(|declaration| constant_row(&declaration?)) - .collect::>()?; - Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) -} - /// `(name, type)` of each dynamic family; `casefold` keys the family by its folded name. -fn family_types<'py>( +pub(crate) fn family_types<'py>( families: &Bound<'py, PyAny>, casefold: bool, ) -> PyResult)>> { @@ -371,73 +165,8 @@ fn family_types<'py>( .collect() } -/// Transfer each assembled model's typed analysis to its retained facts; return how many. -#[pyfunction] -#[pyo3(signature = (*, project, models))] -pub(crate) fn record_model_analyses<'py>( - project: &Bound<'py, NativeCompiledProject>, - models: &Bound<'py, PyAny>, -) -> PyResult { - let py: Python<'py> = project.py(); - let models: Vec> = models.try_iter()?.collect::>()?; - let views: Vec> = models - .iter() - .filter(|model| model.is_instance_of::()) - .map(|model| model.extract::>()) - .collect::>()?; - let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; - let nullable: Bound<'py, PyAny> = enum_member(py, RecordId::InferredNullability, "NULLABLE")?; - for model in &models { - if model.is_instance_of::() { - continue; - } - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_rows")?; - let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { - Vec::new() - } else { - inferred.extract()? - }; - let proof: Bound<'py, PyAny> = native_field(model, "dynamic_column_contract")?; - let proof = if proof.is_none() { - None - } else { - let families: Bound<'py, PyAny> = native_field(&proof, "families")?; - Some(ProofPy { - output_proven: native_field(&proof, "output_proven")?.extract()?, - bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, - families: family_types(&families, true)?, - failure_reason: native_field(&proof, "failure_reason")?.extract()?, - named_families: family_types(&families, false)?, - }) - }; - let analysis = AnalysisPy { - inferred: !inferred.is_none(), - inferred_columns: rows - .iter() - .map(|(name, column_type, _)| (name.clone(), column_type.clone())) - .collect(), - nullable_columns: rows - .iter() - .map(|(_, _, nullability)| nullability.eq(&nullable)) - .collect::>()?, - fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, - unchecked: native_field(model, "unchecked_output_columns")? - .try_iter()? - .map(|column| column?.extract()) - .collect::>()?, - }; - let name: String = native_field(model, "name")?.extract()?; - transferred += usize::from( - project - .borrow_mut() - .record_model_analysis(&name, analysis, proof), - ); - } - Ok(transferred) -} - /// A native project for a `CompiledProject` built without the compile-input stage. -fn hand_built_project<'py>( +pub(crate) fn hand_built_project<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); @@ -457,21 +186,6 @@ fn hand_built_project<'py>( Ok(native) } -/// The compile's retained project, or one recorded from a hand-built project. -#[pyfunction] -pub(crate) fn compiled_project_facts<'py>( - project: &Bound<'py, PyAny>, -) -> PyResult> { - if let Ok(view) = project.downcast::() { - return Ok(view.get().owner().bind(project.py()).clone()); - } - let retained: Bound<'py, PyAny> = native_field(project, "native_project")?; - if !retained.is_none() { - return Ok(retained.downcast_into()?); - } - hand_built_project(project) -} - fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); let relative_path: Bound<'py, PyAny> = native_field(model, "relative_path")?; @@ -535,33 +249,3 @@ fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - for function in [ - wrap_pyfunction!(compiled_model_record, module)?, - wrap_pyfunction!(compiled_audit_record, module)?, - wrap_pyfunction!(compiled_function_record, module)?, - wrap_pyfunction!(compiled_source_record, module)?, - wrap_pyfunction!(compiled_seed_record, module)?, - wrap_pyfunction!(compiled_project_record, module)?, - wrap_pyfunction!(with_binding_diagnostics, module)?, - wrap_pyfunction!(with_type_recovery, module)?, - wrap_pyfunction!(project_with, module)?, - wrap_pyfunction!(inputs_with, module)?, - wrap_pyfunction!(compile_inputs_view, module)?, - wrap_pyfunction!(plain_typed_value, module)?, - wrap_pyfunction!(declaration_rows, module)?, - wrap_pyfunction!(record_model_analyses, module)?, - wrap_pyfunction!(compiled_project_facts, module)?, - ] { - module.add_function(function)?; - } - Ok(()) -} - -/// `declaration_rows` as a Python function. -pub(crate) fn declaration_rows_function( - py: pyo3::Python<'_>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(declaration_rows, py) -} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs index 3a8474c4a1..944f49032e 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs @@ -1,12 +1,25 @@ //! Each audit spec's `(definition name, column argument when it is text)`. -use crate::compiled_views::_helpers::compiled_models::relation_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::compiled_models::relation_keys::UNIQUE_AUDIT_COLUMN_ARGUMENT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each audit spec's `(definition name, column argument when it is text)`. -#[inline] pub fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { - relation_keys::audit_key_rows(audits) + let mut rows: Vec<(String, Option)> = Vec::new(); + for audit in audits.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let argument: Option> = native_field(&audit, "arguments")? + .downcast::()? + .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; + rows.push(( + native_field(&audit, "definition_name")?.extract()?, + argument + .filter(pyo3::types::PyAnyMethods::is_instance_of::) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs index 52eefbb1cb..312064be6d 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs @@ -1,12 +1,13 @@ //! Whether the native session validated the model's bindings. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Whether the native session validated the model's bindings. -#[inline] pub fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult { - model_reads::binding_validated(model) + if let Some(view) = native_model(model)? { + return view.binding_validated(model.py()); + } + native_field(model, "binding_validated")?.is_truthy() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs index d1e11779df..784eea98b3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs @@ -1,12 +1,14 @@ //! The model's `CompileModelConfig`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's `CompileModelConfig`. -#[inline] pub fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::config(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.config(py)?.into_bound(py)); + } + native_field(model, "config") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs index 9031c277f2..19274ea22b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs @@ -1,13 +1,14 @@ //! Each declared column of the attached schema as `(name, type)`, `None` without a schema. -use crate::compiled_views::_helpers::compiled_models::model_reads; +use crate::compiled_views::main::compiled_models::native_model::native_model; use crate::compiled_views::types::DeclaredRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each declared column of the attached schema as `(name, type)`, `None` without a schema. -#[inline] pub fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - model_reads::declared_rows(model) + if let Some(view) = native_model(model)? { + return view.schema_column_rows(model.py()); + } + native_field(model, "schema_column_rows")?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs index 8251d79449..b66d1e8d9a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs @@ -1,17 +1,31 @@ //! The destination with adapter defaults filled in; `render` renders its qualified name. -use crate::compiled_views::_helpers::compiled_models::compiled_model_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DATABASE_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SCHEMA_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_project::models::DestinationFact; /// The destination with adapter defaults filled in; `render` renders its qualified name. -#[inline] pub fn defaulted_destination( current: &DestinationFact, defaults: (Option, Option), render: &Bound<'_, PyAny>, ) -> PyResult { - compiled_model_view::defaulted_destination(current, defaults, render) + let (default_schema, default_database) = defaults; + let schema: Option = current.schema.clone().or(default_schema); + let database: Option = current.database.clone().or(default_database); + let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); + keywords.set_item(DATABASE_KEY, &database)?; + keywords.set_item(SCHEMA_KEY, &schema)?; + keywords.set_item("name", ¤t.name)?; + let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; + Ok(DestinationFact { + database, + schema, + name: current.name.clone(), + qualified_name, + logical_schema: current.logical_schema.clone(), + logical_database: current.logical_database.clone(), + }) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs index 07c12f13f0..9347d09cf7 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs @@ -1,12 +1,13 @@ //! `(resource type, name)` of each dependency. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `(resource type, name)` of each dependency. -#[inline] pub fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { - model_reads::dep_rows(model) + if let Some(view) = native_model(model)? { + return view.dep_rows(model.py()); + } + native_field(model, "dep_rows")?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs index e343eecfd3..f5262b70a1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs @@ -1,12 +1,14 @@ //! The model's dependency `CompiledObjectKey` tuple. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's dependency `CompiledObjectKey` tuple. -#[inline] pub fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::deps(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.deps(py)?.into_bound(py).into_any()); + } + native_field(model, "deps") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs index 61f210511a..e0813b1511 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs @@ -1,12 +1,13 @@ //! The model's destination `CompiledRelationLocation`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's destination `CompiledRelationLocation`. -#[inline] pub fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::destination(model) + if let Some(view) = native_model(model)? { + return Ok(view.destination(model.py())?.into_bound(model.py())); + } + native_field(model, "destination") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs index 4949ac2ff5..71b6881793 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs @@ -1,12 +1,13 @@ //! Whether the attached schema declares dynamic column families. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Whether the attached schema declares dynamic column families. -#[inline] pub fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { - model_reads::dynamic_columns_declared(model) + if let Some(view) = native_model(model)? { + return view.dynamic_columns_declared(model.py()); + } + native_field(model, "has_dynamic_columns")?.is_truthy() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs index a0696b255b..6508f40189 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs @@ -1,19 +1,31 @@ //! The key row of a seed, source or hand-built model from its config values and schema entry. -use crate::compiled_views::_helpers::compiled_models::relation_keys; +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_specs; +use crate::compiled_views::_helpers::compiled_models::relation_keys::relation_key_row; +use crate::compiled_views::main::compiled_models::audit_key_rows::audit_key_rows; use crate::compiled_views::models::RelationKeyRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; /// The key row of a seed, source or hand-built model from its config values and schema entry. -#[inline] pub fn entry_relation_key_row( kind: &str, name: String, values: &Bound<'_, PyDict>, entry: &Bound<'_, PyAny>, ) -> PyResult { - relation_keys::entry_relation_key_row(kind, name, values, entry) + if entry.is_none() { + return relation_key_row(kind, name, values, None); + } + let columns: Vec = + python_column_facts(&declared_specs(entry, "columns")?, None)?; + let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; + relation_key_row( + kind, + name, + values, + Some((&columns, audit_key_rows(&audits)?)), + ) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs index 525272a9ef..9d537e3e7c 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs @@ -1,12 +1,14 @@ //! The model's enum columns by name. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's enum columns by name. -#[inline] pub fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::enum_columns(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.enum_columns(py)?.into_bound(py).into_any()); + } + native_field(model, "enum_columns") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs index 96eb742b03..43c632f13a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs @@ -1,12 +1,14 @@ //! The model's `InferredColumn` tuple, or `None`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's `InferredColumn` tuple, or `None`. -#[inline] pub fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::inferred_columns(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.inferred_columns(py)?.into_bound(py)); + } + native_field(model, "inferred_columns") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs index 3d989f68a0..72b1871db2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs @@ -1,12 +1,10 @@ //! The names of the inferred output columns, `None` when the shape is unknown. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::compiled_models::model_reads::column_names; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// The names of the inferred output columns, `None` when the shape is unknown. -#[inline] pub fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { - model_reads::inferred_names(model) + Ok(inferred_rows(model)?.map(column_names)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs index cca9756c1f..1269f953f2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs @@ -1,13 +1,24 @@ //! Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_project::types::InferredRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. -#[inline] pub fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - model_reads::inferred_rows(model) + if let Some(view) = native_model(model)? { + return view.inferred_rows(model.py()); + } + let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; + if rows.is_none() { + return Ok(None); + } + let mut inferred: Vec = Vec::new(); + for row in rows.try_iter()? { + let (name, column_type, nullability): (String, Option, Bound<'_, PyAny>) = + row?.extract()?; + inferred.push((name, column_type, nullability.str()?.extract()?)); + } + Ok(Some(inferred)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs index 63ed6d3f33..d9148405c9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs @@ -1,13 +1,52 @@ //! Each lineage output with its transform kind and confidence values, `None` without lineage. -use crate::compiled_views::_helpers::compiled_models::model_reads; +use crate::compiled_views::_helpers::compiled_models::model_reads::CONFIDENCE_VALUES; +use crate::compiled_views::_helpers::compiled_models::model_reads::TRANSFORM_VALUES; +use crate::compiled_views::_helpers::compiled_models::model_reads::coded; +use crate::compiled_views::main::compiled_models::native_model::native_model; use crate::compiled_views::models::LineageFactRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each lineage output with its transform kind and confidence values, `None` without lineage. -#[inline] pub fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { - model_reads::lineage_facts(model) + if let Some(view) = native_model(model)? { + let Some(rows) = view.lineage_rows(model.py())? else { + return Ok(None); + }; + let mut facts: Vec = Vec::with_capacity(rows.len()); + for row in rows { + facts.push(LineageFactRow { + output_column: row.output_column, + transform: coded(&TRANSFORM_VALUES, row.transform_code)?, + confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, + sources: row.sources, + }); + } + return Ok(Some(facts)); + } + let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; + if columns.is_none() { + return Ok(None); + } + let mut facts: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in native_field(&output, "upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + sources.push(( + native_field(&source, "resource_type")?.extract()?, + native_field(&source, "resource_name")?.extract()?, + native_field(&source, "column_name")?.extract()?, + )); + } + facts.push(LineageFactRow { + output_column: native_field(&output, "output_column")?.extract()?, + transform: native_field(&output, "transform_kind")?.str()?.extract()?, + confidence: native_field(&output, "confidence")?.str()?.extract()?, + sources, + }); + } + Ok(Some(facts)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs index 2d54843884..ea19e402b3 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs @@ -1,13 +1,31 @@ //! Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. -use crate::compiled_views::_helpers::compiled_models::model_reads; +use crate::compiled_views::main::compiled_models::native_model::native_model; use crate::compiled_views::types::LineageInput; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. -#[inline] pub fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult> { - model_reads::lineage_inputs(model) + if let Some(view) = native_model(model)? { + return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); + } + let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; + if !columns.is_truthy()? { + return Ok(Vec::new()); + } + let mut lineage: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for source in native_field(&output, "upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + upstream.push(( + native_field(&source, "resource_name")?.extract()?, + native_field(&source, "column_name")?.extract()?, + )); + } + lineage.push((native_field(&output, "output_column")?.extract()?, upstream)); + } + Ok(lineage) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs index a80f2cd744..56de1cd5c2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs @@ -1,13 +1,37 @@ //! The model's project graph row: its key, dependencies, tags and folder. -use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::MODEL_RESOURCE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::TAGS_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::VALUES_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_of; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_name; use crate::compiled_views::models::NativeCompiledModel; use crate::graph::models::ResourceRow; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's project graph row: its key, dependencies, tags and folder. -#[inline] pub fn model_graph_row(model: &NativeCompiledModel, py: Python<'_>) -> PyResult { - compiled_model_view::model_graph_row(model, py) + model.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; + let mut tags: Vec = Vec::new(); + if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? + && (value.is_instance_of::() + || value.is_instance_of::()) + { + for tag in value.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + } + let folder: String = record.host.model_file.get().folder(py)?; + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), + tags, + folder: Some(folder), + }) + }) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs index 346592f8b8..9cfd7328bd 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs @@ -1,12 +1,13 @@ //! The model's `CompiledObjectKey`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's `CompiledObjectKey`. -#[inline] pub fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::model_key(model) + if let Some(view) = native_model(model)? { + return Ok(view.key(model.py())?.into_bound(model.py())); + } + native_field(model, "key") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs index 85c69cbf6f..a523cb302e 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs @@ -1,13 +1,22 @@ //! The declared key facts of a model input or compiled model. -use crate::compiled_views::_helpers::compiled_models::relation_keys; +use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::RelationKeyRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The declared key facts of a model input or compiled model. -#[inline] pub fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { - relation_keys::model_key_row(model) + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }); + } + if let Ok(compiled) = model.downcast::() { + return compiled.get().relation_key_row(py); + } + native_field(model, "relation_key_row")?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs index db42985266..2519953905 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs @@ -1,12 +1,13 @@ //! The model's name. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's name. -#[inline] pub fn model_name(model: &Bound<'_, PyAny>) -> PyResult { - model_reads::model_name(model) + if let Some(view) = native_model(model)? { + return view.name(model.py()); + } + native_field(model, "name")?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs index ff4e7b3a68..e36c20de58 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs @@ -1,12 +1,14 @@ //! The model's name as a Python string. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's name as a Python string. -#[inline] pub fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::name_object(model) + if let Some(view) = native_model(model)? { + let name: String = view.name(model.py())?; + return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); + } + native_field(model, "name") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs index 2042400894..a3774bdfea 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs @@ -1,17 +1,17 @@ //! Object key. -use crate::compiled_views::_helpers::compiled_models::compiled_model_view; use crate::compiled_views::models::CompiledViewClasses; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; /// Object key. -#[inline] pub fn object_key( classes: &CompiledViewClasses<'_>, kind: &str, name: &str, ) -> PyResult> { - compiled_model_view::object_key(classes, kind, name) + let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); + fields.set_item("resource_type", kind)?; + fields.set_item("name", name)?; + Ok(classes.object_key.call((), Some(&fields))?.unbind()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs index db83a43784..1d22f5bab5 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs @@ -1,12 +1,32 @@ //! Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. -use crate::compiled_views::_helpers::compiled_models::relation_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use sqlbuild_discovery::project_validation::models::CsvSettings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; +use std::path::PathBuf; /// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. -#[inline] pub fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { - relation_keys::one_row_seed_names(seeds) + let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); + for seed in seeds { + let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; + candidates.push(( + native_field(&entry, "name")?.extract()?, + native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, + seed_csv_settings(&entry)?, + )); + } + let py: Python<'_> = match seeds.first() { + Some(seed) => seed.py(), + None => return Ok(Vec::new()), + }; + Ok(py.detach(|| { + candidates + .into_iter() + .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) + .map(|(name, _, _)| name) + .collect() + })) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs index ff0f9c9203..3fb73f9ca2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs @@ -1,12 +1,14 @@ //! Whether the model's dynamic pivot proof proved its output columns. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Whether the model's dynamic pivot proof proved its output columns. -#[inline] pub fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { - model_reads::output_proven(model) + if let Some(view) = native_model(model)? { + return view.output_proven(model.py()); + } + let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; + Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs index 5bcb0b43c5..6959869ae0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs @@ -1,13 +1,41 @@ //! Declared keys of a compiled project's models, seeds and sources. -use crate::compiled_views::_helpers::compiled_models::relation_keys; +use crate::compiled_views::main::compiled_models::model_key_row::model_key_row; +use crate::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; +use crate::compiled_views::main::compiled_models::relation_keys::relation_keys; +use crate::compiled_views::main::compiled_models::source_key_input::source_key_input; +use crate::compiled_views::models::RelationKeyRow; use crate::compiled_views::types::RelationKeys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Declared keys of a compiled project's models, seeds and sources. -#[inline] pub fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { - relation_keys::project_relation_keys(project) + let py: Python<'_> = project.py(); + let mut rows: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + rows.push(model_key_row(&model?)?); + } + let compiled_seeds: Vec> = native_field(project, "seeds")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; + let mut seeds: Vec> = Vec::new(); + for seed in &compiled_seeds { + seeds.push(native_field(seed, "schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in native_field(project, "sources")?.try_iter()? { + sources.push(source_key_input(&source?)?); + } + relation_keys( + py, + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), + &one_row_seeds, + ) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs index 60839a7a46..2fc54f0561 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs @@ -1,12 +1,17 @@ //! The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. -#[inline] pub fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { - model_reads::proven_inputs(model) + if let Some(view) = native_model(model)? { + return view.proven_inputs(model.py()); + } + let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; + if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { + return Ok(None); + } + native_field(&proof, "input_relations")?.extract().map(Some) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs index 451f3aac2e..e77775791a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs @@ -1,12 +1,13 @@ //! The model's compiled query SQL. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's compiled query SQL. -#[inline] pub fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::query_sql(model) + if let Some(view) = native_model(model)? { + return Ok(view.query_sql(model.py())?.into_any()); + } + native_field(model, "query_sql") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs index d8963ab406..746c48cac4 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs @@ -1,12 +1,14 @@ //! The model's `CompileSqlReference` tuple. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's `CompileSqlReference` tuple. -#[inline] pub fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::references(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.references(py)?.into_bound(py).into_any()); + } + native_field(model, "references") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs index a62f815534..095190dea8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs @@ -1,12 +1,12 @@ //! Register the relation keys functions and classes. -use crate::compiled_views::_helpers::compiled_models::relation_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::compiled_models::relation_keys::compiled_relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_relation_keys; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the relation keys functions and classes. -#[inline] pub fn register_relation_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { - relation_keys::register(module) + module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs index f49e35676b..0ef40b6b31 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs @@ -1,20 +1,47 @@ //! Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. -use crate::compiled_views::_helpers::compiled_models::relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys::RELATION_KIND_SEED; +use crate::compiled_views::_helpers::compiled_models::relation_keys::RELATION_KIND_SOURCE; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_relation; +use crate::compiled_views::_helpers::compiled_models::relation_keys::source_view_key_row; +use crate::compiled_views::main::compiled_models::entry_relation_key_row::entry_relation_key_row; use crate::compiled_views::models::RelationKeyRow; use crate::compiled_views::types::RelationKeys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; +use sqlbuild_rules::relation_keys::models::DeclaredRelation; +use std::collections::BTreeMap; /// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. -#[inline] pub fn relation_keys<'py>( py: Python<'py>, - rows: Vec, + mut rows: Vec, (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), one_row_seeds: &[String], ) -> PyResult { - relation_keys::relation_keys(py, rows, (seeds, sources), one_row_seeds) + for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { + for entry in entries.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if let Some(row) = source_view_key_row(&entry) { + rows.push(row?); + continue; + } + let name: String = native_field(&entry, "name")?.extract()?; + rows.push(entry_relation_key_row( + kind, + name, + &PyDict::new(py), + &entry, + )?); + } + } + let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); + for relation in &mut relations { + relation.at_most_one_row = + relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); + } + let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + Ok(keys.into_iter().collect()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs index 69e5897717..cd255c841a 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs @@ -1,17 +1,25 @@ //! The `CompiledRelationLocation` of a destination. -use crate::compiled_views::_helpers::compiled_models::compiled_model_view; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DATABASE_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::LOGICAL_DATABASE_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::LOGICAL_SCHEMA_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SCHEMA_KEY; use crate::compiled_views::models::CompiledViewClasses; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_project::models::DestinationFact; /// The `CompiledRelationLocation` of a destination. -#[inline] pub fn relation_location( classes: &CompiledViewClasses<'_>, destination: &DestinationFact, ) -> PyResult> { - compiled_model_view::relation_location(classes, destination) + let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); + fields.set_item(DATABASE_KEY, &destination.database)?; + fields.set_item(SCHEMA_KEY, &destination.schema)?; + fields.set_item("name", &destination.name)?; + fields.set_item("qualified_name", &destination.qualified_name)?; + fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; + fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; + Ok(classes.relation_location.call((), Some(&fields))?.unbind()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs index 647befff8f..6354ed8456 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs @@ -1,12 +1,14 @@ //! The model file's relative path. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model file's relative path. -#[inline] pub fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::relative_path(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.relative_path(py)?.into_bound(py)); + } + native_field(model, "relative_path") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs index cb964b3ba4..84c8676b79 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs @@ -1,12 +1,14 @@ //! The model's attached `SchemaModelEntry`, or `None`. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The model's attached `SchemaModelEntry`, or `None`. -#[inline] pub fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - model_reads::schema_entry(model) + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.schema_entry(py)?.into_bound(py)); + } + native_field(model, "schema_entry") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs index 58b5e86718..a9056cd62c 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs @@ -1,12 +1,13 @@ //! What `relation_keys` reads for a source: its native view itself, or its `source_entry`. -use crate::compiled_views::_helpers::compiled_models::relation_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::main::source_views::read_source_view::read_source_view; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. -#[inline] pub fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - relation_keys::source_key_input(source) + if read_source_view(source, |_, _| Ok(())).is_some() { + return Ok(source.clone()); + } + native_field(source, "source_entry") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs index e0113cb1d2..d6cb2376aa 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs @@ -1,12 +1,19 @@ //! `(has star, star resolved)` of the model's analysed query. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `(has star, star resolved)` of the model's analysed query. -#[inline] pub fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { - model_reads::star_flags(model) + if let Some(view) = native_model(model)? { + return Ok(( + view.fast_lineage_has_star(model.py())?, + view.fast_lineage_star_resolved(model.py())?, + )); + } + Ok(( + native_field(model, "fast_lineage_has_star")?.is_truthy()?, + native_field(model, "fast_lineage_star_resolved")?.is_truthy()?, + )) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs index 0563ef98ab..976f0b4572 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs @@ -1,12 +1,16 @@ //! Output columns whose types semantic recovery forgot, in any order. -use crate::compiled_views::_helpers::compiled_models::model_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Output columns whose types semantic recovery forgot, in any order. -#[inline] pub fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { - model_reads::unchecked_columns(model) + if let Some(view) = native_model(model)? { + return Ok(view.unchecked_columns().to_vec()); + } + native_field(model, "unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs index 19206139c3..73b1ed3634 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs @@ -1,13 +1,25 @@ //! The declared column facts of `model_input`'s attached schema, empty without one. -use crate::compiled_views::_helpers::model_inputs::model_input_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The declared column facts of `model_input`'s attached schema, empty without one. -#[inline] pub fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { - model_input_view::attached_columns(model_input) + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read_with_owner(py, |owner, record| { + Ok(owner + .input_schema_fact(record) + .map(|fact| fact.columns.clone()) + .unwrap_or_default()) + }); + } + let entry: Bound<'_, PyAny> = native_field(model_input, "schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + python_column_facts(&native_field(&entry, "columns")?, None) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs index 442cd1b07f..179767ea94 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs @@ -1,13 +1,33 @@ //! The attached schema of `model_input`; a native input answers without building its entry. -use crate::compiled_views::_helpers::model_inputs::model_input_view; use crate::compiled_views::models::AttachedSchema; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; /// The attached schema of `model_input`; a native input answers without building its entry. -#[inline] pub fn attached_schema<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { - model_input_view::attached_schema(model_input) + let py: Python<'py> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read(py, |record, py| { + let Some(schema) = &record.schema else { + return Ok(None); + }; + Ok(Some(AttachedSchema { + name: record.name.clone(), + dynamic_columns: schema.host.dynamic_columns.read(py)?, + type_enforcement: schema.type_enforcement, + })) + }); + } + let entry: Bound<'py, PyAny> = native_field(model_input, "schema_entry")?; + if entry.is_none() { + return Ok(None); + } + Ok(Some(AttachedSchema { + name: native_field(&entry, "name")?.extract()?, + dynamic_columns: family_specs(&native_field(&entry, "dynamic_columns")?)?, + type_enforcement: native_field(&entry, "type_enforcement")?.is_truthy()?, + })) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs index aa8f388ae9..8b1ecd7ee1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs @@ -1,12 +1,16 @@ //! The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -use crate::compiled_views::_helpers::model_inputs::model_input_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -#[inline] pub fn input_analysis_sql(model_input: &Bound<'_, PyAny>, dialect: &str) -> PyResult { - model_input_view::input_analysis_sql(model_input, dialect) + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) + }); + } + native_field(model_input, "query_sql")?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs index ebf0a2f342..8d0adf5eb8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs @@ -1,12 +1,18 @@ //! The name of `model_input`, its file stem; a native input answers from its record. -use crate::compiled_views::_helpers::model_inputs::model_input_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The name of `model_input`, its file stem; a native input answers from its record. -#[inline] pub fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { - model_input_view::input_model_name(model_input) + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(model_input.py(), |record, _| Ok(record.name.clone())); + } + native_field(model_input, "model_file")? + .getattr("file_path")? + .getattr("stem")? + .extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs index 9137bb66f6..9d51115d52 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs @@ -1,13 +1,21 @@ //! Each reference of `model_input` as `(kind, name, package, call argument count)`. -use crate::compiled_views::_helpers::model_inputs::model_input_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::model_inputs::model_input_view::reference_row_of; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Each reference of `model_input` as `(kind, name, package, call argument count)`. -#[inline] pub fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { - model_input_view::input_reference_rows(model_input) + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(py, |record, _| Ok(record.references.clone())); + } + native_field(model_input, "references")? + .try_iter()? + .map(|reference| reference_row_of(&reference?)) + .collect() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs index de3213f3d9..adbf94a3dd 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs @@ -1,13 +1,15 @@ //! What the analysed SQL of a native model input must show for its SQL macro calls to fit. -use crate::compiled_views::_helpers::model_inputs::model_input_view; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; /// What the analysed SQL of a native model input must show for its SQL macro calls to fit. -#[inline] pub fn input_sql_macro_checks(model_input: &Bound<'_, PyAny>) -> PyResult> { - model_input_view::input_sql_macro_checks(model_input) + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(record.sql_macro_checks.clone()) + }); + } + Ok(Vec::new()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs index 0068f4eb7d..6c1f3cd3ef 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs @@ -1,19 +1,36 @@ //! `CompileSqlReference` records of `rows`, built once into `cache`. -use crate::compiled_views::_helpers::model_inputs::model_input_view; use crate::compiled_views::models::InputViewClasses; -use pyo3::prelude::Py; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::PyTuple; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_project::models::ReferenceRow; /// `CompileSqlReference` records of `rows`, built once into `cache`. -#[inline] pub fn reference_tuple( classes: &InputViewClasses<'_>, rows: &[ReferenceRow], cache: &PyOnceLock>, ) -> PyResult> { - model_input_view::reference_tuple(classes, rows, cache) + let py: Python<'_> = classes.sql_reference.py(); + cache + .get_or_try_init(py, || { + let mut references: Vec> = Vec::with_capacity(rows.len()); + for ReferenceRow { + kind, + name, + package, + call_argument_count: count, + } in rows + { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; + fields.set_item("ref_name", name)?; + fields.set_item("ref_package", package)?; + fields.set_item("call_argument_count", count)?; + references.push(classes.sql_reference.call((), Some(&fields))?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) + }) + .map(|references| references.clone_ref(py)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs index 5e485dbf78..092dde72df 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs @@ -1,17 +1,13 @@ //! The classes the owner's views project attributes with. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::model_inputs::model_input_view; use crate::compiled_views::models::InputViewClasses; -use pyo3::prelude::Py; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Py, PyAnyMethods, PyResult, Python}; /// The classes the owner's views project attributes with. -#[inline] pub fn view_classes<'py>( owner: &Py, py: Python<'py>, ) -> PyResult> { - model_input_view::view_classes(owner, py) + owner.bind(py).borrow().input_classes(py)?.extract() } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs index 4bb78901c7..a7956469d9 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs @@ -1,12 +1,15 @@ //! The project's compiled audits. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::AUDITS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled audits. -#[inline] pub fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::audits(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.audits(py).into_bound(py)); + } + native_field(project, AUDITS_FIELD) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs index 6cc7667444..a4f8bca998 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs @@ -1,12 +1,26 @@ //! The project's resource collection named `field`; any other field is read by attribute. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::AUDITS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::FUNCTIONS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::MODELS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::SEEDS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::SOURCES_FIELD; +use crate::compiled_views::main::project::audits::audits; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's resource collection named `field`; any other field is read by attribute. -#[inline] pub fn collection<'py>(project: &Bound<'py, PyAny>, field: &str) -> PyResult> { - project_reads::collection(project, field) + match field { + MODELS_FIELD => compiled_models(project), + SOURCES_FIELD => sources(project), + SEEDS_FIELD => seeds(project), + FUNCTIONS_FIELD => functions(project), + AUDITS_FIELD => audits(project), + _ => native_field(project, field), + } } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs index 095a175f91..6c092c4f49 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs @@ -1,17 +1,37 @@ //! The native owner's inputs view when it holds every model input; else a record. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// The native owner's inputs view when it holds every model input; else a record. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, fields, native_project))] pub fn compile_inputs_view<'py>( fields: &Bound<'py, PyDict>, native_project: Option>, ) -> PyResult> { - records::compile_inputs_view(fields, native_project) + let py: Python<'py> = fields.py(); + if let Some(project) = &native_project { + let mut native: bool = true; + for model in fields.as_any().get_item("model_inputs")?.try_iter()? { + if !model?.is_instance_of::() { + native = false; + break; + } + } + if native { + let view: NativeInputsView = + NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; + return view.into_bound_py_any(py); + } + } + let keywords: Bound<'py, PyDict> = fields.copy()?; + keywords.set_item("native_project", native_project)?; + new_record(py, RecordId::CompileProjectInputs, (), Some(&keywords)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs index e33fba8e33..7d86acd4e1 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs @@ -1,12 +1,12 @@ //! The audit as a `CompiledAudit`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The audit as a `CompiledAudit`, projected from its owner when it is a native view. -#[inline] +#[pyfunction] pub fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - records::compiled_audit_record(audit) + record(audit, RecordId::CompiledAudit) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs index 22243f5df2..c7aca51088 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs @@ -1,12 +1,12 @@ //! The function as a `CompiledFunction`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The function as a `CompiledFunction`, projected from its owner when it is a native view. -#[inline] +#[pyfunction] pub fn compiled_function_record<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - records::compiled_function_record(function) + record(function, RecordId::CompiledFunction) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs index 62876a2a3a..bacd080555 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs @@ -1,12 +1,12 @@ //! The model as a `CompiledModel`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The model as a `CompiledModel`, projected from its owner when it is a native view. -#[inline] +#[pyfunction] pub fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - records::compiled_model_record(model) + record(model, RecordId::CompiledModel) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs index e061552284..7052ab01eb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs @@ -1,12 +1,15 @@ //! The project's compiled models, in compile order. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::MODELS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled models, in compile order. -#[inline] pub fn compiled_models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::models(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.models(py).into_bound(py).into_any()); + } + native_field(project, MODELS_FIELD) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs index fcc6df7b04..c6f60aa64c 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs @@ -1,15 +1,23 @@ //! The compile's retained project, or one recorded from a hand-built project. use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::hand_built_project; +use crate::compiled_views::models::NativeProjectView; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compile's retained project, or one recorded from a hand-built project. -#[inline] +#[pyfunction] pub fn compiled_project_facts<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { - records::compiled_project_facts(project) + if let Ok(view) = project.downcast::() { + return Ok(view.get().owner().bind(project.py()).clone()); + } + let retained: Bound<'py, PyAny> = native_field(project, "native_project")?; + if !retained.is_none() { + return Ok(retained.downcast_into()?); + } + hand_built_project(project) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs index 6dd53396d3..0bb27207d6 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs @@ -1,12 +1,12 @@ //! The seed as a `CompiledSeed`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The seed as a `CompiledSeed`, projected from its owner when it is a native view. -#[inline] +#[pyfunction] pub fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - records::compiled_seed_record(seed) + record(seed, RecordId::CompiledSeed) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs index 22765746a8..e7c8678f22 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs @@ -1,12 +1,12 @@ //! The source as a `CompiledSource`, projected from its owner when it is a native view. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The source as a `CompiledSource`, projected from its owner when it is a native view. -#[inline] +#[pyfunction] pub fn compiled_source_record<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - records::compiled_source_record(source) + record(source, RecordId::CompiledSource) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs index c6725009b3..91431fa49b 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs @@ -1,16 +1,26 @@ //! Enum and constant declarations as native rows, in the given order. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyList; +use crate::compiled_views::_helpers::project::records::constant_row; +use crate::compiled_views::_helpers::project::records::enum_row; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyList, PyTuple}; /// Enum and constant declarations as native rows, in the given order. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, enums, constants))] pub fn declaration_rows<'py>( enums: &Bound<'py, PyAny>, constants: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { - records::declaration_rows(enums, constants) + let py: Python<'py> = enums.py(); + let enum_rows: Vec> = enums + .try_iter()? + .map(|declaration| enum_row(&declaration?)) + .collect::>()?; + let constant_rows: Vec> = constants + .try_iter()? + .map(|declaration| constant_row(&declaration?)) + .collect::>()?; + Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs index 395c418e72..2752047bb0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs @@ -1,11 +1,10 @@ //! `declaration_rows` as a Python function. -use crate::compiled_views::_helpers::project::records; +use crate::compiled_views::main::project::declaration_rows::declaration_rows; /// `declaration_rows` as a Python function. -#[inline] pub fn declaration_rows_function( py: pyo3::Python<'_>, ) -> pyo3::PyResult> { - records::declaration_rows_function(py) + pyo3::wrap_pyfunction!(declaration_rows, py) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs index 34b730bbe7..8726bb548f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs @@ -1,12 +1,15 @@ //! The project's compiled functions. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::FUNCTIONS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled functions. -#[inline] pub fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::functions(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.functions(py).into_bound(py)); + } + native_field(project, FUNCTIONS_FIELD) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs index b632dbbea2..ea9d9baacf 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs @@ -1,16 +1,17 @@ //! The compile inputs with `changes`; a native view shares its owner's model inputs. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::with_changes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::records::types::RecordId; /// The compile inputs with `changes`; a native view shares its owner's model inputs. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, inputs, **changes))] pub fn inputs_with<'py>( inputs: &Bound<'py, PyAny>, changes: Option>, ) -> PyResult> { - records::inputs_with(inputs, changes) + with_changes(inputs, RecordId::CompileProjectInputs, changes) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs index e8d05bdfc1..653df3def8 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs @@ -1,12 +1,36 @@ //! A typed SQL value as the plain value native consumers read. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::COLLECTION_KINDS; +use crate::compiled_views::_helpers::project::records::DECIMAL_KIND; +use crate::compiled_views::_helpers::project::records::SCALAR_KINDS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A typed SQL value as the plain value native consumers read. -#[inline] +#[pyfunction] pub fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - records::plain_typed_value(value) + let py: Python<'py> = value.py(); + let kind: String = native_field(value, "kind")?.getattr("value")?.extract()?; + let inner: Bound<'py, PyAny> = native_field(value, "value")?; + if kind == DECIMAL_KIND { + return Ok(inner.str()?.into_any()); + } + if SCALAR_KINDS.contains(&kind.as_str()) { + return Ok(inner); + } + if COLLECTION_KINDS.contains(&kind.as_str()) { + let items: Vec> = inner + .try_iter()? + .map(|item| plain_typed_value(&item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for pair in inner.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + mapping.set_item(key, plain_typed_value(&item)?)?; + } + Ok(mapping.into_any()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs index db14f06169..2265e0dcf0 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs @@ -1,16 +1,17 @@ //! The project with `changes`; a native view shares its owner's models. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::with_changes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::records::types::RecordId; /// The project with `changes`; a native view shares its owner's models. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, **changes))] pub fn project_with<'py>( project: &Bound<'py, PyAny>, changes: Option>, ) -> PyResult> { - records::project_with(project, changes) + with_changes(project, RecordId::CompiledProject, changes) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs index 477189486e..bddd236ce2 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs @@ -1,16 +1,75 @@ //! Transfer each assembled model's typed analysis to its retained facts; return how many. -use crate::compiled_project::models::NativeCompiledProject; -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::models::{AnalysisPy, NativeCompiledProject, ProofPy}; +use crate::compiled_views::_helpers::project::records::family_types; +use crate::compiled_views::models::NativeCompiledModel; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Transfer each assembled model's typed analysis to its retained facts; return how many. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, models))] pub fn record_model_analyses<'py>( project: &Bound<'py, NativeCompiledProject>, models: &Bound<'py, PyAny>, ) -> PyResult { - records::record_model_analyses(project, models) + let py: Python<'py> = project.py(); + let models: Vec> = models.try_iter()?.collect::>()?; + let views: Vec> = models + .iter() + .filter(|model| model.is_instance_of::()) + .map(|model| model.extract::>()) + .collect::>()?; + let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; + let nullable: Bound<'py, PyAny> = enum_member(py, RecordId::InferredNullability, "NULLABLE")?; + for model in &models { + if model.is_instance_of::() { + continue; + } + let inferred: Bound<'py, PyAny> = native_field(model, "inferred_rows")?; + let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { + Vec::new() + } else { + inferred.extract()? + }; + let proof: Bound<'py, PyAny> = native_field(model, "dynamic_column_contract")?; + let proof = if proof.is_none() { + None + } else { + let families: Bound<'py, PyAny> = native_field(&proof, "families")?; + Some(ProofPy { + output_proven: native_field(&proof, "output_proven")?.extract()?, + bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, + families: family_types(&families, true)?, + failure_reason: native_field(&proof, "failure_reason")?.extract()?, + named_families: family_types(&families, false)?, + }) + }; + let analysis = AnalysisPy { + inferred: !inferred.is_none(), + inferred_columns: rows + .iter() + .map(|(name, column_type, _)| (name.clone(), column_type.clone())) + .collect(), + nullable_columns: rows + .iter() + .map(|(_, _, nullability)| nullability.eq(&nullable)) + .collect::>()?, + fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, + unchecked: native_field(model, "unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect::>()?, + }; + let name: String = native_field(model, "name")?.extract()?; + transferred += usize::from( + project + .borrow_mut() + .record_model_analysis(&name, analysis, proof), + ); + } + Ok(transferred) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs index f9ff66dc81..9a0aa56f4f 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs @@ -1,12 +1,43 @@ //! Register the records functions and classes. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::compiled_project_record; +use crate::compiled_views::main::project::compile_inputs_view::compile_inputs_view; +use crate::compiled_views::main::project::compiled_audit_record::compiled_audit_record; +use crate::compiled_views::main::project::compiled_function_record::compiled_function_record; +use crate::compiled_views::main::project::compiled_model_record::compiled_model_record; +use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use crate::compiled_views::main::project::compiled_seed_record::compiled_seed_record; +use crate::compiled_views::main::project::compiled_source_record::compiled_source_record; +use crate::compiled_views::main::project::declaration_rows::declaration_rows; +use crate::compiled_views::main::project::inputs_with::inputs_with; +use crate::compiled_views::main::project::plain_typed_value::plain_typed_value; +use crate::compiled_views::main::project::project_with::project_with; +use crate::compiled_views::main::project::record_model_analyses::record_model_analyses; +use crate::compiled_views::main::project::with_binding_diagnostics::with_binding_diagnostics; +use crate::compiled_views::main::project::with_type_recovery::with_type_recovery; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the records functions and classes. -#[inline] pub fn register_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - records::register(module) + for function in [ + wrap_pyfunction!(compiled_model_record, module)?, + wrap_pyfunction!(compiled_audit_record, module)?, + wrap_pyfunction!(compiled_function_record, module)?, + wrap_pyfunction!(compiled_source_record, module)?, + wrap_pyfunction!(compiled_seed_record, module)?, + wrap_pyfunction!(compiled_project_record, module)?, + wrap_pyfunction!(with_binding_diagnostics, module)?, + wrap_pyfunction!(with_type_recovery, module)?, + wrap_pyfunction!(project_with, module)?, + wrap_pyfunction!(inputs_with, module)?, + wrap_pyfunction!(compile_inputs_view, module)?, + wrap_pyfunction!(plain_typed_value, module)?, + wrap_pyfunction!(declaration_rows, module)?, + wrap_pyfunction!(record_model_analyses, module)?, + wrap_pyfunction!(compiled_project_facts, module)?, + ] { + module.add_function(function)?; + } + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs index 315b0e7cdc..fe804a6c28 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs @@ -1,15 +1,19 @@ //! `dataclasses.replace(value, **changes)`. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::records::DATACLASSES_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::boundary::main::imported::imported; /// `dataclasses.replace(value, **changes)`. -#[inline] pub fn replace<'py>( value: &Bound<'py, PyAny>, changes: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - records::replace(value, changes) + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, change) in changes { + keywords.set_item(*name, change)?; + } + imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs index bf99d00c84..41c2f79b61 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs @@ -1,12 +1,15 @@ //! The project's compiled seeds. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::SEEDS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled seeds. -#[inline] pub fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::seeds(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.seeds(py).into_bound(py)); + } + native_field(project, SEEDS_FIELD) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs index 433f25a2df..b9264ac789 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs @@ -1,12 +1,15 @@ //! The project's compiled sources. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::SOURCES_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled sources. -#[inline] pub fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::sources(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sources(py).into_bound(py)); + } + native_field(project, SOURCES_FIELD) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs index 9fb5eb97fb..5a7f8b71c4 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs @@ -1,12 +1,14 @@ //! The project's compiled SQL tests. -use crate::compiled_views::_helpers::project::project_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's compiled SQL tests. -#[inline] pub fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - project_reads::sql_tests(project) + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sql_tests(py).into_bound(py)); + } + native_field(project, "sql_tests") } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs index aa9b9f2a52..64e9ccc734 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs @@ -1,15 +1,21 @@ //! The model with these final binding diagnostics. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// The model with these final binding diagnostics. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics))] pub fn with_binding_diagnostics<'py>( model: &Bound<'py, PyAny>, diagnostics: &Bound<'py, PyAny>, ) -> PyResult> { - records::with_binding_diagnostics(model, diagnostics) + let py: Python<'py> = model.py(); + if model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { + return replace(model, &[("binding_diagnostics", diagnostics.clone())]); + } + model.call_method1("with_binding_diagnostics", (diagnostics,)) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs index 9494c5b900..8b6842ce90 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs @@ -1,17 +1,58 @@ //! The model with its recovered bindings and its poisoned outputs' types forgotten. -use crate::compiled_views::_helpers::project::records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PyFrozenSetMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The model with its recovered bindings and its poisoned outputs' types forgotten. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics, poisoned))] pub fn with_type_recovery<'py>( model: &Bound<'py, PyAny>, diagnostics: &Bound<'py, PyAny>, poisoned: &Bound<'py, PyFrozenSet>, ) -> PyResult> { - records::with_type_recovery(model, diagnostics, poisoned) + let py: Python<'py> = model.py(); + let name: String = native_field(model, "name")?.extract()?; + let mut unchecked: Vec = Vec::new(); + for pair in poisoned.iter() { + let (model_name, column): (String, String) = pair.extract()?; + if model_name == name { + unchecked.push(column); + } + } + if !model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { + return model.call_method1("with_type_recovery", (diagnostics, unchecked)); + } + let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; + let columns: Bound<'py, PyAny> = if inferred.is_none() { + inferred + } else { + let mut columns: Vec> = Vec::new(); + for column in inferred.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let key = (name.as_str(), native_field(&column, "name")?); + columns.push(if poisoned.contains(key)? { + replace(&column, &[("type", py.None().into_bound(py))])? + } else { + column + }); + } + PyTuple::new(py, columns)?.into_any() + }; + replace( + model, + &[ + ("binding_diagnostics", diagnostics.clone()), + ("inferred_columns", columns), + ( + "unchecked_output_columns", + PyFrozenSet::new(py, &unchecked)?.into_any(), + ), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index 0f54ef21a8..f078f57117 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -17,12 +17,8 @@ use crate::compiled_views::_helpers::compiled_models::compiled_model_view::ViewP use crate::compiled_views::_helpers::compiled_models::compiled_model_view::casefolded_families; use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_of; use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_record; -use crate::compiled_views::_helpers::compiled_models::compiled_model_view::defaulted_destination; use crate::compiled_views::_helpers::compiled_models::compiled_model_view::lineage_input; -use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_graph_row; use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_name; -use crate::compiled_views::_helpers::compiled_models::compiled_model_view::object_key; -use crate::compiled_views::_helpers::compiled_models::compiled_model_view::relation_location; use crate::compiled_views::_helpers::model_inputs::inputs_view::InputsFields; use crate::compiled_views::_helpers::model_inputs::model_input_view::attached_schema_name; use crate::compiled_views::_helpers::model_inputs::model_input_view::authored_expansion; @@ -34,12 +30,16 @@ use crate::compiled_views::_helpers::model_inputs::model_input_view::read_input_ use crate::compiled_views::_helpers::model_inputs::model_input_view::references; use crate::compiled_views::_helpers::model_inputs::model_input_view::rejected_opt_out; use crate::compiled_views::_helpers::model_inputs::model_input_view::schema_entry; -use crate::compiled_views::_helpers::model_inputs::model_input_view::view_classes; use crate::compiled_views::_helpers::project::project_view::CompilerHandles; use crate::compiled_views::_helpers::project::project_view::ProjectFields; use crate::compiled_views::_helpers::project::project_view::SCHEMA_ENTRY_ATTRIBUTE; use crate::compiled_views::_helpers::project::project_view::TAGS_ATTRIBUTE; use crate::compiled_views::_helpers::project::project_view::resource_row; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; use crate::compiled_views::types::DeclaredRow; use crate::compiled_views::types::LineageInput; use crate::compiled_views::types::ModelInputRecord; diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs index a5d31f1819..861ac32076 100644 --- a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs @@ -2,11 +2,9 @@ use std::collections::HashMap; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use serde_json::Value; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; @@ -21,10 +19,9 @@ use sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::main::compiled_models::enum_columns::enum_columns; use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; -use crate::contracts::_helpers::model_contracts::ConflictRow; -use crate::contracts::_helpers::model_contracts::DiagnosticRow; -use crate::contracts::_helpers::model_contracts::evaluate_project_model_contracts; -use crate::contracts::_helpers::model_contracts::project_promotion_conflicts; +use crate::contracts::_helpers::model_contracts::{ + DiagnosticRow, evaluate_project_model_contracts, +}; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -32,14 +29,14 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::sqltext::main::location_rows::location_rows; use sqlbuild_python_front::sqltext::models::ColumnLocation; -const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; const GENERIC_DIALECT: &str = "generic"; const OUTPUT_LABEL: &str = "output"; -const CONTRACT_PHASE: &str = "contract"; +pub(crate) const CONTRACT_PHASE: &str = "contract"; const MODEL_RESOURCE: &str = "model"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; -fn keyword_call<'py>( +pub(crate) fn keyword_call<'py>( callable: &Bound<'py, PyAny>, pairs: Vec<(&str, Bound<'py, PyAny>)>, ) -> PyResult> { @@ -51,7 +48,7 @@ fn keyword_call<'py>( } /// The compile's retained native project, read through `_native.compiled_project_facts`. -fn native_project<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn native_project<'py>(project: &Bound<'py, PyAny>) -> PyResult> { Ok(compiled_project_facts(project)?.into_any()) } @@ -283,7 +280,7 @@ fn enum_literal_diagnostic<'py>( ) } -fn model_resource_type(py: Python<'_>) -> PyResult> { +pub(crate) fn model_resource_type(py: Python<'_>) -> PyResult> { enum_from_value(py, RecordId::CompiledResourceType, MODEL_RESOURCE) } @@ -349,59 +346,3 @@ pub(crate) fn evaluate_model_contracts<'py>( vec![("diagnostics", PyTuple::new(py, diagnostics)?.into_any())], ) } - -/// One K011 error per enforced-contract table model under immediate promotion. -#[pyfunction] -#[pyo3(signature = (*, project, adapter_default, settings_file))] -pub(crate) fn promotion_conflict_diagnostics<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, - adapter_default: &Bound<'py, PyAny>, - settings_file: String, -) -> PyResult> { - let explicit_mode: Option = - native_field(&native_field(project, "settings")?, "table_promotion_mode")?.extract()?; - let native: Bound<'py, PyAny> = native_project(project)?; - let native: PyRef<'py, NativeCompiledProject> = native.extract()?; - let conflicts: Vec = project_promotion_conflicts( - py, - ( - explicit_mode, - adapter_default.str()?.extract()?, - settings_file, - ), - &native, - )?; - let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; - let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; - let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); - for ConflictRow { - name, - relative_path, - code, - message, - help, - } in conflicts - { - diagnostics.push(keyword_call( - &diagnostic_type, - vec![ - ("phase", CONTRACT_PHASE.into_pyobject(py)?.into_any()), - ("severity", "error".into_pyobject(py)?.into_any()), - ("code", code.into_pyobject(py)?.into_any()), - ("message", message.into_pyobject(py)?.into_any()), - ("resource_type", model_resource_type(py)?), - ("resource_name", name.into_pyobject(py)?.into_any()), - ("path", path_type.call1((relative_path,))?), - ("help", help.into_pyobject(py)?.into_any()), - ], - )?); - } - PyTuple::new(py, diagnostics) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_model_contracts, module)?)?; - module.add_function(wrap_pyfunction!(promotion_conflict_diagnostics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs index 5e24f800c2..f3cb42f2e7 100644 --- a/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs @@ -2,7 +2,6 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; create_exception!( sqlbuild._native, @@ -10,10 +9,3 @@ create_exception!( PyValueError, "An authored value cannot be represented by the typed SQL value domain." ); - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add( - "SqlValueValidationError", - module.py().get_type::(), - ) -} diff --git a/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs index e215029be9..4b3d34f96e 100644 --- a/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs +++ b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs @@ -1,10 +1,7 @@ //! Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. use crate::contracts::_helpers::contract_diagnostics; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. #[inline] diff --git a/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs index c233ef8ec7..131ddfada9 100644 --- a/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs @@ -1,24 +1,65 @@ //! One K011 error per enforced-contract table model under immediate promotion. -use crate::contracts::_helpers::contract_diagnostics; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compiled_project::models::NativeCompiledProject; +use crate::contracts::_helpers::contract_diagnostics::CONTRACT_PHASE; +use crate::contracts::_helpers::contract_diagnostics::PATHLIB_MODULE; +use crate::contracts::_helpers::contract_diagnostics::keyword_call; +use crate::contracts::_helpers::contract_diagnostics::model_resource_type; +use crate::contracts::_helpers::contract_diagnostics::native_project; +use crate::contracts::_helpers::model_contracts::{ConflictRow, project_promotion_conflicts}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// One K011 error per enforced-contract table model under immediate promotion. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, adapter_default, settings_file))] pub fn promotion_conflict_diagnostics<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, adapter_default: &Bound<'py, PyAny>, settings_file: String, ) -> PyResult> { - contract_diagnostics::promotion_conflict_diagnostics( + let explicit_mode: Option = + native_field(&native_field(project, "settings")?, "table_promotion_mode")?.extract()?; + let native: Bound<'py, PyAny> = native_project(project)?; + let native: PyRef<'py, NativeCompiledProject> = native.extract()?; + let conflicts: Vec = project_promotion_conflicts( py, - project, - adapter_default, - settings_file, - ) + ( + explicit_mode, + adapter_default.str()?.extract()?, + settings_file, + ), + &native, + )?; + let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; + let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; + let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); + for ConflictRow { + name, + relative_path, + code, + message, + help, + } in conflicts + { + diagnostics.push(keyword_call( + &diagnostic_type, + vec![ + ("phase", CONTRACT_PHASE.into_pyobject(py)?.into_any()), + ("severity", "error".into_pyobject(py)?.into_any()), + ("code", code.into_pyobject(py)?.into_any()), + ("message", message.into_pyobject(py)?.into_any()), + ("resource_type", model_resource_type(py)?), + ("resource_name", name.into_pyobject(py)?.into_any()), + ("path", path_type.call1((relative_path,))?), + ("help", help.into_pyobject(py)?.into_any()), + ], + )?); + } + PyTuple::new(py, diagnostics) } diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs index f237d80000..e98b71d02d 100644 --- a/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs @@ -1,12 +1,13 @@ //! Register the contract diagnostics functions and classes. -use crate::contracts::_helpers::contract_diagnostics; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::contract_diagnostics::evaluate_model_contracts; +use crate::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the contract diagnostics functions and classes. -#[inline] pub fn register_contract_diagnostics(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_diagnostics::register(module) + module.add_function(wrap_pyfunction!(evaluate_model_contracts, module)?)?; + module.add_function(wrap_pyfunction!(promotion_conflict_diagnostics, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs index 7640032d19..0b0e25703e 100644 --- a/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs +++ b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs @@ -1,12 +1,12 @@ //! Register the exception types functions and classes. -use crate::contracts::_helpers::exception_types; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::exception_types::SqlValueValidationError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the exception types functions and classes. -#[inline] pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { - exception_types::register(module) + module.add( + "SqlValueValidationError", + module.py().get_type::(), + ) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs index 475ff3013d..cc35719d0b 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs @@ -1,30 +1,26 @@ //! Latest fingerprint rows decoded into `Fingerprint` models and indexed into a `FingerprintSet`. +use crate::fingerprints::main::row_fingerprint::row_fingerprint; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::ffi::CString; use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBytes, PyDateTime, PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyString}; use sqlbuild_analysis::fingerprint_storage::errors::StorageDecodeError; use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage; use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; -use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; const BASE64_MODULE: &str = "base64"; -const TIMESTAMP_COLUMN: usize = 11; +pub(crate) const TIMESTAMP_COLUMN: usize = 11; /// `FingerprintInputError(message)`. pub(crate) fn fingerprint_input_error(py: Python<'_>, message: &str) -> PyErr { @@ -55,39 +51,7 @@ pub(crate) fn fingerprint_set_from_rows<'py>( indexed_set(py, schema, &fingerprints) } -/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. -#[pyfunction] -#[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] -pub(crate) fn select_latest_fingerprints<'py>( - py: Python<'py>, - fingerprint_set: &Bound<'py, PyAny>, - node_names: Vec, - filtered_node_types: Vec, -) -> PyResult> { - let schema: String = native_field(fingerprint_set, "schema")?.extract()?; - if node_names.is_empty() && filtered_node_types.is_empty() { - return indexed_set(py, &schema, &[]); - } - let by_identity: Bound<'py, PyAny> = native_field(fingerprint_set, "fingerprints_by_identity")?; - let latest: Vec> = if by_identity.is_truthy()? { - mapping_values(&by_identity)? - .try_iter()? - .collect::>()? - } else { - Vec::new() - }; - let identities: Vec<(String, String)> = - latest.iter().map(identity_of).collect::>()?; - let positions: Vec = - selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); - let selected: Vec> = positions - .into_iter() - .map(|position| latest[position].clone()) - .collect(); - indexed_set(py, &schema, &selected) -} - -fn indexed_set<'py>( +pub(crate) fn indexed_set<'py>( py: Python<'py>, schema: &str, fingerprints: &[Bound<'py, PyAny>], @@ -112,14 +76,14 @@ fn indexed_set<'py>( new_record(py, RecordId::FingerprintSet, (), Some(&kwargs)) } -fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { +pub(crate) fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { Ok(( native_field(fingerprint, "node_type")?.extract()?, native_field(fingerprint, "node_name")?.extract()?, )) } -fn borrowed(identities: &[(String, String)]) -> Vec> { +pub(crate) fn borrowed(identities: &[(String, String)]) -> Vec> { identities .iter() .map(|(node_type, node_name)| FingerprintIdentity { @@ -129,45 +93,9 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -pub(crate) fn row_fingerprint<'py>( - fingerprint_class: &Bound<'py, PyAny>, - from_iso: &Bound<'py, PyAny>, - row: &Bound<'py, PyAny>, - qualified_name: &str, -) -> PyResult> { - let py: Python<'py> = row.py(); - let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; - let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { - raw_ts - } else { - from_iso.call1((raw_ts.str()?,))? - }; - let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; - let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; - let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; - let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; - let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; - let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; - let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; - let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; - let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; - let kwargs: Bound<'py, PyDict> = PyDict::new(py); - kwargs.set_item("node_type", node_type)?; - kwargs.set_item("node_name", node_name)?; - kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; - kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; - kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; - kwargs.set_item("run_id", row.get_item(5)?.str()?)?; - kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; - kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; - kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; - kwargs.set_item("definition", definition)?; - kwargs.set_item("metadata_json", metadata_json)?; - kwargs.set_item("ts", ts)?; - fingerprint_class.call((), Some(&kwargs)) -} - -fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn optional_text<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult>> { if value.is_none() { return Ok(None); } @@ -175,7 +103,7 @@ fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult, node_name: &Bound<'_, PyString>, qualified_name: &str, @@ -244,8 +172,3 @@ fn decode_error(storage: &Bound<'_, PyString>, failure: StorageDecodeError) -> P } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs index f9d56acbf5..970087d552 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs @@ -1,8 +1,7 @@ //! Query and function-definition change fingerprints. -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; use sqlbuild_analysis::sql_tokens::main::normalized_query_text::normalized_query_text; @@ -14,7 +13,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; /// `QueryFingerprintError(message)`, or the compiler error a native panic becomes. -fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { if is_compiler_panic(&message) { return compiler_error(message); } @@ -24,22 +23,10 @@ fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { } } -/// The persisted SHA-256 fingerprint of the query's canonical token stream. -#[pyfunction] -#[pyo3(signature = (*, query_sql, dialect))] -pub(crate) fn compute_query_hash( - py: Python<'_>, - query_sql: &str, - dialect: Option<&str>, -) -> PyResult { - py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) - .map_err(|error| query_fingerprint_error(py, error)) -} - /// A function definition's fingerprint: SQL bodies like queries, other languages exactly. #[pyfunction] #[pyo3(signature = (*, fingerprint_sql, language, dialect))] -fn compute_function_definition_hash( +pub(crate) fn compute_function_definition_hash( py: Python<'_>, fingerprint_sql: &str, language: &str, @@ -51,13 +38,6 @@ fn compute_function_definition_hash( /// Query SQL stripped with its whitespace runs collapsed. #[pyfunction] -fn normalize_query_sql(query_sql: &str) -> String { +pub(crate) fn normalize_query_sql(query_sql: &str) -> String { normalized_query_text(query_sql) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_query_hash, module)?)?; - module.add_function(wrap_pyfunction!(compute_function_definition_hash, module)?)?; - module.add_function(wrap_pyfunction!(normalize_query_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs index 482711d083..cbf9f38911 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs @@ -2,11 +2,9 @@ use pyo3::IntoPyObject; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::main::create_table_sql::create_table_sql; use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; use sqlbuild_analysis::fingerprint_storage::main::relevant_read_sql::relevant_read_sql; @@ -14,10 +12,12 @@ use sqlbuild_analysis::fingerprint_storage::models::RelevantRead; use sqlbuild_host::entries::models::AdapterHost; -use crate::fingerprints::_helpers::latest_rows::fingerprint_input_error; -use crate::fingerprints::_helpers::latest_rows::fingerprint_set_from_rows; -use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_batches; -use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_sql; +use crate::fingerprints::_helpers::latest_rows::{ + fingerprint_input_error, fingerprint_set_from_rows, +}; +use crate::fingerprints::_helpers::storage_sql::{ + fingerprint_insert_batches, fingerprint_insert_sql, +}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -274,7 +274,7 @@ fn present<'py>(request: &Request<'_, 'py>, name: &str) -> PyResult, schema: &str, render_qualified_name: &Bound<'_, PyAny>, @@ -291,7 +291,7 @@ fn build_create_table_sql( /// The windowed latest-row read of a target schema's fingerprint table. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name))] -fn build_read_latest_sql( +pub(crate) fn build_read_latest_sql( database: Option<&str>, schema: &str, render_qualified_name: &Bound<'_, PyAny>, @@ -392,7 +392,7 @@ pub(crate) fn read_latest_fingerprints<'py>( /// Append one fingerprint row, retrying transient concurrent-writer conflicts. With `ensure_table` false the table is assumed to exist and is set up only after the insert fails; returns whether the write set up the table. #[pyfunction] #[pyo3(signature = (**request))] -fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult { +pub(crate) fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult { let request = Request::new("write_fingerprint", request)?; let py: Python<'py> = request.py(); let fingerprint: Bound<'py, PyAny> = request.get("fingerprint")?; @@ -446,7 +446,7 @@ fn batch_warning<'py>(batch: &[Bound<'py, PyAny>], error: &PyErr) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { +pub(crate) fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { let request = Request::new("write_fingerprints", request)?; let py: Python<'py> = request.py(); let fingerprints: Vec> = request @@ -487,12 +487,3 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> } Ok(()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_create_table_sql, module)?)?; - module.add_function(wrap_pyfunction!(build_read_latest_sql, module)?)?; - module.add_function(wrap_pyfunction!(read_latest_fingerprints, module)?)?; - module.add_function(wrap_pyfunction!(write_fingerprint, module)?)?; - module.add_function(wrap_pyfunction!(write_fingerprints, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs index 86111b2adb..3021a285f9 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs @@ -1,15 +1,19 @@ //! The persisted SHA-256 fingerprint of the query's canonical token stream. -use crate::fingerprints::_helpers::query_hashes; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::fingerprints::_helpers::query_hashes::query_fingerprint_error; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use sqlbuild_python_base::types::CompilerDetach; /// The persisted SHA-256 fingerprint of the query's canonical token stream. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, query_sql, dialect))] pub fn compute_query_hash( py: Python<'_>, query_sql: &str, dialect: Option<&str>, ) -> PyResult { - query_hashes::compute_query_hash(py, query_sql, dialect) + py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) + .map_err(|error| query_fingerprint_error(py, error)) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs index b2ca7b0b3c..d197e13523 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs @@ -1,9 +1,7 @@ //! The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. use crate::fingerprints::_helpers::storage_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs index 11b572ab79..eebc173054 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs @@ -1,12 +1,11 @@ //! Register the latest rows functions and classes. -use crate::fingerprints::_helpers::latest_rows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the latest rows functions and classes. -#[inline] pub fn register_latest_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { - latest_rows::register(module) + module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs index 997ac33603..f7015075e6 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs @@ -1,12 +1,15 @@ //! Register the query hashes functions and classes. -use crate::fingerprints::_helpers::query_hashes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::fingerprints::_helpers::query_hashes::compute_function_definition_hash; +use crate::fingerprints::_helpers::query_hashes::normalize_query_sql; +use crate::fingerprints::main::compute_query_hash::compute_query_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the query hashes functions and classes. -#[inline] pub fn register_query_hashes(module: &Bound<'_, PyModule>) -> PyResult<()> { - query_hashes::register(module) + module.add_function(wrap_pyfunction!(compute_query_hash, module)?)?; + module.add_function(wrap_pyfunction!(compute_function_definition_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize_query_sql, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs index 863938c25b..8ce88cc13e 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs @@ -1,12 +1,19 @@ //! Register the storage io functions and classes. -use crate::fingerprints::_helpers::storage_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::fingerprints::_helpers::storage_io::build_create_table_sql; +use crate::fingerprints::_helpers::storage_io::build_read_latest_sql; +use crate::fingerprints::_helpers::storage_io::read_latest_fingerprints; +use crate::fingerprints::_helpers::storage_io::write_fingerprint; +use crate::fingerprints::_helpers::storage_io::write_fingerprints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the storage io functions and classes. -#[inline] pub fn register_storage_io(module: &Bound<'_, PyModule>) -> PyResult<()> { - storage_io::register(module) + module.add_function(wrap_pyfunction!(build_create_table_sql, module)?)?; + module.add_function(wrap_pyfunction!(build_read_latest_sql, module)?)?; + module.add_function(wrap_pyfunction!(read_latest_fingerprints, module)?)?; + module.add_function(wrap_pyfunction!(write_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(write_fingerprints, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs index 939bb6fe8c..04054c98a2 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs @@ -1,17 +1,46 @@ //! Row fingerprint. -use crate::fingerprints::_helpers::latest_rows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::fingerprints::_helpers::latest_rows::TIMESTAMP_COLUMN; +use crate::fingerprints::_helpers::latest_rows::decoded; +use crate::fingerprints::_helpers::latest_rows::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDateTime, PyDict, PyString}; /// Row fingerprint. -#[inline] pub fn row_fingerprint<'py>( fingerprint_class: &Bound<'py, PyAny>, from_iso: &Bound<'py, PyAny>, row: &Bound<'py, PyAny>, qualified_name: &str, ) -> PyResult> { - latest_rows::row_fingerprint(fingerprint_class, from_iso, row, qualified_name) + let py: Python<'py> = row.py(); + let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; + let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { + raw_ts + } else { + from_iso.call1((raw_ts.str()?,))? + }; + let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; + let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; + let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; + let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; + let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; + let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; + let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; + let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; + let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("node_type", node_type)?; + kwargs.set_item("node_name", node_name)?; + kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; + kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; + kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; + kwargs.set_item("run_id", row.get_item(5)?.str()?)?; + kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; + kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; + kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; + kwargs.set_item("definition", definition)?; + kwargs.set_item("metadata_json", metadata_json)?; + kwargs.set_item("ts", ts)?; + fingerprint_class.call((), Some(&kwargs)) } diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs index 4a60426de9..bfc725ca9b 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs @@ -1,18 +1,42 @@ //! `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. -use crate::fingerprints::_helpers::latest_rows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::fingerprints::_helpers::latest_rows::borrowed; +use crate::fingerprints::_helpers::latest_rows::identity_of; +use crate::fingerprints::_helpers::latest_rows::indexed_set; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; /// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] pub fn select_latest_fingerprints<'py>( py: Python<'py>, fingerprint_set: &Bound<'py, PyAny>, node_names: Vec, filtered_node_types: Vec, ) -> PyResult> { - latest_rows::select_latest_fingerprints(py, fingerprint_set, node_names, filtered_node_types) + let schema: String = native_field(fingerprint_set, "schema")?.extract()?; + if node_names.is_empty() && filtered_node_types.is_empty() { + return indexed_set(py, &schema, &[]); + } + let by_identity: Bound<'py, PyAny> = native_field(fingerprint_set, "fingerprints_by_identity")?; + let latest: Vec> = if by_identity.is_truthy()? { + mapping_values(&by_identity)? + .try_iter()? + .collect::>()? + } else { + Vec::new() + }; + let identities: Vec<(String, String)> = + latest.iter().map(identity_of).collect::>()?; + let positions: Vec = + selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); + let selected: Vec> = positions + .into_iter() + .map(|position| latest[position].clone()) + .collect(); + indexed_set(py, &schema, &selected) } diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs index 880f655b82..6b9118af75 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs @@ -1,44 +1,19 @@ //! Closure, edge inversion and path nodes over any Python-keyed edge mapping. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyTuple, -}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; +use crate::graph::main::nodes::closure::closure; +use crate::graph::main::nodes::inverted::inverted; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use sqlbuild_core::graph_walk::main::directed_path_nodes::directed_path_nodes; use crate::graph::models::PyNodeSet; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; const BUILTINS_MODULE: &str = "builtins"; -/// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -pub(crate) fn closure<'py>( - starts: Vec>, - edges: &Bound<'py, PyAny>, - (include_starts, max_depth): (bool, Option), -) -> PyResult> { - let py: Python<'py> = edges.py(); - let mut visited: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); - if include_starts { - for start in &starts { - visited.0.add(start)?; - } - } - let none: Bound<'py, PyTuple> = PyTuple::empty(py); - breadth_first_reach( - starts, - max_depth.map(|maximum| u64::try_from(maximum).unwrap_or(0)), - &mut visited, - |current| neighbours(edges, current, &none), - )?; - Ok(visited.0) -} - /// The keys `edges.get(current, ())` lists. -fn neighbours<'py>( +pub(crate) fn neighbours<'py>( edges: &Bound<'py, PyAny>, current: &Bound<'py, PyAny>, none: &Bound<'py, PyTuple>, @@ -46,24 +21,10 @@ fn neighbours<'py>( mapping_get_or(edges, current, none)?.try_iter()?.collect() } -/// Keys within `max_depth` edges of any start, plus the starts when requested. -#[pyfunction] -#[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] -pub(crate) fn transitive_closure_many<'py>( - starts: &Bound<'py, PyAny>, - edges: &Bound<'py, PyAny>, - include_starts: bool, - max_depth: Option, -) -> PyResult> { - let starts: Vec> = starts.try_iter()?.collect::>()?; - let visited: Bound<'py, PySet> = closure(starts, edges, (include_starts, max_depth))?; - PyFrozenSet::new(edges.py(), visited.iter()) -} - /// All graph keys reachable from a starting key. #[pyfunction] #[pyo3(signature = (*, start, edges, max_depth = None))] -fn transitive_closure<'py>( +pub(crate) fn transitive_closure<'py>( start: Bound<'py, PyAny>, edges: &Bound<'py, PyAny>, max_depth: Option, @@ -72,35 +33,10 @@ fn transitive_closure<'py>( PyFrozenSet::new(edges.py(), visited.iter()) } -/// Reversed edges: every source key first, then each new target in first-seen order. -pub(crate) fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = edges.py(); - let inverted: Bound<'py, PyDict> = PyDict::new(py); - for key in edges.try_iter()? { - inverted.set_item(key?, PyList::empty(py))?; - } - for item in mapping_items(edges)?.try_iter()? { - let (source, targets): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - for target in targets.try_iter()? { - let target: Bound<'py, PyAny> = target?; - let sources: Bound<'py, PyList> = match inverted.get_item(&target)? { - Some(sources) => sources.downcast_into()?, - None => { - let sources: Bound<'py, PyList> = PyList::empty(py); - inverted.set_item(&target, &sources)?; - sources - } - }; - sources.append(&source)?; - } - } - Ok(inverted) -} - /// Reversed directed graph edges, each source list sorted by `sort_key` when given. #[pyfunction] #[pyo3(signature = (*, edges, sort_key = None))] -fn invert_edges<'py>( +pub(crate) fn invert_edges<'py>( edges: &Bound<'py, PyAny>, sort_key: Option<&Bound<'py, PyAny>>, ) -> PyResult> { @@ -128,7 +64,7 @@ fn invert_edges<'py>( /// Keys on directed paths between two endpoints, or `None` when `end` is not downstream. #[pyfunction] #[pyo3(signature = (*, start, end, downstream))] -fn path_nodes<'py>( +pub(crate) fn path_nodes<'py>( start: Bound<'py, PyAny>, end: Bound<'py, PyAny>, downstream: &Bound<'py, PyAny>, @@ -151,11 +87,3 @@ fn path_nodes<'py>( } Ok(Some(PyFrozenSet::new(py, path.0.iter())?)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(transitive_closure, module)?)?; - module.add_function(wrap_pyfunction!(transitive_closure_many, module)?)?; - module.add_function(wrap_pyfunction!(invert_edges, module)?)?; - module.add_function(wrap_pyfunction!(path_nodes, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs index 1b2c776580..11709d459f 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs @@ -1,18 +1,12 @@ //! Ordering edges attached audits and model hooks add, and the keys hooks and SQL refs name. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; /// The model config keys whose entries are hooks, in lifecycle order. -const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +pub(crate) const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; -fn keyword_call<'py>( +pub(crate) fn keyword_call<'py>( callable: &Bound<'py, PyAny>, pairs: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -22,61 +16,3 @@ fn keyword_call<'py>( } callable.call((), Some(&keywords)) } - -/// The Python hooks attached to one model, in lifecycle order without repeats. -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn model_python_hook_names<'py>( - py: Python<'py>, - model: &Bound<'py, PyAny>, -) -> PyResult> { - let entry_type: Bound<'_, PyAny> = record_class(py, RecordId::PythonHookEntry)?; - let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; - let mut names: Vec = Vec::new(); - for hook_key in MODEL_HOOK_KEYS { - let entries: Bound<'_, PyAny> = mapping_get(&values, hook_key)?; - if entries.is_instance(&entry_type)? { - names.push(native_field(&entries, "name")?.extract()?); - } else if entries.is_instance_of::() || entries.is_instance_of::() { - for entry in entries.try_iter()? { - let entry: Bound<'_, PyAny> = entry?; - if entry.is_instance(&entry_type)? { - names.push(native_field(&entry, "name")?.extract()?); - } - } - } - } - let mut unique: Vec = Vec::with_capacity(names.len()); - for name in names { - if !unique.contains(&name) { - unique.push(name); - } - } - PyTuple::new(py, unique) -} - -/// The compiled graph key a typed model, source or seed reference names. -#[pyfunction] -pub(crate) fn sql_ref_key<'py>( - py: Python<'py>, - reference: &Bound<'py, PyAny>, -) -> PyResult> { - let resource_type: Bound<'py, PyAny> = enum_from_value( - py, - RecordId::CompiledResourceType, - native_field(reference, "kind")?.getattr("value")?, - )?; - keyword_call( - &record_class(py, RecordId::CompiledObjectKey)?, - &[ - ("resource_type", resource_type), - ("name", native_field(reference, "name")?), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_python_hook_names, module)?)?; - module.add_function(wrap_pyfunction!(sql_ref_key, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs deleted file mode 100644 index ce305b908c..0000000000 --- a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! `CompiledObjectKey`s for native `(resource type, name)` pairs, converted once per pair. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The native `(resource type, name)` pair of a `CompiledObjectKey`. -pub(crate) fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - Ok(( - native_field(key, "resource_type")?.str()?.extract()?, - native_field(key, "name")?.extract()?, - )) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs index 34146c3c53..dc46b80f5a 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs @@ -1,3 +1,2 @@ pub(crate) mod algorithms; pub(crate) mod execution_gates; -pub(crate) mod keys; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs index d67a57bfa9..0bf317f686 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs @@ -1,17 +1,9 @@ //! The compiled project graph and selector resolution, held natively for Python's graph users. -use crate::graph::models::CompiledProjectGraph; -use crate::graph::models::NativeProjectGraph; -use crate::graph::models::ResourceRow; use sqlbuild_python_base::positional_record; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; -use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::models::GraphResource; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; type Edges = Vec<(ObjectKey, Vec)>; positional_record! { @@ -46,31 +38,3 @@ pub(crate) fn outcome(result: Result, SelectorError>) -> KeysOutc fn failure(error: SelectorError) -> FailureRow { (error.code, error.message, error.help) } - -/// The project graph indexes of these resources, in project order. -pub(crate) fn project_graph(resources: Vec) -> PyResult { - compiler_guard(|| { - let resources: Vec = resources - .into_iter() - .map( - |ResourceRow { - key, - deps, - tags, - folder, - }| GraphResource { - key, - deps, - tags, - folder, - }, - ) - .collect(); - Ok(NativeProjectGraph::new(build_project_graph(&resources))) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs index 0eb817a63d..ef98ca9639 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs @@ -1,40 +1,29 @@ //! The native project graph's indexes as the Python dicts graph consumers read, and back. +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::main::project::edge_dict::edge_dict; +use crate::graph::main::project::graph_from_view_mappings::graph_from_view_mappings; use crate::graph::models::GraphViewMappings; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::{GraphIndexes, ProjectGraph}; -use crate::graph::_helpers::nodes::keys::native_key; use crate::graph::models::KeyCache; use crate::graph::models::NativeProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; /// The native graph's pairs for `keys`, in order. #[pyfunction] -fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { keys.try_iter()?.map(|key| native_key(&key?)).collect() } -/// `CompiledObjectKey`s for native graph key pairs. -#[pyfunction] -pub(crate) fn compiled_graph_keys<'py>( - py: Python<'py>, - keys: Vec, -) -> PyResult> { - let cache: KeyCache<'py> = KeyCache::new(py)?; - PyFrozenSet::new(py, cache.keys(&keys)?) -} - /// Selector name to key, in project order, without the other graph views. #[pyfunction] pub(crate) fn lineage_graph_names<'py>( @@ -49,20 +38,11 @@ pub(crate) fn lineage_graph_names<'py>( Ok(names) } -pub(crate) fn edge_dict<'py>( - cache: &KeyCache<'py>, - edges: &[(ObjectKey, Vec)], -) -> PyResult> { - let py: Python<'py> = cache.py(); - let dict: Bound<'py, PyDict> = PyDict::new(py); - for (key, deps) in edges { - dict.set_item(cache.key(key)?, PyTuple::new(py, cache.keys(deps)?)?)?; - } - Ok(dict) -} - /// The native graph's indexes as `LineageGraphViews` dicts, in project order. -fn graph_views<'py>(py: Python<'py>, graph: &ProjectGraph) -> PyResult> { +pub(crate) fn graph_views<'py>( + py: Python<'py>, + graph: &ProjectGraph, +) -> PyResult> { let cache: KeyCache<'py> = KeyCache::new(py)?; let tags: Bound<'py, PyDict> = PyDict::new(py); for (tag, keys) in &graph.tags { @@ -94,18 +74,9 @@ pub(crate) fn lineage_graph_views<'py>( graph_views(py, graph.graph()) } -/// The project's lineage edges and selector indexes, from its native graph. -#[pyfunction] -pub(crate) fn project_lineage_views<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let graph: PyRef<'py, NativeProjectGraph> = - native_field(project, "lineage_graph")?.extract()?; - graph_views(py, graph.graph()) -} - -fn native_edges(mapping: &Bound<'_, PyAny>) -> PyResult)>> { +pub(crate) fn native_edges( + mapping: &Bound<'_, PyAny>, +) -> PyResult)>> { let mut edges: Vec<(ObjectKey, Vec)> = Vec::new(); for item in mapping_items(mapping)?.try_iter()? { let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; @@ -114,14 +85,16 @@ fn native_edges(mapping: &Bound<'_, PyAny>) -> PyResult PyResult { +pub(crate) fn graph_from_dicts(indexes: GraphIndexes) -> PyResult { compiler_guard(|| Ok(NativeProjectGraph::new(graph_from_indexes(indexes)))) } /// A native graph over these indexes, keeping their order. #[pyfunction] #[pyo3(signature = (**request))] -fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn native_graph_from_views( + request: Option<&Bound<'_, PyDict>>, +) -> PyResult { let Some(request) = request else { return Err(pyo3::exceptions::PyTypeError::new_err( "native_graph_from_views() missing keyword arguments", @@ -142,58 +115,3 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult, -) -> PyResult { - let mut names: Vec<(String, ObjectKey)> = Vec::new(); - for item in mapping_items(mappings.all_keys)?.try_iter()? { - let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; - names.push((name, native_key(&key)?)); - } - let mut tags: Vec<(String, Vec)> = Vec::new(); - for item in mapping_items(mappings.tag_index)?.try_iter()? { - let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; - tags.push((tag, native_graph_keys(&keys)?)); - } - let mut paths: Vec<(ObjectKey, String)> = Vec::new(); - for item in mapping_items(mappings.path_index)?.try_iter()? { - let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; - paths.push((native_key(&key)?, folder)); - } - graph_from_dicts(GraphIndexes { - names, - upstream: native_edges(mappings.upstream)?, - downstream: native_edges(mappings.downstream)?, - tags, - paths, - }) -} - -/// A native graph holding these edges in their order, with no selector indexes. -#[pyfunction] -#[pyo3(signature = (*, upstream, downstream))] -pub(crate) fn native_graph_from_edges( - upstream: &Bound<'_, PyAny>, - downstream: &Bound<'_, PyAny>, -) -> PyResult { - graph_from_dicts(GraphIndexes { - names: Vec::new(), - upstream: native_edges(upstream)?, - downstream: native_edges(downstream)?, - tags: Vec::new(), - paths: Vec::new(), - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(native_graph_keys, module)?)?; - module.add_function(wrap_pyfunction!(compiled_graph_keys, module)?)?; - module.add_function(wrap_pyfunction!(lineage_graph_names, module)?)?; - module.add_function(wrap_pyfunction!(lineage_graph_views, module)?)?; - module.add_function(wrap_pyfunction!(project_lineage_views, module)?)?; - module.add_function(wrap_pyfunction!(native_graph_from_views, module)?)?; - module.add_function(wrap_pyfunction!(native_graph_from_edges, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs index 03bc0a19c2..b9c88351a2 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs @@ -1,73 +1,12 @@ //! Lifecycle regions of a run's selected Python nodes, classified natively. -use std::collections::{BTreeSet, HashMap}; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; -use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; -use sqlbuild_python_base::records::main::construct::new_record::new_record; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::python_plan::models::LifecycleNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; const EXTERNAL_CONNECTION_MODE: &str = "external"; -/// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -#[pyfunction] -#[pyo3(signature = (*, selection, python_graph))] -pub(crate) fn build_python_sql_run_lifecycle<'py>( - selection: &Bound<'py, PyAny>, - python_graph: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = selection.py(); - let selected: BTreeSet = native_field(selection, "python_node_names")? - .try_iter()? - .map(|name| name?.extract()) - .collect::>()?; - let mut nodes: HashMap = HashMap::new(); - for item in native_field(python_graph, "nodes_by_name")? - .call_method0("items")? - .try_iter()? - { - let (name, node): (String, Bound<'py, PyAny>) = item?.extract()?; - nodes.insert(name, lifecycle_node(&node)?); - } - let mut upstream: HashMap> = HashMap::new(); - for item in native_field(python_graph, "upstream_deps")? - .call_method0("items")? - .try_iter()? - { - let (name, deps): (String, Bound<'py, PyAny>) = item?.extract()?; - let deps: Vec = deps - .try_iter()? - .map(|dep| dep?.extract()) - .collect::>()?; - upstream.insert(name, deps); - } - let lifecycle: RunLifecycle = py.detach(|| run_lifecycle(&selected, &nodes, &upstream)); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item( - "ingress_python_node_names", - PyFrozenSet::new(py, lifecycle.ingress_names)?, - )?; - arguments.set_item( - "ingress_loader_names", - PyFrozenSet::new(py, lifecycle.ingress_loader_names)?, - )?; - arguments.set_item( - "read_side_python_node_names", - PyFrozenSet::new(py, lifecycle.read_side_names)?, - )?; - new_record( - py, - RecordId::PythonSqlRunLifecyclePlan, - (), - Some(&arguments), - ) -} - -fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { let loader: Bound<'_, PyAny> = native_field(node, "loader")?; let external_loader: bool = !loader.is_none() && native_field(&loader, "connection_mode")?.extract::()? @@ -77,7 +16,3 @@ fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { external_loader, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index 7eb1c11e83..3a20a71b6d 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -1,20 +1,16 @@ //! The executable Python-node graph built from discovered project inputs. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use crate::graph::_helpers::project::views::compiled_graph_keys; -use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::python_files::node_identity::node_identity; @@ -23,7 +19,7 @@ use sqlbuild_python_front::discovery::main::python_files::provider_usages::provi const NAME_KEY: &str = "name"; /// One discovered function collection, its node kind and whether it carries documentation. -const KINDS: [(&str, &str, bool); 4] = [ +pub(crate) const KINDS: [(&str, &str, bool); 4] = [ ("loader_functions", "LOADER", false), ("task_functions", "TASK", true), ("asset_functions", "ASSET", true), @@ -31,11 +27,11 @@ const KINDS: [(&str, &str, bool); 4] = [ ]; /// Python classes and shims every node is built with. -struct Vocabulary<'py> { - node: Bound<'py, PyAny>, - kinds: Bound<'py, PyAny>, - sql_ref: Bound<'py, PyAny>, - providers: Bound<'py, PyDict>, +pub(crate) struct Vocabulary<'py> { + pub(crate) node: Bound<'py, PyAny>, + pub(crate) kinds: Bound<'py, PyAny>, + pub(crate) sql_ref: Bound<'py, PyAny>, + pub(crate) providers: Bound<'py, PyDict>, } fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { @@ -91,7 +87,7 @@ fn kind_metadata<'py>( } /// One `DiscoveredPythonNode` with its identity and provider usages read by the user shims. -fn python_node<'py>( +pub(crate) fn python_node<'py>( vocabulary: &Vocabulary<'py>, (kind, documented): (&str, bool), discovered: &Bound<'py, PyAny>, @@ -135,7 +131,7 @@ fn python_node<'py>( } /// Each node's upstream Python-node names, resolving decorated dependencies through the shim. -fn upstream_names( +pub(crate) fn upstream_names( py: Python<'_>, nodes: &[Bound<'_, PyAny>], sql_ref: &Bound<'_, PyAny>, @@ -171,7 +167,10 @@ fn upstream_names( .collect() } -fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { +pub(crate) fn inventory_node( + node: &Bound<'_, PyAny>, + upstream: Vec, +) -> PyResult { Ok(InventoryNode { name: native_field(node, "name")?.extract()?, tags: native_field(node, "tags")?.extract()?, @@ -182,42 +181,9 @@ fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult( - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(discovered_inputs, "providers")?.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - providers.set_item(native_field(&provider, "name")?, provider)?; - } - let vocabulary: Vocabulary<'py> = Vocabulary { - node: record_class(py, RecordId::DiscoveredPythonNode)?, - kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, - sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, - providers, - }; - let mut nodes: Vec> = Vec::new(); - within_identity_pass(py, || { - for (collection, kind, documented) in KINDS { - for discovered in native_field(discovered_inputs, collection)?.try_iter()? { - nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); - } - } - Ok(()) - })?; - let inventory: Vec = nodes - .iter() - .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) - .map(|(node, upstream)| inventory_node(node, upstream)) - .collect::>()?; - graph_record(py, &nodes, graph_indexes(&inventory)) -} - type NameRows = Vec<(String, Vec)>; -fn graph_record<'py>( +pub(crate) fn graph_record<'py>( py: Python<'py>, nodes: &[Bound<'py, PyAny>], indexes: InventoryIndexes, @@ -290,40 +256,12 @@ fn graph_record<'py>( ) } -/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. -pub(crate) fn run_selection_from_inputs<'py>( - (select, exclude): (Vec, Vec), - project_graph: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_graph.py(); - let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; - let (sql_keys, python_node_names) = - resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; - new_record_with( - py, - RecordId::PythonSqlRunSelection, - &[ - ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), - ( - "python_node_names", - PyFrozenSet::new(py, python_node_names)?.into_any(), - ), - ], - ) -} - /// Build the internal executable Python-node graph from discovered project inputs. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_node_graph<'py>( +pub(crate) fn build_python_node_graph<'py>( options: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("build_python_node_graph", options)?; python_node_graph(&request.get("discovered_inputs")?) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index 4d4c79fced..6d7c157329 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -1,20 +1,17 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use crate::graph::main::python_nodes::python_node_names::python_node_names; use sqlbuild_python_base::positional_record; use std::borrow::Cow; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; -use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; -use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, @@ -22,8 +19,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::compiled_project::main::needed_jobs::needed_jobs; -use crate::graph::models::CompiledProjectGraph; -use crate::graph::models::NativeProjectGraph; +use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -34,7 +30,7 @@ type NameRows = Vec<(String, Vec)>; positional_record! { extract; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. - struct InventoryRow { + pub(crate) struct InventoryRow { name: String, tags: Vec, relative_path: String, @@ -45,7 +41,7 @@ positional_record! { positional_record! { into; /// `(edges, upstream, downstream, tags, paths)` in Python's dict order. - struct IndexesRow { + pub(crate) struct IndexesRow { edges: Vec<(String, String)>, upstream: NameRows, downstream: NameRows, @@ -55,7 +51,7 @@ positional_record! { } /// The `PlannerInputError` Python raises for a selector failure. -fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { +pub(crate) fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { let raised = (|| -> PyResult> { let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("code", error.code)?; @@ -136,23 +132,10 @@ pub(crate) fn project_sources( .collect() } -/// The Python node names `select` minus `exclude` resolve to; every node when unselected. -pub(crate) fn python_node_names<'py>( - graph: &Bound<'py, PyAny>, - (select, exclude): (Vec, Vec), -) -> PyResult> { - let py: Python<'py> = graph.py(); - let native: PythonSelectionGraph = selection_graph(graph)?; - let names: Vec = - compiler_guard(|| Ok(resolve_python_node_selectors(&native, &select, &exclude)))? - .map_err(|error| planner_error(py, error))?; - PyFrozenSet::new(py, names) -} - /// The Python node names `select` minus `exclude` resolve to; every node when unselected. #[pyfunction] #[pyo3(signature = (*, graph, select, exclude))] -fn resolve_python_node_selection<'py>( +pub(crate) fn resolve_python_node_selection<'py>( graph: &Bound<'py, PyAny>, select: Vec, exclude: Vec, @@ -160,21 +143,6 @@ fn resolve_python_node_selection<'py>( python_node_names(graph, (select, exclude)) } -/// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. -#[pyfunction] -#[pyo3(signature = (*, graph, exclude))] -pub(crate) fn resolve_python_check_excludes<'py>( - graph: &Bound<'py, PyAny>, - exclude: Vec, -) -> PyResult> { - let py: Python<'py> = graph.py(); - let native: PythonSelectionGraph = selection_graph(graph)?; - let names: Vec = - compiler_guard(|| Ok(resolve_python_node_excludes(&native, &exclude)))? - .map_err(|error| planner_error(py, error))?; - PyFrozenSet::new(py, names) -} - /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] @@ -220,7 +188,7 @@ pub(crate) fn resolve_python_sql_run_selection( /// The dependency edges, adjacency, tag and path indexes of discovered Python nodes. #[pyfunction] -fn python_node_graph_indexes(nodes: Vec) -> PyResult { +pub(crate) fn python_node_graph_indexes(nodes: Vec) -> PyResult { compiler_guard(|| { let nodes: Vec = nodes .into_iter() @@ -254,10 +222,3 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { }) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_python_node_selection, module)?)?; - module.add_function(wrap_pyfunction!(resolve_python_check_excludes, module)?)?; - module.add_function(wrap_pyfunction!(resolve_python_sql_run_selection, module)?)?; - module.add_function(wrap_pyfunction!(python_node_graph_indexes, module)?) -} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs index 7247706e2a..8c353124fd 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs @@ -1,17 +1,30 @@ //! Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -use crate::graph::_helpers::nodes::algorithms; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PySet; +use crate::graph::_helpers::nodes::algorithms::neighbours; +use crate::graph::models::PyNodeSet; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyTuple}; +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; /// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -#[inline] pub fn closure<'py>( starts: Vec>, edges: &Bound<'py, PyAny>, (include_starts, max_depth): (bool, Option), ) -> PyResult> { - algorithms::closure(starts, edges, (include_starts, max_depth)) + let py: Python<'py> = edges.py(); + let mut visited: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); + if include_starts { + for start in &starts { + visited.0.add(start)?; + } + } + let none: Bound<'py, PyTuple> = PyTuple::empty(py); + breadth_first_reach( + starts, + max_depth.map(|maximum| u64::try_from(maximum).unwrap_or(0)), + &mut visited, + |current| neighbours(edges, current, &none), + )?; + Ok(visited.0) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs index a735594aeb..831736a48c 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs @@ -1,13 +1,30 @@ //! Reversed edges: every source key first, then each new target in first-seen order. -use crate::graph::_helpers::nodes::algorithms; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// Reversed edges: every source key first, then each new target in first-seen order. -#[inline] pub fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { - algorithms::inverted(edges) + let py: Python<'py> = edges.py(); + let inverted: Bound<'py, PyDict> = PyDict::new(py); + for key in edges.try_iter()? { + inverted.set_item(key?, PyList::empty(py))?; + } + for item in mapping_items(edges)?.try_iter()? { + let (source, targets): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + for target in targets.try_iter()? { + let target: Bound<'py, PyAny> = target?; + let sources: Bound<'py, PyList> = match inverted.get_item(&target)? { + Some(sources) => sources.downcast_into()?, + None => { + let sources: Bound<'py, PyList> = PyList::empty(py); + inverted.set_item(&target, &sources)?; + sources + } + }; + sources.append(&source)?; + } + } + Ok(inverted) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs index fd5028aa03..0b665c5f0e 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs @@ -1,17 +1,42 @@ //! The Python hooks attached to one model, in lifecycle order without repeats. -use crate::graph::_helpers::nodes::execution_gates; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyTuple; +use crate::graph::_helpers::nodes::execution_gates::MODEL_HOOK_KEYS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; /// The Python hooks attached to one model, in lifecycle order without repeats. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model))] pub fn model_python_hook_names<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { - execution_gates::model_python_hook_names(py, model) + let entry_type: Bound<'_, PyAny> = record_class(py, RecordId::PythonHookEntry)?; + let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; + let mut names: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + let entries: Bound<'_, PyAny> = mapping_get(&values, hook_key)?; + if entries.is_instance(&entry_type)? { + names.push(native_field(&entries, "name")?.extract()?); + } else if entries.is_instance_of::() || entries.is_instance_of::() { + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if entry.is_instance(&entry_type)? { + names.push(native_field(&entry, "name")?.extract()?); + } + } + } + } + let mut unique: Vec = Vec::with_capacity(names.len()); + for name in names { + if !unique.contains(&name) { + unique.push(name); + } + } + PyTuple::new(py, unique) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs index 9e3e75a810..d7d6e4ea3f 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs @@ -1,12 +1,12 @@ //! The native `(resource type, name)` pair of a `CompiledObjectKey`. -use crate::graph::_helpers::nodes::keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native `(resource type, name)` pair of a `CompiledObjectKey`. -#[inline] pub fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - keys::native_key(key) + Ok(( + native_field(key, "resource_type")?.str()?.extract()?, + native_field(key, "name")?.extract()?, + )) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs index c6dd33da76..dca2b53cc1 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs @@ -1,12 +1,17 @@ //! Register the algorithms functions and classes. -use crate::graph::_helpers::nodes::algorithms; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::nodes::algorithms::invert_edges; +use crate::graph::_helpers::nodes::algorithms::path_nodes; +use crate::graph::_helpers::nodes::algorithms::transitive_closure; +use crate::graph::main::nodes::transitive_closure_many::transitive_closure_many; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the algorithms functions and classes. -#[inline] pub fn register_algorithms(module: &Bound<'_, PyModule>) -> PyResult<()> { - algorithms::register(module) + module.add_function(wrap_pyfunction!(transitive_closure, module)?)?; + module.add_function(wrap_pyfunction!(transitive_closure_many, module)?)?; + module.add_function(wrap_pyfunction!(invert_edges, module)?)?; + module.add_function(wrap_pyfunction!(path_nodes, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs index 560abdebc1..424f5bdc42 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs @@ -1,12 +1,13 @@ //! Register the execution gates functions and classes. -use crate::graph::_helpers::nodes::execution_gates; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; +use crate::graph::main::nodes::sql_ref_key::sql_ref_key; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the execution gates functions and classes. -#[inline] pub fn register_execution_gates(module: &Bound<'_, PyModule>) -> PyResult<()> { - execution_gates::register(module) + module.add_function(wrap_pyfunction!(model_python_hook_names, module)?)?; + module.add_function(wrap_pyfunction!(sql_ref_key, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs index 63b9a30e04..28bb786901 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs @@ -1,16 +1,29 @@ //! The compiled graph key a typed model, source or seed reference names. -use crate::graph::_helpers::nodes::execution_gates; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::graph::_helpers::nodes::execution_gates::keyword_call; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The compiled graph key a typed model, source or seed reference names. -#[inline] +#[pyfunction] pub fn sql_ref_key<'py>( py: Python<'py>, reference: &Bound<'py, PyAny>, ) -> PyResult> { - execution_gates::sql_ref_key(py, reference) + let resource_type: Bound<'py, PyAny> = enum_from_value( + py, + RecordId::CompiledResourceType, + native_field(reference, "kind")?.getattr("value")?, + )?; + keyword_call( + &record_class(py, RecordId::CompiledObjectKey)?, + &[ + ("resource_type", resource_type), + ("name", native_field(reference, "name")?), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs index cb447c3b7b..6d35aa5f5b 100644 --- a/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs @@ -1,18 +1,20 @@ //! Keys within `max_depth` edges of any start, plus the starts when requested. -use crate::graph::_helpers::nodes::algorithms; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::graph::main::nodes::closure::closure; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PySet, PySetMethods}; /// Keys within `max_depth` edges of any start, plus the starts when requested. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] pub fn transitive_closure_many<'py>( starts: &Bound<'py, PyAny>, edges: &Bound<'py, PyAny>, include_starts: bool, max_depth: Option, ) -> PyResult> { - algorithms::transitive_closure_many(starts, edges, include_starts, max_depth) + let starts: Vec> = starts.try_iter()?.collect::>()?; + let visited: Bound<'py, PySet> = closure(starts, edges, (include_starts, max_depth))?; + PyFrozenSet::new(edges.py(), visited.iter()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs index a60832e5ac..1f9556250d 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs @@ -1,17 +1,17 @@ //! `CompiledObjectKey`s for native graph key pairs. -use crate::graph::_helpers::project::views; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::graph::models::KeyCache; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyFrozenSet; use sqlbuild_analysis::assembly::project::types::ObjectKey; /// `CompiledObjectKey`s for native graph key pairs. -#[inline] +#[pyfunction] pub fn compiled_graph_keys<'py>( py: Python<'py>, keys: Vec, ) -> PyResult> { - views::compiled_graph_keys(py, keys) + let cache: KeyCache<'py> = KeyCache::new(py)?; + PyFrozenSet::new(py, cache.keys(&keys)?) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs index e686143404..3bc1defbff 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs @@ -1,17 +1,19 @@ //! Edge dict. -use crate::graph::_helpers::project::views; use crate::graph::models::KeyCache; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; /// Edge dict. -#[inline] pub fn edge_dict<'py>( cache: &KeyCache<'py>, edges: &[(ObjectKey, Vec)], ) -> PyResult> { - views::edge_dict(cache, edges) + let py: Python<'py> = cache.py(); + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, deps) in edges { + dict.set_item(cache.key(key)?, PyTuple::new(py, cache.keys(deps)?)?)?; + } + Ok(dict) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs index 58e1f1ba79..6ce8b63b45 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs @@ -1,14 +1,40 @@ //! A native graph over Python's index mappings, keeping their order. -use crate::graph::_helpers::project::views; +use crate::graph::_helpers::project::views::graph_from_dicts; +use crate::graph::_helpers::project::views::native_edges; +use crate::graph::_helpers::project::views::native_graph_keys; +use crate::graph::main::nodes::native_key::native_key; use crate::graph::models::GraphViewMappings; use crate::graph::models::NativeProjectGraph; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::models::GraphIndexes; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// A native graph over Python's index mappings, keeping their order. -#[inline] pub fn graph_from_view_mappings( mappings: GraphViewMappings<'_, '_>, ) -> PyResult { - views::graph_from_view_mappings(mappings) + let mut names: Vec<(String, ObjectKey)> = Vec::new(); + for item in mapping_items(mappings.all_keys)?.try_iter()? { + let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; + names.push((name, native_key(&key)?)); + } + let mut tags: Vec<(String, Vec)> = Vec::new(); + for item in mapping_items(mappings.tag_index)?.try_iter()? { + let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; + tags.push((tag, native_graph_keys(&keys)?)); + } + let mut paths: Vec<(ObjectKey, String)> = Vec::new(); + for item in mapping_items(mappings.path_index)?.try_iter()? { + let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; + paths.push((native_key(&key)?, folder)); + } + graph_from_dicts(GraphIndexes { + names, + upstream: native_edges(mappings.upstream)?, + downstream: native_edges(mappings.downstream)?, + tags, + paths, + }) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs index ca951027b7..40fa31ee1f 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs @@ -1,16 +1,24 @@ //! A native graph holding these edges in their order, with no selector indexes. -use crate::graph::_helpers::project::views; +use crate::graph::_helpers::project::views::graph_from_dicts; +use crate::graph::_helpers::project::views::native_edges; use crate::graph::models::NativeProjectGraph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::graph::models::GraphIndexes; /// A native graph holding these edges in their order, with no selector indexes. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, upstream, downstream))] pub fn native_graph_from_edges( upstream: &Bound<'_, PyAny>, downstream: &Bound<'_, PyAny>, ) -> PyResult { - views::native_graph_from_edges(upstream, downstream) + graph_from_dicts(GraphIndexes { + names: Vec::new(), + upstream: native_edges(upstream)?, + downstream: native_edges(downstream)?, + tags: Vec::new(), + paths: Vec::new(), + }) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs index 1afb981d97..eded391051 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs @@ -1,12 +1,31 @@ //! The project graph indexes of these resources, in project order. -use crate::graph::_helpers::project::project_graph; use crate::graph::models::NativeProjectGraph; use crate::graph::models::ResourceRow; use pyo3::prelude::PyResult; +use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; +use sqlbuild_analysis::graph::models::GraphResource; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The project graph indexes of these resources, in project order. -#[inline] pub fn project_graph(resources: Vec) -> PyResult { - project_graph::project_graph(resources) + compiler_guard(|| { + let resources: Vec = resources + .into_iter() + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) + .collect(); + Ok(NativeProjectGraph::new(build_project_graph(&resources))) + }) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs index f61255e529..6064feaa28 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs @@ -1,16 +1,18 @@ //! The project's lineage edges and selector indexes, from its native graph. -use crate::graph::_helpers::project::views; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::graph::_helpers::project::views::graph_views; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project's lineage edges and selector indexes, from its native graph. -#[inline] +#[pyfunction] pub fn project_lineage_views<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, ) -> PyResult> { - views::project_lineage_views(py, project) + let graph: PyRef<'py, NativeProjectGraph> = + native_field(project, "lineage_graph")?.extract()?; + graph_views(py, graph.graph()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs index 73da7dd6e7..6bdd076d1c 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs @@ -1,12 +1,11 @@ //! Register the project graph functions and classes. -use crate::graph::_helpers::project::project_graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::models::CompiledProjectGraph; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the project graph functions and classes. -#[inline] pub fn register_project_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_graph::register(module) + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs index b004bc34e4..c89a849f29 100644 --- a/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs @@ -1,12 +1,23 @@ //! Register the views functions and classes. -use crate::graph::_helpers::project::views; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::project::views::lineage_graph_names; +use crate::graph::_helpers::project::views::lineage_graph_views; +use crate::graph::_helpers::project::views::native_graph_from_views; +use crate::graph::_helpers::project::views::native_graph_keys; +use crate::graph::main::project::compiled_graph_keys::compiled_graph_keys; +use crate::graph::main::project::native_graph_from_edges::native_graph_from_edges; +use crate::graph::main::project::project_lineage_views::project_lineage_views; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the views functions and classes. -#[inline] pub fn register_views(module: &Bound<'_, PyModule>) -> PyResult<()> { - views::register(module) + module.add_function(wrap_pyfunction!(native_graph_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_graph_keys, module)?)?; + module.add_function(wrap_pyfunction!(lineage_graph_names, module)?)?; + module.add_function(wrap_pyfunction!(lineage_graph_views, module)?)?; + module.add_function(wrap_pyfunction!(project_lineage_views, module)?)?; + module.add_function(wrap_pyfunction!(native_graph_from_views, module)?)?; + module.add_function(wrap_pyfunction!(native_graph_from_edges, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs index 6b9b06f140..3c85e9cb2d 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs @@ -1,15 +1,66 @@ //! `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -use crate::graph::_helpers::python_nodes::python_lifecycle; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_lifecycle::lifecycle_node; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyFrozenSet}; +use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; +use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use std::collections::{BTreeSet, HashMap}; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, selection, python_graph))] pub fn build_python_sql_run_lifecycle<'py>( selection: &Bound<'py, PyAny>, python_graph: &Bound<'py, PyAny>, ) -> PyResult> { - python_lifecycle::build_python_sql_run_lifecycle(selection, python_graph) + let py: Python<'py> = selection.py(); + let selected: BTreeSet = native_field(selection, "python_node_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let mut nodes: HashMap = HashMap::new(); + for item in native_field(python_graph, "nodes_by_name")? + .call_method0("items")? + .try_iter()? + { + let (name, node): (String, Bound<'py, PyAny>) = item?.extract()?; + nodes.insert(name, lifecycle_node(&node)?); + } + let mut upstream: HashMap> = HashMap::new(); + for item in native_field(python_graph, "upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (name, deps): (String, Bound<'py, PyAny>) = item?.extract()?; + let deps: Vec = deps + .try_iter()? + .map(|dep| dep?.extract()) + .collect::>()?; + upstream.insert(name, deps); + } + let lifecycle: RunLifecycle = py.detach(|| run_lifecycle(&selected, &nodes, &upstream)); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item( + "ingress_python_node_names", + PyFrozenSet::new(py, lifecycle.ingress_names)?, + )?; + arguments.set_item( + "ingress_loader_names", + PyFrozenSet::new(py, lifecycle.ingress_loader_names)?, + )?; + arguments.set_item( + "read_side_python_node_names", + PyFrozenSet::new(py, lifecycle.read_side_names)?, + )?; + new_record( + py, + RecordId::PythonSqlRunLifecyclePlan, + (), + Some(&arguments), + ) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs index a24ec554af..53c6bd7bfa 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs @@ -1,14 +1,51 @@ //! The internal executable Python-node graph of discovered project inputs. -use crate::graph::_helpers::python_nodes::python_node_graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_node_graph::KINDS; +use crate::graph::_helpers::python_nodes::python_node_graph::Vocabulary; +use crate::graph::_helpers::python_nodes::python_node_graph::graph_record; +use crate::graph::_helpers::python_nodes::python_node_graph::inventory_node; +use crate::graph::_helpers::python_nodes::python_node_graph::python_node; +use crate::graph::_helpers::python_nodes::python_node_graph::upstream_names; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::models::InventoryNode; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The internal executable Python-node graph of discovered project inputs. -#[inline] pub fn python_node_graph<'py>( discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - python_node_graph::python_node_graph(discovered_inputs) + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in native_field(discovered_inputs, "providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(native_field(&provider, "name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: record_class(py, RecordId::DiscoveredPythonNode)?, + kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, + sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, + providers, + }; + let mut nodes: Vec> = Vec::new(); + within_identity_pass(py, || { + for (collection, kind, documented) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + Ok(()) + })?; + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + graph_record(py, &nodes, graph_indexes(&inventory)) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs index 6ebd5b8578..d729b6c6dd 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs @@ -1,16 +1,22 @@ //! The Python node names `select` minus `exclude` resolve to; every node when unselected. -use crate::graph::_helpers::python_nodes::python_selection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The Python node names `select` minus `exclude` resolve to; every node when unselected. -#[inline] pub fn python_node_names<'py>( graph: &Bound<'py, PyAny>, (select, exclude): (Vec, Vec), ) -> PyResult> { - python_selection::python_node_names(graph, (select, exclude)) + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_selectors(&native, &select, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs index b634ee17cb..2d5251fcc9 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs @@ -1,12 +1,10 @@ //! Register the python lifecycle functions and classes. -use crate::graph::_helpers::python_nodes::python_lifecycle; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python lifecycle functions and classes. -#[inline] pub fn register_python_lifecycle(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_lifecycle::register(module) + module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs index 15e439c377..1827619dab 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs @@ -1,12 +1,11 @@ //! Register the python node graph functions and classes. -use crate::graph::_helpers::python_nodes::python_node_graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_node_graph::build_python_node_graph; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python node graph functions and classes. -#[inline] pub fn register_python_node_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_node_graph::register(module) + module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs index 94d80dec33..37fc6b5f03 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs @@ -1,12 +1,16 @@ //! Register the python selection functions and classes. -use crate::graph::_helpers::python_nodes::python_selection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_selection::python_node_graph_indexes; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_node_selection; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; +use crate::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python selection functions and classes. -#[inline] pub fn register_python_selection(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_selection::register(module) + module.add_function(wrap_pyfunction!(resolve_python_node_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_check_excludes, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_sql_run_selection, module)?)?; + module.add_function(wrap_pyfunction!(python_node_graph_indexes, module)?) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs index 5956df1a1e..6b9ba1bc31 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs @@ -1,16 +1,25 @@ //! The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. -use crate::graph::_helpers::python_nodes::python_selection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, graph, exclude))] pub fn resolve_python_check_excludes<'py>( graph: &Bound<'py, PyAny>, exclude: Vec, ) -> PyResult> { - python_selection::resolve_python_check_excludes(graph, exclude) + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_excludes(&native, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs index dfb821cfd5..6006812be3 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs @@ -1,20 +1,32 @@ //! `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. -use crate::graph::_helpers::python_nodes::python_node_graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; +use crate::graph::main::project::compiled_graph_keys::compiled_graph_keys; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. -#[inline] pub fn run_selection_from_inputs<'py>( (select, exclude): (Vec, Vec), project_graph: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - python_node_graph::run_selection_from_inputs( - (select, exclude), - project_graph, - discovered_inputs, + let py: Python<'py> = project_graph.py(); + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; + let (sql_keys, python_node_names) = + resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; + new_record_with( + py, + RecordId::PythonSqlRunSelection, + &[ + ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), + ( + "python_node_names", + PyFrozenSet::new(py, python_node_names)?.into_any(), + ), + ], ) } diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index de033e9274..c55cc47781 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -5,11 +5,11 @@ use crate::graph::_helpers::project::project_graph::KeysOutcome; use crate::graph::_helpers::project::project_graph::UniqueIdMap; use crate::graph::_helpers::project::project_graph::dependency_edges; use crate::graph::_helpers::project::project_graph::outcome; -use crate::graph::_helpers::project::project_graph::project_graph; use crate::graph::_helpers::project::views::lineage_graph_views; -use crate::graph::_helpers::python_nodes::python_node_graph::python_node_graph; use crate::graph::_helpers::python_nodes::python_selection::project_sources; use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use crate::graph::main::project::project_graph::project_graph; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; diff --git a/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs b/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs index 6d640799ea..caf95780e4 100644 --- a/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs +++ b/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs @@ -1,12 +1,10 @@ //! Sequence behaviour of the natively hosted `CompactLineageFacts` record. -use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyIndexError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PySlice, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::positional_record; -use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -14,7 +12,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; /// `ColumnTransformKind` members by native transform code. -const TRANSFORM_KINDS: [&str; 6] = [ +pub(crate) const TRANSFORM_KINDS: [&str; 6] = [ "DIRECT", "CAST", "EXPRESSION", @@ -23,20 +21,10 @@ const TRANSFORM_KINDS: [&str; 6] = [ "CONSTANT", ]; /// `ColumnLineageConfidence` members by native confidence code. -const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; - -/// The `ColumnTransformKind` member of a native transform code. -pub(crate) fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { - coded_member(py, (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), code) -} - -/// The `ColumnLineageConfidence` member of a native confidence code. -pub(crate) fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { - coded_member(py, (RecordId::ColumnLineageConfidence, &CONFIDENCES), code) -} +pub(crate) const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; /// The enum member at `code` of `members`. -fn coded_member( +pub(crate) fn coded_member( py: Python<'_>, (class, members): (RecordId, &[&str]), code: usize, @@ -48,7 +36,7 @@ fn coded_member( } /// `CompactLineageFacts.resource_name(index)`: the model's name for a native relation index. -fn resource_name<'py>( +pub(crate) fn resource_name<'py>( facts: &Bound<'py, PyAny>, index: &Bound<'py, PyAny>, ) -> PyResult> { @@ -62,16 +50,6 @@ fn resource_name<'py>( native_field(facts, "string_pool")?.get_item(pooled) } -/// `CompactLineageFacts.resource_name(index)`. -pub(crate) fn facts_resource_name( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; - Ok(resource_name(instance, &index)?.unbind()) -} - positional_record! { extract; /// One compact lineage row: output name index, transform and confidence codes, sources. @@ -84,7 +62,7 @@ positional_record! { } /// One compact row projected to a `CompiledLineageColumnFact`. -fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = facts.py(); let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; let CompactColumnRow { @@ -128,81 +106,3 @@ fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let projected: Vec> = native_field(instance, "rows")? - .try_iter()? - .map(|row| project_row(instance, &row?)) - .collect::>()?; - Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) -} - -/// `len(facts)`. -pub(crate) fn facts_len( - instance: &Bound<'_, PyAny>, - _args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - native_field(instance, "rows")? - .len()? - .into_py_any(instance.py()) -} - -/// `facts[index]`: one fact, or a tuple of facts for a slice. -pub(crate) fn facts_getitem( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let index: Bound<'_, PyAny> = args.get_item(0)?; - let projected: Bound<'_, PyAny> = native_field(instance, "facts")?; - if index.is_instance_of::() { - return Ok(projected.get_item(index)?.unbind()); - } - let length: isize = isize::try_from(projected.len()?)?; - let requested: isize = index.extract()?; - let normalized: isize = if requested >= 0 { - requested - } else { - length + requested - }; - if normalized < 0 || normalized >= length { - return Err(PyIndexError::new_err(index.unbind())); - } - Ok(projected.get_item(normalized)?.unbind()) -} - -/// `iter(facts)`. -pub(crate) fn facts_iter( - instance: &Bound<'_, PyAny>, - _args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - Ok(native_field(instance, "facts")? - .try_iter()? - .into_any() - .unbind()) -} - -/// `facts == other`: item-wise equality with any sequence. -pub(crate) fn facts_eq( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let py: Python<'_> = instance.py(); - let other: Bound<'_, PyAny> = args.get_item(0)?; - let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; - if !other.is_instance(&sequence)? { - return Ok(py.NotImplemented()); - } - let mine: Bound<'_, PyAny> = native_field(instance, "facts")?; - let theirs: Bound<'_, PyTuple> = PyTuple::new( - py, - other - .try_iter()? - .collect::>>>()?, - )?; - mine.eq(theirs)?.into_py_any(py) -} diff --git a/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs index b9bd54bb61..3adb6c3356 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs @@ -1,13 +1,11 @@ //! The `ColumnLineageConfidence` member of a native confidence code. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::lineage::_helpers::lineage_facts::CONFIDENCES; +use crate::lineage::_helpers::lineage_facts::coded_member; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::types::RecordId; /// The `ColumnLineageConfidence` member of a native confidence code. -#[inline] pub fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { - lineage_facts::confidence_member(py, code) + coded_member(py, (RecordId::ColumnLineageConfidence, &CONFIDENCES), code) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs index e71aa93d42..8121e58eb4 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs @@ -1,19 +1,28 @@ //! `facts == other`: item-wise equality with any sequence. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `facts == other`: item-wise equality with any sequence. -#[inline] pub fn facts_eq( instance: &Bound<'_, PyAny>, args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - lineage_facts::facts_eq(instance, args, _kwargs) + let py: Python<'_> = instance.py(); + let other: Bound<'_, PyAny> = args.get_item(0)?; + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + if !other.is_instance(&sequence)? { + return Ok(py.NotImplemented()); + } + let mine: Bound<'_, PyAny> = native_field(instance, "facts")?; + let theirs: Bound<'_, PyTuple> = PyTuple::new( + py, + other + .try_iter()? + .collect::>>>()?, + )?; + mine.eq(theirs)?.into_py_any(py) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs index 43e211872f..9c214bbf3d 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs @@ -1,19 +1,30 @@ //! `facts[index]`: one fact, or a tuple of facts for a slice. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PySlice, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `facts[index]`: one fact, or a tuple of facts for a slice. -#[inline] pub fn facts_getitem( instance: &Bound<'_, PyAny>, args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - lineage_facts::facts_getitem(instance, args, _kwargs) + let index: Bound<'_, PyAny> = args.get_item(0)?; + let projected: Bound<'_, PyAny> = native_field(instance, "facts")?; + if index.is_instance_of::() { + return Ok(projected.get_item(index)?.unbind()); + } + let length: isize = isize::try_from(projected.len()?)?; + let requested: isize = index.extract()?; + let normalized: isize = if requested >= 0 { + requested + } else { + length + requested + }; + if normalized < 0 || normalized >= length { + return Err(PyIndexError::new_err(index.unbind())); + } + Ok(projected.get_item(normalized)?.unbind()) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs index a26ff38bf9..0c7a7a09cd 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs @@ -1,19 +1,17 @@ //! `iter(facts)`. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `iter(facts)`. -#[inline] pub fn facts_iter( instance: &Bound<'_, PyAny>, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - lineage_facts::facts_iter(instance, _args, _kwargs) + Ok(native_field(instance, "facts")? + .try_iter()? + .into_any() + .unbind()) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs index f099913346..507ce9ea26 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs @@ -1,19 +1,17 @@ //! `len(facts)`. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `len(facts)`. -#[inline] pub fn facts_len( instance: &Bound<'_, PyAny>, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - lineage_facts::facts_len(instance, _args, _kwargs) + native_field(instance, "rows")? + .len()? + .into_py_any(instance.py()) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs index 089bad1fd7..06306bbecc 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs @@ -1,13 +1,15 @@ //! `CompactLineageFacts.facts`: every row projected once, in order. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::lineage::_helpers::lineage_facts::project_row; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `CompactLineageFacts.facts`: every row projected once, in order. -#[inline] pub fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { - lineage_facts::facts_projected(instance) + let projected: Vec> = native_field(instance, "rows")? + .try_iter()? + .map(|row| project_row(instance, &row?)) + .collect::>()?; + Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs index 59dc28803e..a94ef72be9 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs @@ -1,19 +1,16 @@ //! `CompactLineageFacts.resource_name(index)`. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::lineage::_helpers::lineage_facts::resource_name; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::main::call_argument::call_argument; /// `CompactLineageFacts.resource_name(index)`. -#[inline] pub fn facts_resource_name( instance: &Bound<'_, PyAny>, args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - lineage_facts::facts_resource_name(instance, args, kwargs) + let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; + Ok(resource_name(instance, &index)?.unbind()) } diff --git a/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs index 4a9b2cc04f..0d4671f8df 100644 --- a/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs +++ b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs @@ -1,13 +1,11 @@ //! The `ColumnTransformKind` member of a native transform code. -use crate::lineage::_helpers::lineage_facts; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::lineage::_helpers::lineage_facts::TRANSFORM_KINDS; +use crate::lineage::_helpers::lineage_facts::coded_member; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::types::RecordId; /// The `ColumnTransformKind` member of a native transform code. -#[inline] pub fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { - lineage_facts::transform_kind_member(py, code) + coded_member(py, (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), code) } diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs index ca91f52089..1f001e8d52 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs @@ -1,39 +1,24 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use crate::native_host::models::CalendarRequest; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; +use std::collections::BTreeSet; -use chrono::{DateTime, TimeDelta, Timelike, Utc}; +use chrono::{DateTime, TimeDelta, Utc}; use chrono_tz::Tz; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; use serde_json::{Value, json}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; -use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_orchestration::declarations::models::Schedule; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; -use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::graph::models::ProjectSelection; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::constants::CALENDAR_WINDOW_PARAMETER; use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; -use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; -use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; -use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; -use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; -use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; -use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; -use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; -use sqlbuild_python_front::discovery::models::SecretPolicy; fn horizon(value: &str) -> PyResult { let invalid = || { @@ -63,7 +48,7 @@ fn horizon(value: &str) -> PyResult { TimeDelta::try_seconds(seconds).ok_or_else(invalid) } -fn calendar_times( +pub(crate) fn calendar_times( (authored, timezone): (&Bound<'_, PyAny>, &str), session: &Bound<'_, PyAny>, providers: &Bound<'_, PyDict>, @@ -142,132 +127,7 @@ fn calendar_times( }) } -/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. -pub(crate) fn compile_calendars( - py: Python<'_>, - project_dir: &Path, - (calendars, requests): (&BTreeMap>, Vec), - mut definitions: Definitions, -) -> PyResult { - if requests.is_empty() { - return Ok(definitions); - } - let providers = discover_provider_classes( - imported(py, "pathlib", "Path")?.call1((project_dir,))?, - SecretPolicy::Defer.name(), - )?; - let by_name = provider_by_name(&providers)?; - let session = call_with( - &entry(py, UserCodeHost::BuildProviderSession)?, - &[ - ("discovered_providers", providers), - ("allow_shared", PyBool::new(py, false).to_owned().into_any()), - ], - )?; - let now = Utc::now(); - let start = now - .with_nanosecond(now.timestamp_subsec_micros() * 1_000) - .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; - within(&session, |_| { - let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); - for request in requests { - let key = (request.calendar.clone(), request.timezone.clone()); - let schedule = match evaluated.get(&key) { - Some(schedule) => schedule.clone(), - None => { - let calendar = calendars.get(&request.calendar).ok_or_else(|| { - PyValueError::new_err(format!( - "calendar '{}' is not declared by a calendar module", - request.calendar - )) - })?; - let schedule = - calendar_times((calendar, &request.timezone), &session, &by_name, start) - .map_err(|error| { - PyValueError::new_err(format!( - "calendar '{}': {error}", - request.calendar - )) - })?; - evaluated.insert(key, schedule.clone()); - schedule - } - }; - let mut trigger = TriggerDefinition { - schedule, - ..request.trigger - }; - trigger.starts_at.get_or_insert(start); - definitions - .triggers - .insert(trigger.job_key.clone(), trigger); - } - Ok(definitions) - }) -} - -pub(crate) fn compile_sensors( - py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), - authored: Vec, - mut definitions: Definitions, -) -> PyResult { - let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { - orchestration_code_hash(project_dir)? - } else { - String::new() - }; - for sensor in authored { - if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) - { - continue; - } - let condition = match (sensor.when, sensor.function) { - (Some(when), None) => { - let mut condition = json!({"kind": when.kind}); - if when.kind == SOURCE_STALE_CONDITION { - condition["after_seconds"] = when.after_seconds; - } else if when.kind != SOURCE_NEW_DATA_CONDITION { - condition["kinds"] = json!(when.kinds); - } - if let Some(job) = when.job { - condition["job_key"] = json!(job); - } - if let Some(select) = when.select { - let selectors: Vec = - select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; - } - if let Some(selectors) = when.executed_successfully { - condition["executed_successfully"] = - sensor_nodes(graph, &selectors, "executed_successfully")?; - } - condition - } - (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, - "code_hash": callback_code_hash(function.bind(py), &sources)?}), - _ => { - return Err(PyValueError::new_err( - "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", - )); - } - }; - let definition: SensorDefinition = serde_json::from_value(json!({ - "description": sensor.description, - "job_key": sensor.run, - "when": condition, - "minimum_interval_seconds": sensor.minimum_interval_seconds, - "default_status": sensor.default_status, - "targets": sensor.targets, - "declared_at": declared_at(project_dir, sensor.declared_at)?, - })) - .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; - definitions.sensors.insert(sensor.name, definition); - } - Ok(definitions) -} - -fn sensor_nodes( +pub(crate) fn sensor_nodes( (graph, selected): (&ProjectGraph, &ProjectSelection), selectors: &[String], field: &str, diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs index b5be81b2b1..19194f7922 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs @@ -1,22 +1,83 @@ //! Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. -use crate::native_host::_helpers::orchestration_triggers; +use crate::native_host::_helpers::orchestration_triggers::calendar_times; use crate::native_host::models::CalendarRequest; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_orchestration::declarations::models::Definitions; +use chrono::{Timelike, Utc}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; +use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; +use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; +use sqlbuild_python_front::discovery::models::SecretPolicy; use std::collections::BTreeMap; use std::path::Path; /// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. -#[inline] pub fn compile_calendars( py: Python<'_>, project_dir: &Path, (calendars, requests): (&BTreeMap>, Vec), - definitions: Definitions, + mut definitions: Definitions, ) -> PyResult { - orchestration_triggers::compile_calendars(py, project_dir, (calendars, requests), definitions) + if requests.is_empty() { + return Ok(definitions); + } + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); + for request in requests { + let key = (request.calendar.clone(), request.timezone.clone()); + let schedule = match evaluated.get(&key) { + Some(schedule) => schedule.clone(), + None => { + let calendar = calendars.get(&request.calendar).ok_or_else(|| { + PyValueError::new_err(format!( + "calendar '{}' is not declared by a calendar module", + request.calendar + )) + })?; + let schedule = + calendar_times((calendar, &request.timezone), &session, &by_name, start) + .map_err(|error| { + PyValueError::new_err(format!( + "calendar '{}': {error}", + request.calendar + )) + })?; + evaluated.insert(key, schedule.clone()); + schedule + } + }; + let mut trigger = TriggerDefinition { + schedule, + ..request.trigger + }; + trigger.starts_at.get_or_insert(start); + definitions + .triggers + .insert(trigger.job_key.clone(), trigger); + } + Ok(definitions) + }) } diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs index 1003be8915..f61aef685c 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs @@ -1,22 +1,80 @@ //! Compile sensors. use crate::graph::models::ProjectSelection; -use crate::native_host::_helpers::orchestration_triggers; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::native_host::_helpers::orchestration_triggers::sensor_nodes; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use serde_json::json; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::declarations::models::Definitions; +use sqlbuild_orchestration::sensors::models::SensorDefinition; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; +use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; +use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; +use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; +use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; use std::path::Path; /// Compile sensors. -#[inline] pub fn compile_sensors( py: Python<'_>, (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), authored: Vec, - definitions: Definitions, + mut definitions: Definitions, ) -> PyResult { - orchestration_triggers::compile_sensors(py, (project_dir, input, graph), authored, definitions) + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; + for sensor in authored { + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) + { + continue; + } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind}); + if when.kind == SOURCE_STALE_CONDITION { + condition["after_seconds"] = when.after_seconds; + } else if when.kind != SOURCE_NEW_DATA_CONDITION { + condition["kinds"] = json!(when.kinds); + } + if let Some(job) = when.job { + condition["job_key"] = json!(job); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run, + "when": condition, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + "targets": sensor.targets, + "declared_at": declared_at(project_dir, sensor.declared_at)?, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index ae4fff6db3..aacdb803c1 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -1,42 +1,25 @@ //! Planner execution edges, their injected origins, stable execution order and buildability over compiled keys. +use crate::planner_identity::main::execution_deps::execution_deps; +use crate::planner_identity::main::execution_origins::execution_origins; +use crate::planner_identity::main::graph_key::graph_key; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySetMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::edge_keys::edge_keys; -use sqlbuild_analysis::graph::models::EdgeDirection; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; -use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::{ - AvailableUpstreams, ExecutionTest, GraphKey, MissingUpstream, -}; +use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; -use crate::compiled_views::main::project::compiled_models::compiled_models; -use crate::compiled_views::main::project::functions::functions; -use crate::compiled_views::main::project::seeds::seeds; -use crate::compiled_views::main::project::sources::sources; -use crate::compiled_views::main::project::sql_tests::sql_tests; use crate::graph::main::nodes::closure::closure; use crate::graph::main::nodes::inverted::inverted; -use crate::graph::main::nodes::native_key::native_key; use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; -use crate::graph::models::NativeProjectGraph; -use crate::planner_identity::_helpers::hashing::graph_key; -use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; -use crate::references::models::GateRow; -use crate::references::models::HookRow; -use crate::resource_views::main::resource_reads::graph_row::graph_row; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -45,7 +28,7 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const TABLE_FN_TEST_MODE: &str = "table_fn"; +pub(crate) const TABLE_FN_TEST_MODE: &str = "table_fn"; const INLINE_SQL_HOOK: &str = "inline SQL"; /// Python keys by graph key, keeping the first object seen for each. @@ -82,7 +65,7 @@ impl<'py> KeyObjects<'py> { } } -fn hook_label(hook_name: &str, kind: &str) -> String { +pub(crate) fn hook_label(hook_name: &str, kind: &str) -> String { if kind == INLINE_SQL_HOOK { return format!("inline SQL hook {hook_name}"); } @@ -97,45 +80,6 @@ pub(crate) fn build_execution_upstream_deps<'py>( edge_dict(&KeyCache::new(project.py())?, &execution_deps(project)?) } -/// Execution dependencies kept native until a public graph or plan asks for its dict view. -pub(crate) fn execution_deps( - project: &Bound<'_, PyAny>, -) -> PyResult)>> { - let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); - let collections: [Bound<'_, PyAny>; 4] = [ - compiled_models(project)?, - sources(project)?, - seeds(project)?, - functions(project)?, - ]; - for collection in collections { - for resource in collection.try_iter()? { - let row = graph_row(&resource?)?; - resources.push((row.key, row.deps)); - } - } - let mut tests: Vec = Vec::new(); - for test in sql_tests(project)?.try_iter()? { - let test = test?; - tests.push(ExecutionTest { - key: native_key(&native_field(&test, "key")?)?, - scope_deps: native_field(&test, "scope_deps")? - .try_iter()? - .map(|key| native_key(&key?)) - .collect::>()?, - table_function: native_field(&test, "mode")?.str()?.extract::()? - == TABLE_FN_TEST_MODE, - }); - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let injected: Vec<_> = gates - .into_iter() - .map(|gate| (gate.gated, gate.read)) - .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) - .collect(); - Ok(execution_upstream(resources, &tests, &injected)) -} - /// Human-readable origins for execution edges injected beyond pure lineage. #[pyfunction] pub(crate) fn build_execution_edge_origins<'py>( @@ -150,95 +94,6 @@ pub(crate) fn build_execution_edge_origins<'py>( Ok(result) } -/// Injected-edge descriptions remain native for cycle and buildability diagnostics. -pub(crate) fn execution_origins( - project: &Bound<'_, PyAny>, -) -> PyResult> { - let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); - for test in sql_tests(project)?.try_iter()? { - let test = test?; - let test_name: String = native_field(&test, "name")?.extract()?; - for target_key in native_field(&test, "scope_deps")?.try_iter()? { - let target_key = native_key(&target_key?)?; - let target_name = &target_key.1; - let description = format!("SQL test '{test_name}' runs before '{target_name}'"); - origins.push(( - (target_key, native_key(&native_field(&test, "key")?)?), - description, - )); - } - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let lineage = native_field(project, "lineage_graph")?; - let lineage = lineage.cast::()?.borrow(); - let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { - edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) - }; - let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); - for GateRow { - audit_name, - target, - gated, - read, - } in &gates - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), - )); - } - for HookRow { - hook_name, - gated, - read, - hook_type: kind, - .. - } in &hooks - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!( - "{} on '{}' reads '{}'", - hook_label(hook_name, kind), - gated.1, - read.1 - ), - )); - } - Ok(origins) -} - -/// Stable native execution order, projecting no Python graph or key objects. -pub(crate) fn execution_order( - py: Python<'_>, - upstream: &[(ObjectKey, Vec)], - origins: &[((ObjectKey, ObjectKey), String)], -) -> PyResult> { - let key = |(kind, name): &ObjectKey| GraphKey { - kind: kind.clone(), - name: name.clone(), - }; - let edges: Vec<_> = upstream - .iter() - .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) - .collect(); - let origins: HashMap<(GraphKey, GraphKey), String> = origins - .iter() - .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) - .collect(); - Ok(topological_order(&edges, &origins) - .map_err(|message| planner_input_error(py, message))? - .into_iter() - .map(|key| (key.kind, key.name)) - .collect()) -} - /// Downstream edges keyed by upstream key, each list sorted by `(resource_type, name)`. #[pyfunction] pub(crate) fn build_downstream_deps<'py>( @@ -263,7 +118,7 @@ pub(crate) fn build_downstream_deps<'py>( Ok(result) } -fn planner_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn planner_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::PlannerInput, &message) } @@ -417,16 +272,3 @@ pub(crate) fn missing_upstream_message<'py>( self::edge_origins(edge_origins, &mut objects)?; Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; - module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; - module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; - module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; - module.add_function(wrap_pyfunction!( - build_downstream_model_name_closure, - module - )?)?; - module.add_function(wrap_pyfunction!(check_buildability, module)?)?; - module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) -} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs index 2b4ad21b55..6dd300ebc1 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs @@ -1,16 +1,11 @@ //! Query and function definition fingerprints for change detection, failing as planner input errors. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{FINGERPRINT_HELP, QUERY_FINGERPRINT_FAILED}; -use sqlbuild_analysis::planner_identity::models::GraphKey; use sqlbuild_analysis::sql_tokens::main::baseline_fingerprint::baseline_fingerprint; use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; -use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -30,35 +25,8 @@ pub(crate) fn model_query_hash( .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) } -/// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. -pub(crate) fn baselined_model_query_hash( - py: Python<'_>, - model_name: &str, - (query_sql, dialect): (&str, Option<&str>), - recorded: &Bound<'_, PyAny>, -) -> PyResult { - let current: String = model_query_hash(py, model_name, (query_sql, dialect))?; - recorded_baseline(py, current, recorded, |sql| { - compiler_query_fingerprint(sql, dialect) - }) -} - -/// The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. -pub(crate) fn baselined_function_definition_hash( - py: Python<'_>, - function_name: &str, - (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), - recorded: &Bound<'_, PyAny>, -) -> PyResult { - let current: String = - function_definition_hash(py, function_name, (fingerprint_sql, language, dialect))?; - recorded_baseline(py, current, recorded, |sql| { - definition_fingerprint(sql, language, dialect) - }) -} - /// `current`, or the fingerprint recorded with a `Fingerprint` row whose definition fingerprints to `current`. -fn recorded_baseline( +pub(crate) fn recorded_baseline( py: Python<'_>, current: String, recorded: &Bound<'_, PyAny>, @@ -79,17 +47,7 @@ fn recorded_baseline( .map_err(compiler_error) } -/// The definition fingerprint of a function, or the planner input error naming the function. -pub(crate) fn function_definition_hash( - py: Python<'_>, - function_name: &str, - (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), -) -> PyResult { - py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) - .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) -} - -fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { +pub(crate) fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { if is_compiler_panic(&message) { return compiler_error(message); } @@ -114,27 +72,3 @@ fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { Err(error) => error, } } - -/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. -pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { - Ok(GraphKey { - kind: native_field(key, "resource_type")?.str()?.extract()?, - name: native_field(key, "name")?.extract()?, - }) -} - -/// Fingerprint a model query for change detection, naming the model when it cannot. -#[pyfunction] -#[pyo3(signature = (*, model_name, query_sql, dialect))] -pub(crate) fn model_definition_hash( - py: Python<'_>, - model_name: &str, - query_sql: &str, - dialect: Option<&str>, -) -> PyResult { - model_query_hash(py, model_name, (query_sql, dialect)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_definition_hash, module)?) -} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs index fe7a7755e8..a64b87a9bd 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs @@ -1,23 +1,20 @@ //! The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_identity::_helpers::hashing::recorded_baseline; +use crate::planner_identity::main::function_definition_hash::function_definition_hash; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; /// The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. -#[inline] pub fn baselined_function_definition_hash( py: Python<'_>, function_name: &str, (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), recorded: &Bound<'_, PyAny>, ) -> PyResult { - hashing::baselined_function_definition_hash( - py, - function_name, - (fingerprint_sql, language, dialect), - recorded, - ) + let current: String = + function_definition_hash(py, function_name, (fingerprint_sql, language, dialect))?; + recorded_baseline(py, current, recorded, |sql| { + definition_fingerprint(sql, language, dialect) + }) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs index ac498893a9..4327a91cd3 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs @@ -1,18 +1,19 @@ //! The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_identity::_helpers::hashing::model_query_hash; +use crate::planner_identity::_helpers::hashing::recorded_baseline; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; /// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. -#[inline] pub fn baselined_model_query_hash( py: Python<'_>, model_name: &str, (query_sql, dialect): (&str, Option<&str>), recorded: &Bound<'_, PyAny>, ) -> PyResult { - hashing::baselined_model_query_hash(py, model_name, (query_sql, dialect), recorded) + let current: String = model_query_hash(py, model_name, (query_sql, dialect))?; + recorded_baseline(py, current, recorded, |sql| { + compiler_query_fingerprint(sql, dialect) + }) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs index e8814c91d3..4e7fb11f96 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs @@ -1,13 +1,54 @@ //! Execution dependencies kept native until a public graph or plan asks for its dict view. -use crate::planner_identity::_helpers::graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_identity::_helpers::graph::TABLE_FN_TEST_MODE; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::models::{GateRow, HookRow}; +use crate::resource_views::main::resource_reads::graph_row::graph_row; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; +use sqlbuild_analysis::planner_identity::models::ExecutionTest; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Execution dependencies kept native until a public graph or plan asks for its dict view. -#[inline] pub fn execution_deps(project: &Bound<'_, PyAny>) -> PyResult)>> { - graph::execution_deps(project) + let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); + let collections: [Bound<'_, PyAny>; 4] = [ + compiled_models(project)?, + sources(project)?, + seeds(project)?, + functions(project)?, + ]; + for collection in collections { + for resource in collection.try_iter()? { + let row = graph_row(&resource?)?; + resources.push((row.key, row.deps)); + } + } + let mut tests: Vec = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + tests.push(ExecutionTest { + key: native_key(&native_field(&test, "key")?)?, + scope_deps: native_field(&test, "scope_deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + table_function: native_field(&test, "mode")?.str()?.extract::()? + == TABLE_FN_TEST_MODE, + }); + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let injected: Vec<_> = gates + .into_iter() + .map(|gate| (gate.gated, gate.read)) + .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) + .collect(); + Ok(execution_upstream(resources, &tests, &injected)) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs index bbb02f50a2..d0bb0282e4 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs @@ -1,16 +1,33 @@ //! Stable native execution order, projecting no Python graph or key objects. -use crate::planner_identity::_helpers::graph; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_identity::_helpers::graph::planner_input_error; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use std::collections::HashMap; /// Stable native execution order, projecting no Python graph or key objects. -#[inline] pub fn execution_order( py: Python<'_>, upstream: &[(ObjectKey, Vec)], origins: &[((ObjectKey, ObjectKey), String)], ) -> PyResult> { - graph::execution_order(py, upstream, origins) + let key = |(kind, name): &ObjectKey| GraphKey { + kind: kind.clone(), + name: name.clone(), + }; + let edges: Vec<_> = upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) + .collect(); + let origins: HashMap<(GraphKey, GraphKey), String> = origins + .iter() + .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) + .collect(); + Ok(topological_order(&edges, &origins) + .map_err(|message| planner_input_error(py, message))? + .into_iter() + .map(|key| (key.kind, key.name)) + .collect()) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs index 17deec03e6..75e0769e49 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs @@ -1,15 +1,78 @@ //! Injected-edge descriptions remain native for cycle and buildability diagnostics. -use crate::planner_identity::_helpers::graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::models::NativeProjectGraph; +use crate::planner_identity::_helpers::graph::hook_label; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::models::{GateRow, HookRow}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashSet; /// Injected-edge descriptions remain native for cycle and buildability diagnostics. -#[inline] pub fn execution_origins( project: &Bound<'_, PyAny>, ) -> PyResult> { - graph::execution_origins(project) + let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + let test_name: String = native_field(&test, "name")?.extract()?; + for target_key in native_field(&test, "scope_deps")?.try_iter()? { + let target_key = native_key(&target_key?)?; + let target_name = &target_key.1; + let description = format!("SQL test '{test_name}' runs before '{target_name}'"); + origins.push(( + (target_key, native_key(&native_field(&test, "key")?)?), + description, + )); + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let lineage = native_field(project, "lineage_graph")?; + let lineage = lineage.cast::()?.borrow(); + let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { + edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) + }; + let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); + for GateRow { + audit_name, + target, + gated, + read, + } in &gates + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + )); + } + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in &hooks + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 + ), + )); + } + Ok(origins) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs index fef31d807d..4deefe82b7 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs @@ -1,15 +1,16 @@ //! The definition fingerprint of a function, or the planner input error naming the function. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_identity::_helpers::hashing::fingerprint_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; +use sqlbuild_python_base::types::CompilerDetach; /// The definition fingerprint of a function, or the planner input error naming the function. -#[inline] pub fn function_definition_hash( py: Python<'_>, function_name: &str, (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), ) -> PyResult { - hashing::function_definition_hash(py, function_name, (fingerprint_sql, language, dialect)) + py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs index bee99c8141..62d4be2fcb 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs @@ -1,13 +1,13 @@ //! The graph key of a `CompiledObjectKey`-like value: its resource type text and name. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_identity::models::GraphKey; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. -#[inline] pub fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { - hashing::graph_key(key) + Ok(GraphKey { + kind: native_field(key, "resource_type")?.str()?.extract()?, + name: native_field(key, "name")?.extract()?, + }) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs index 1def34d9b3..22837bfd48 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs @@ -1,16 +1,17 @@ //! Fingerprint a model query for change detection, naming the model when it cannot. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_identity::_helpers::hashing::model_query_hash; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; /// Fingerprint a model query for change detection, naming the model when it cannot. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model_name, query_sql, dialect))] pub fn model_definition_hash( py: Python<'_>, model_name: &str, query_sql: &str, dialect: Option<&str>, ) -> PyResult { - hashing::model_definition_hash(py, model_name, query_sql, dialect) + model_query_hash(py, model_name, (query_sql, dialect)) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs index d24ebee179..c7aa14798a 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs @@ -1,12 +1,25 @@ //! Register the graph functions and classes. -use crate::planner_identity::_helpers::graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::graph::build_downstream_deps; +use crate::planner_identity::_helpers::graph::build_downstream_model_name_closure; +use crate::planner_identity::_helpers::graph::build_execution_edge_origins; +use crate::planner_identity::_helpers::graph::build_execution_upstream_deps; +use crate::planner_identity::_helpers::graph::check_buildability; +use crate::planner_identity::_helpers::graph::missing_upstream_message; +use crate::planner_identity::_helpers::graph::topologically_order_keys; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the graph functions and classes. -#[inline] pub fn register_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { - graph::register(module) + module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; + module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; + module.add_function(wrap_pyfunction!( + build_downstream_model_name_closure, + module + )?)?; + module.add_function(wrap_pyfunction!(check_buildability, module)?)?; + module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs index 7065675431..86e326bd6e 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs @@ -1,12 +1,10 @@ //! Register the hashing functions and classes. -use crate::planner_identity::_helpers::hashing; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_identity::main::model_definition_hash::model_definition_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the hashing functions and classes. -#[inline] pub fn register_hashing(module: &Bound<'_, PyModule>) -> PyResult<()> { - hashing::register(module) + module.add_function(wrap_pyfunction!(model_definition_hash, module)?) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs deleted file mode 100644 index d8ae2709c2..0000000000 --- a/crates/sqlbuild-python-project/src/planner_phases/_helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod objects; diff --git a/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs b/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs deleted file mode 100644 index dc83f41ecb..0000000000 --- a/crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Boundary helpers shared by the native planner phases. - -use sqlbuild_host::entries::models::HostId; - -use crate::compiled_views::main::project::collection::collection; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyFrozenSet, PyString, PyTuple}; -use pyo3::{BoundObject, IntoPyObject}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::model_config::main::config::config_text::config_text; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; - -/// A Python `str`. -pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { - PyString::new(py, value).into_any() -} - -/// `None`, or the text. -pub(crate) fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { - value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) -} - -/// The staying-Python class `class(**fields)`. -pub(crate) fn construct<'py>( - py: Python<'py>, - class: impl Into, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - call_with(&host_entry(py, class)?, fields) -} - -/// A planner model dataclass. -pub(crate) fn planner_model<'py>( - py: Python<'py>, - name: RecordId, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - new_record_with(py, name, fields) -} - -/// The native record `name(**fields)`. -pub(crate) fn native<'py>( - py: Python<'py>, - name: RecordId, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - new_record_with(py, name, fields) -} - -/// `dataclasses.replace(value, **fields)`. -pub(crate) fn replace<'py>( - value: &Bound<'py, PyAny>, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - replaced_keywords(value, &keywords(value.py(), fields)?) -} - -/// A staying-Python enum member from its value. -pub(crate) fn member<'py>( - py: Python<'py>, - class: impl Into, - value: &str, -) -> PyResult> { - host_entry(py, class)?.call1((value,)) -} - -/// A native string enum member from its value. -pub(crate) fn native_member<'py>( - py: Python<'py>, - class: RecordId, - value: &str, -) -> PyResult> { - new_record(py, class, (value,), None) -} - -/// `PlannerInputError(message, code=code, help=help)`. -pub(crate) fn planner_error( - py: Python<'_>, - message: &str, - (code, help): (Option<&str>, Option<&str>), -) -> PyErr { - let raised = error_class(py, ErrorId::PlannerInput).and_then(|class| { - let fields = keywords( - py, - &[ - ("code", optional_text(py, code)), - ("help", optional_text(py, help)), - ], - )?; - class.call((message,), Some(&fields)) - }); - match raised { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} - -/// A tuple of the items. -pub(crate) fn tuple<'py>( - py: Python<'py>, - items: Vec>, -) -> PyResult> { - Ok(PyTuple::new(py, items)?.into_any()) -} - -/// A frozenset of the items. -pub(crate) fn frozenset<'py>( - py: Python<'py>, - items: &[Bound<'py, PyAny>], -) -> PyResult> { - Ok(PyFrozenSet::new(py, items)?.into_any()) -} - -/// `sorted(items)` as a tuple of strings. -pub(crate) fn sorted_names<'py>( - py: Python<'py>, - mut names: Vec, -) -> PyResult> { - names.sort(); - tuple(py, names.iter().map(|name| text(py, name)).collect()) -} - -/// The strings of any Python iterable, in iteration order. -pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { - values - .try_iter()? - .map(|value| value?.extract::()) - .collect() -} - -/// The project nodes of `field` by name. -pub(crate) fn by_name<'py>( - project: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - let nodes = pyo3::types::PyDict::new(project.py()); - for node in collection(project, field)?.try_iter()? { - let node = node?; - pyo3::types::PyDictMethods::set_item( - &nodes, - crate::resource_views::main::resource_reads::name::name(&node)?, - node, - )?; - } - Ok(nodes) -} - -/// A model config value as text, `None` when absent or not text. -pub(crate) fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { - config_text(model.py(), &model_values(model)?, key) -} - -/// The compiled object key of the model `name`. -pub(crate) fn model_key<'py>( - py: Python<'py>, - name: &Bound<'py, PyAny>, -) -> PyResult> { - new_record_with( - py, - RecordId::CompiledObjectKey, - &[ - ( - "resource_type", - enum_member(py, RecordId::CompiledResourceType, "MODEL")?, - ), - ("name", name.clone()), - ], - ) -} - -/// A native helper's Rust result as a Python object. -pub(crate) fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> -where - T: IntoPyObject<'py>, - PyErr: From, -{ - Ok(value.into_pyobject(py)?.into_bound().into_any()) -} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs index e96b975c0c..130b9c42fc 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs @@ -1,15 +1,21 @@ //! The project nodes of `field` by name. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::collection::collection; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The project nodes of `field` by name. -#[inline] pub fn by_name<'py>( project: &Bound<'py, PyAny>, field: &str, ) -> PyResult> { - objects::by_name(project, field) + let nodes = pyo3::types::PyDict::new(project.py()); + for node in collection(project, field)?.try_iter()? { + let node = node?; + pyo3::types::PyDictMethods::set_item( + &nodes, + crate::resource_views::main::resource_reads::name::name(&node)?, + node, + )?; + } + Ok(nodes) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs index baf636fa88..da19090d81 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs @@ -1,12 +1,10 @@ //! A model config value as text, `None` when absent or not text. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// A model config value as text, `None` when absent or not text. -#[inline] pub fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { - objects::config_str(model, key) + config_text(model.py(), &model_values(model)?, key) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs index 2b86df19cc..daece070e4 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs @@ -1,18 +1,15 @@ //! The staying-Python class `class(**fields)`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_host::entries::models::HostId; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// The staying-Python class `class(**fields)`. -#[inline] pub fn construct<'py>( py: Python<'py>, class: impl Into, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - objects::construct(py, class, fields) + call_with(&host_entry(py, class)?, fields) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs index 5dc8500540..e59e3f13db 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs @@ -1,13 +1,9 @@ //! A frozenset of the items. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; /// A frozenset of the items. -#[inline] pub fn frozenset<'py>(py: Python<'py>, items: &[Bound<'py, PyAny>]) -> PyResult> { - objects::frozenset(py, items) + Ok(PyFrozenSet::new(py, items)?.into_any()) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs index 0b9930f1c9..14dddccf85 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/member.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs @@ -1,18 +1,14 @@ //! A staying-Python enum member from its value. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::entries::models::HostId; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// A staying-Python enum member from its value. -#[inline] pub fn member<'py>( py: Python<'py>, class: impl Into, value: &str, ) -> PyResult> { - objects::member(py, class, value) + host_entry(py, class)?.call1((value,)) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs index b98b22f067..c8c1e92d9b 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs @@ -1,13 +1,21 @@ //! The compiled object key of the model `name`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// The compiled object key of the model `name`. -#[inline] pub fn model_key<'py>(py: Python<'py>, name: &Bound<'py, PyAny>) -> PyResult> { - objects::model_key(py, name) + new_record_with( + py, + RecordId::CompiledObjectKey, + &[ + ( + "resource_type", + enum_member(py, RecordId::CompiledResourceType, "MODEL")?, + ), + ("name", name.clone()), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs index f44ccea1fb..c1c20a1760 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/native.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs @@ -1,18 +1,14 @@ //! The native record `name(**fields)`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; /// The native record `name(**fields)`. -#[inline] pub fn native<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - objects::native(py, name, fields) + new_record_with(py, name, fields) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs index 83504d8648..a1112e6bd3 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs @@ -1,18 +1,14 @@ //! A native string enum member from its value. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; /// A native string enum member from its value. -#[inline] pub fn native_member<'py>( py: Python<'py>, class: RecordId, value: &str, ) -> PyResult> { - objects::native_member(py, class, value) + new_record(py, class, (value,), None) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/object.rs b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs index c79d808128..683570850d 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/object.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs @@ -1,19 +1,13 @@ //! A native helper's Rust result as a Python object. -use crate::planner_phases::_helpers::objects; -use pyo3::IntoPyObject; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyErr, PyResult, Python}; +use pyo3::{BoundObject, IntoPyObject}; /// A native helper's Rust result as a Python object. -#[inline] pub fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> where T: IntoPyObject<'py>, PyErr: From, { - objects::object(py, value) + Ok(value.into_pyobject(py)?.into_bound().into_any()) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs index 172453dc34..55bb892586 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs @@ -1,12 +1,9 @@ //! `None`, or the text. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::Python; +use crate::planner_phases::main::text::text; +use pyo3::prelude::{Bound, PyAny, Python}; /// `None`, or the text. -#[inline] pub fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { - objects::optional_text(py, value) + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs index 6343200690..0e363eefd3 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs @@ -1,15 +1,29 @@ //! `PlannerInputError(message, code=code, help=help)`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use crate::planner_phases::main::optional_text::optional_text; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// `PlannerInputError(message, code=code, help=help)`. -#[inline] pub fn planner_error( py: Python<'_>, message: &str, (code, help): (Option<&str>, Option<&str>), ) -> PyErr { - objects::planner_error(py, message, (code, help)) + let raised = error_class(py, ErrorId::PlannerInput).and_then(|class| { + let fields = keywords( + py, + &[ + ("code", optional_text(py, code)), + ("help", optional_text(py, help)), + ], + )?; + class.call((message,), Some(&fields)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs index ce3819ce10..299978c4b4 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs @@ -1,18 +1,14 @@ //! A planner model dataclass. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; /// A planner model dataclass. -#[inline] pub fn planner_model<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - objects::planner_model(py, name, fields) + new_record_with(py, name, fields) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs index b6666f0a7a..735c470993 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs @@ -1,15 +1,13 @@ //! `dataclasses.replace(value, **fields)`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; /// `dataclasses.replace(value, **fields)`. -#[inline] pub fn replace<'py>( value: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - objects::replace(value, fields) + replaced_keywords(value, &keywords(value.py(), fields)?) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs index cdddb0b35e..93a2a16f3d 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs @@ -1,13 +1,11 @@ //! `sorted(items)` as a tuple of strings. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_phases::main::text::text; +use crate::planner_phases::main::tuple::tuple; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// `sorted(items)` as a tuple of strings. -#[inline] -pub fn sorted_names<'py>(py: Python<'py>, names: Vec) -> PyResult> { - objects::sorted_names(py, names) +pub fn sorted_names<'py>(py: Python<'py>, mut names: Vec) -> PyResult> { + names.sort(); + tuple(py, names.iter().map(|name| text(py, name)).collect()) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs index f2c03bea04..21950fbc5b 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs @@ -1,12 +1,11 @@ //! The strings of any Python iterable, in iteration order. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The strings of any Python iterable, in iteration order. -#[inline] pub fn strings(values: &Bound<'_, PyAny>) -> PyResult> { - objects::strings(values) + values + .try_iter()? + .map(|value| value?.extract::()) + .collect() } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs index 6f33a4c39b..b7116c021f 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/text.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs @@ -1,12 +1,9 @@ //! A Python `str`. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, Python}; +use pyo3::types::PyString; /// A Python `str`. -#[inline] pub fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { - objects::text(py, value) + PyString::new(py, value).into_any() } diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs index 33b17ae72a..c482382182 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs @@ -1,13 +1,9 @@ //! A tuple of the items. -use crate::planner_phases::_helpers::objects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; /// A tuple of the items. -#[inline] pub fn tuple<'py>(py: Python<'py>, items: Vec>) -> PyResult> { - objects::tuple(py, items) + Ok(PyTuple::new(py, items)?.into_any()) } diff --git a/crates/sqlbuild-python-project/src/planner_phases/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/mod.rs index efdc766b58..74d7b06897 100644 --- a/crates/sqlbuild-python-project/src/planner_phases/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_phases/mod.rs @@ -1,4 +1,3 @@ //! Planner phase objects and scopes. -pub(crate) mod _helpers; pub mod main; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs index 2c6bb4bacc..bce0b2f527 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs @@ -2,14 +2,11 @@ use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; -use crate::planner_scope::_helpers::values::dependency_names; -use crate::planner_scope::_helpers::values::edges; -use crate::planner_scope::_helpers::values::keys; -use crate::planner_scope::_helpers::values::loader_graph; -use crate::planner_scope::_helpers::values::loader_names; -use crate::planner_scope::_helpers::values::project_graph; +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::project_graph::project_graph; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::main::{ expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, @@ -20,18 +17,6 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::loader_parts::loader_parts; use std::collections::HashSet; -#[pyfunction] -#[pyo3(signature = (*, loader_function, loader_functions))] -pub(crate) fn upstream_loader_dependency_names<'py>( - loader_function: &Bound<'py, PyAny>, - loader_functions: &Bound<'py, PyAny>, -) -> PyResult> { - let graph = loader_graph(loader_functions)?; - let names = loader_names(loader_functions)?; - let deps = dependency_names(loader_function, &names)?; - PyTuple::new(loader_function.py(), graph.upstream(&deps)) -} - pub(crate) fn loader_to_source_entry<'py>( project: &Bound<'py, PyAny>, loader: &Bound<'py, PyAny>, @@ -94,22 +79,6 @@ pub(crate) fn intermediate_map<'py>( Ok(result) } -#[pyfunction] -#[pyo3(signature = (*, project, selected_keys))] -pub(crate) fn build_intermediate_source_map<'py>( - project: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult> { - intermediate_map(project, &keys(selected_keys)?, false) -} - -pub(crate) fn build_upstream_intermediate_source_map<'py>( - project: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult> { - intermediate_map(project, &keys(selected_keys)?, true) -} - #[pyfunction] #[pyo3(signature = (*, project, selected_keys, upstream_deps, executable_dependency_source_keys = None))] pub(crate) fn expand_selected_loader_dependencies<'py>( @@ -134,24 +103,3 @@ pub(crate) fn expand_selected_loader_dependencies<'py>( edge_dict(&cache, &upstream)?, )) } - -/// `build_intermediate_source_map` as a function of `module`. -pub(crate) fn build_intermediate_source_map_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_intermediate_source_map, module) -} - -/// `expand_selected_loader_dependencies` as a function of `module`. -pub(crate) fn expand_selected_loader_dependencies_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(expand_selected_loader_dependencies, module) -} - -/// `upstream_loader_dependency_names` as a function of `module`. -pub(crate) fn upstream_loader_dependency_names_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(upstream_loader_dependency_names, module) -} diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs index 51d8b6a255..bd56463a03 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -1,5 +1,7 @@ //! Native planner scope ownership, with projection only at public record boundaries. +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; use crate::planner_scope::models::RetainedScope; use crate::planner_scope::models::ScopeData; use std::collections::HashMap; @@ -8,13 +10,8 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::native_host::models::ModelHandle; -use crate::planner_scope::_helpers::values::edges; -use crate::planner_scope::_helpers::values::keys; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::models::NativeRecord; -use sqlbuild_python_base::records::types::RecordId; /// Read native ownership directly, or import a caller-authored public scope at the API edge. pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { @@ -57,38 +54,3 @@ pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { .collect(), })) } - -/// Preserve public record identity and dataclass behavior on explicit projection. -pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let Ok(scope) = value.cast::() else { - return Ok(value.clone()); - }; - scope - .get() - .projection - .get_or_try_init(value.py(), || { - let names = [ - "upstream_deps", - "downstream_deps", - "all_keys", - "models_by_name", - "selected_keys", - "execution_order", - "user_selected_keys", - "python_read_source_names", - "sql_test_selection", - "load_sources", - "end_audit_source_keys", - ]; - let fields = names - .iter() - .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) - .collect::>>()?; - Ok::<_, pyo3::PyErr>( - new_record_with(value.py(), RecordId::PlannerScope, &fields)? - .cast_into::() - .map(Bound::unbind)?, - ) - }) - .map(|value| value.bind(scope.py()).clone().into_any()) -} diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs index 53f73cb188..639297c7ae 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs @@ -8,8 +8,8 @@ use crate::native_host::models::ModelHandle; use crate::planner_identity::main::execution_deps::execution_deps; use crate::planner_identity::main::execution_order::execution_order; use crate::planner_identity::main::execution_origins::execution_origins; -use crate::planner_scope::_helpers::values::keys; -use crate::planner_scope::_helpers::values::project_graph; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::project_graph::project_graph; use crate::planner_scope::models::ScopeData; use crate::planner_selection::main::py_strings::py_strings; use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs index 9348f3b9c7..36585d9e3c 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs @@ -1,18 +1,13 @@ //! Existing source and load plan models materialised at the Python boundary. +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::source_load_map::source_load_map; use sqlbuild_host::entries::models::ExecutorHost; -use crate::compiled_views::main::project::sources::sources; -use crate::graph::main::nodes::native_key::native_key; -use crate::planner_scope::_helpers::loaders::intermediate_map; -use crate::planner_scope::_helpers::values::edges; -use crate::planner_scope::_helpers::values::keys; -use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyDictMethods; use pyo3::types::{PyBool, PyDict, PyTuple}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -33,34 +28,6 @@ pub(crate) fn build_source_load_map<'py>( ) } -/// Select source entries from native reachability; the resulting entries cross the executor edge. -pub(crate) fn source_load_map<'py>( - project: &Bound<'py, PyAny>, - selected: &[ObjectKey], - upstream_deps: Option<&[(ObjectKey, Vec)]>, -) -> PyResult> { - let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); - let result = PyDict::new(project.py()); - for source in sources(project)?.try_iter()? { - let source = source?; - if let Some(keys) = &relevant - && !keys.contains(&native_key( - &crate::resource_views::main::source_reads::key::key(&source)?, - )?) - { - continue; - } - let entry = source_entry(&source)?; - result.set_item(native_field(&entry, "name")?, entry)?; - } - for upstream in [false, true] { - for (key, entry) in intermediate_map(project, selected, upstream)?.iter() { - result.set_item(key, entry)?; - } - } - Ok(result) -} - #[pyfunction] #[pyo3(signature = (*, execution_order, selected_keys, source_map, is_reload))] pub(crate) fn build_source_load_entries<'py>( @@ -134,17 +101,3 @@ pub(crate) fn build_source_load_entries<'py>( } PyTuple::new(py, entries) } - -/// `build_source_load_entries` as a function of `module`. -pub(crate) fn build_source_load_entries_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_source_load_entries, module) -} - -/// `build_source_load_map` as a function of `module`. -pub(crate) fn build_source_load_map_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_source_load_map, module) -} diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs index 34cdb9d8bd..07251d4cdf 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs @@ -1,32 +1,10 @@ //! Python function identity and compiled graph conversion. -use crate::compiled_views::main::project::sources::sources; -use crate::graph::main::nodes::native_key::native_key; -use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; -use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::models::LoaderGraph; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; - -pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { - values - .try_iter()? - .map(|value| native_key(&value?)) - .collect() -} - -pub(crate) fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { - mapping_items(values)? - .try_iter()? - .map(|item| { - let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - Ok((native_key(&key)?, keys(&deps)?)) - }) - .collect() -} /// Resolve dependencies using the same Python dictionary lookup as the authored function boundary. pub(crate) fn loader_names<'py>(loaders: &Bound<'py, PyAny>) -> PyResult> { @@ -66,15 +44,3 @@ pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult } Ok(graph) } - -pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { - let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; - for source in sources(project)?.try_iter()? { - let entry = source_entry(&source?)?; - graph.sources.push(( - native_field(&entry, "name")?.extract()?, - native_field(&entry, "loader")?.extract()?, - )); - } - Ok(graph) -} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs index b0f5781bc7..807222a240 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs @@ -1,16 +1,16 @@ //! Build intermediate source map. -use crate::planner_scope::_helpers::loaders; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; use pyo3::types::PyDict; /// Build intermediate source map. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] pub fn build_intermediate_source_map<'py>( project: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult> { - loaders::build_intermediate_source_map(project, selected_keys) + intermediate_map(project, &keys(selected_keys)?, false) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs index 2ae00ddc8f..56dc7ac73f 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs @@ -1,11 +1,10 @@ //! `build_intermediate_source_map` as a function of `module`. -use crate::planner_scope::_helpers::loaders; +use crate::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; /// `build_intermediate_source_map` as a function of `module`. -#[inline] pub fn build_intermediate_source_map_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - loaders::build_intermediate_source_map_function(module) + pyo3::wrap_pyfunction!(build_intermediate_source_map, module) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs index 54514bf26d..c474c1deb4 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs @@ -1,10 +1,7 @@ //! Build planner scope. use crate::planner_scope::_helpers::scope; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyDict; /// Build planner scope. diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs index 284b37217e..e7b23b78c3 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs @@ -1,11 +1,8 @@ //! Build source load entries. use crate::planner_scope::_helpers::sources; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyTuple}; /// Build source load entries. #[inline] diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs index 7f38b930dc..20292d5327 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs @@ -1,11 +1,10 @@ //! `build_source_load_entries` as a function of `module`. -use crate::planner_scope::_helpers::sources; +use crate::planner_scope::_helpers::sources::build_source_load_entries; /// `build_source_load_entries` as a function of `module`. -#[inline] pub fn build_source_load_entries_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - sources::build_source_load_entries_function(module) + pyo3::wrap_pyfunction!(build_source_load_entries, module) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs index d5aad260b7..b5ae8b9a02 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs @@ -1,11 +1,10 @@ //! `build_source_load_map` as a function of `module`. -use crate::planner_scope::_helpers::sources; +use crate::planner_scope::_helpers::sources::build_source_load_map; /// `build_source_load_map` as a function of `module`. -#[inline] pub fn build_source_load_map_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - sources::build_source_load_map_function(module) + pyo3::wrap_pyfunction!(build_source_load_map, module) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs index 11efd56bf0..7ddd568f53 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs @@ -1,16 +1,14 @@ //! Build upstream intermediate source map. -use crate::planner_scope::_helpers::loaders; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Build upstream intermediate source map. -#[inline] pub fn build_upstream_intermediate_source_map<'py>( project: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult> { - loaders::build_upstream_intermediate_source_map(project, selected_keys) + intermediate_map(project, &keys(selected_keys)?, true) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs index 73eaf1822f..9b5e76e884 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs @@ -1,13 +1,18 @@ //! Edges. -use crate::planner_scope::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// Edges. -#[inline] pub fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { - values::edges(values) + mapping_items(values)? + .try_iter()? + .map(|item| { + let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + Ok((native_key(&key)?, keys(&deps)?)) + }) + .collect() } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs index 5718aeb8a1..5748044e8d 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs @@ -1,11 +1,10 @@ //! `expand_selected_loader_dependencies` as a function of `module`. -use crate::planner_scope::_helpers::loaders; +use crate::planner_scope::_helpers::loaders::expand_selected_loader_dependencies; /// `expand_selected_loader_dependencies` as a function of `module`. -#[inline] pub fn expand_selected_loader_dependencies_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - loaders::expand_selected_loader_dependencies_function(module) + pyo3::wrap_pyfunction!(expand_selected_loader_dependencies, module) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs index 913a26182a..5d86ce0c14 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs @@ -1,13 +1,13 @@ //! Keys. -use crate::planner_scope::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::main::nodes::native_key::native_key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; /// Keys. -#[inline] pub fn keys(values: &Bound<'_, PyAny>) -> PyResult> { - values::keys(values) + values + .try_iter()? + .map(|value| native_key(&value?)) + .collect() } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs index ccdab14bc1..23304c0abc 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs @@ -1,13 +1,21 @@ //! Project graph. -use crate::planner_scope::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::sources::sources; +use crate::planner_scope::_helpers::values::loader_graph; +use crate::resource_views::main::source_reads::source_entry::source_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_scope::models::LoaderGraph; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Project graph. -#[inline] pub fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { - values::project_graph(project) + let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; + graph.sources.push(( + native_field(&entry, "name")?.extract()?, + native_field(&entry, "loader")?.extract()?, + )); + } + Ok(graph) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs index a05306cd50..734d1789c4 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs @@ -1,12 +1,42 @@ //! Preserve public record identity and dataclass behavior on explicit projection. -use crate::planner_scope::_helpers::retained; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_scope::models::RetainedScope; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::models::NativeRecord; +use sqlbuild_python_base::records::types::RecordId; /// Preserve public record identity and dataclass behavior on explicit projection. -#[inline] pub fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - retained::project_scope(value) + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs index de17d920f6..bddaaf8095 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs @@ -2,9 +2,7 @@ use crate::planner_scope::_helpers::retained; use crate::planner_scope::models::ScopeData; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use std::sync::Arc; /// Read native ownership directly, or import a caller-authored public scope at the API edge. diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs index dee238edc0..2595423ad1 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs @@ -1,18 +1,40 @@ //! Select source entries from native reachability; the resulting entries cross the executor edge. -use crate::planner_scope::_helpers::sources; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::main::project::sources::sources; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::resource_views::main::source_reads::source_entry::source_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Select source entries from native reachability; the resulting entries cross the executor edge. -#[inline] pub fn source_load_map<'py>( project: &Bound<'py, PyAny>, selected: &[ObjectKey], upstream_deps: Option<&[(ObjectKey, Vec)]>, ) -> PyResult> { - sources::source_load_map(project, selected, upstream_deps) + let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); + let result = PyDict::new(project.py()); + for source in sources(project)?.try_iter()? { + let source = source?; + if let Some(keys) = &relevant + && !keys.contains(&native_key( + &crate::resource_views::main::source_reads::key::key(&source)?, + )?) + { + continue; + } + let entry = source_entry(&source)?; + result.set_item(native_field(&entry, "name")?, entry)?; + } + for upstream in [false, true] { + for (key, entry) in intermediate_map(project, selected, upstream)?.iter() { + result.set_item(key, entry)?; + } + } + Ok(result) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs index 892acbee20..911338e101 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs @@ -1,16 +1,18 @@ //! Upstream loader dependency names. -use crate::planner_scope::_helpers::loaders; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_scope::_helpers::values::{dependency_names, loader_graph, loader_names}; +use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; use pyo3::types::PyTuple; /// Upstream loader dependency names. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, loader_function, loader_functions))] pub fn upstream_loader_dependency_names<'py>( loader_function: &Bound<'py, PyAny>, loader_functions: &Bound<'py, PyAny>, ) -> PyResult> { - loaders::upstream_loader_dependency_names(loader_function, loader_functions) + let graph = loader_graph(loader_functions)?; + let names = loader_names(loader_functions)?; + let deps = dependency_names(loader_function, &names)?; + PyTuple::new(loader_function.py(), graph.upstream(&deps)) } diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs index d4eeff3d83..4fb4f65fea 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs @@ -1,11 +1,10 @@ //! `upstream_loader_dependency_names` as a function of `module`. -use crate::planner_scope::_helpers::loaders; +use crate::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; /// `upstream_loader_dependency_names` as a function of `module`. -#[inline] pub fn upstream_loader_dependency_names_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - loaders::upstream_loader_dependency_names_function(module) + pyo3::wrap_pyfunction!(upstream_loader_dependency_names, module) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs index c3209dac10..92be781877 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs @@ -1,18 +1,9 @@ //! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyString; pub(crate) const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; -/// Python strings for each item of an iterable of `str`. -pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { - values - .try_iter()? - .map(|value| Ok(value?.downcast_into::()?)) - .collect() -} - pub(crate) fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { let code: u32 = character .py() diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs index cbcf3939a9..b3a9a34c20 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs @@ -1,23 +1,12 @@ //! `+` marker splitting and SQLBuild-owned model selectors for dbt interop. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::PyBool; -use pyo3::types::{PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; -use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; -use sqlbuild_analysis::planner_selection::models::{ - ModelSelectorNames, SelectableModel, SelectorExpansion, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_analysis::planner_selection::models::SelectableModel; -use crate::compiled_views::main::project::compiled_models::compiled_models; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; @@ -25,7 +14,7 @@ const TAGS_KEY: &str = "tags"; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; -fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { +pub(crate) fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { let raised = imported(carrier.py(), SHARED_EXCEPTIONS_MODULE, "SharedInputError") .and_then(|class| class.call1((carrier.back(message)?,))); match raised { @@ -34,36 +23,6 @@ fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { } } -/// Split `+selector+` marker syntax from the selector core without resolving the core. -#[pyfunction] -pub(crate) fn split_selector_expansion<'py>( - raw: &Bound<'py, PyString>, -) -> PyResult> { - let py: Python<'py> = raw.py(); - let carrier: Carrier<'py> = Carrier::single(raw)?; - let expansion: SelectorExpansion = - split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; - new_record_with( - py, - RecordId::SelectorExpansion, - &[ - ("core", carrier.back(&expansion.core)?.into_any()), - ( - "upstream", - PyBool::new(py, expansion.upstream).to_owned().into_any(), - ), - ( - "downstream", - expansion - .downstream - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ], - ) -} - fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { let tags: Bound<'_, PyAny> = config_value(model.py(), &model_values(model)?, TAGS_KEY)?; if !tags.is_instance_of::() && !tags.is_instance_of::() { @@ -72,7 +31,7 @@ fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { tags.try_iter()?.map(|tag| tag?.str()?.extract()).collect() } -fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { Ok(SelectableModel { name: native_field(model, "name")?.extract()?, tags: model_tags(model)?, @@ -81,40 +40,3 @@ fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { .extract()?, }) } - -/// One SQLBuild-owned model selector core's model names, and its path selector when translated. -pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( - project: &Bound<'py, PyAny>, - term: &Bound<'py, PyString>, -) -> PyResult> { - let py: Python<'py> = project.py(); - let models: Vec = compiled_models(project)? - .try_iter()? - .map(|model| selectable_model(&model?)) - .collect::>()?; - let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { - Ok(models - .iter() - .flat_map(|model| [model.name.clone(), model.folder.clone()]) - .chain(models.iter().flat_map(|model| model.tags.clone())) - .collect()) - })?; - let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); - let translated: Option> = resolved - .translated - .as_deref() - .map(|path| carrier.back(path)) - .transpose()?; - PyTuple::new( - py, - [ - PyTuple::new(py, resolved.names)?.into_any(), - translated.into_pyobject(py)?.into_any(), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs index cc380deaaf..e464a53b32 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs @@ -1,31 +1,23 @@ //! Planner selector parsing and resolution over native project graphs, raising `PlannerInputError`. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyFrozenSet, PySetMethods, PyString}; +use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyBool, PyFrozenSet}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; -use sqlbuild_analysis::graph::main::match_selector::match_selector; -use sqlbuild_analysis::graph::main::parse_selector::parse_selector; -use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; -use crate::graph::main::nodes::closure::closure; use crate::graph::main::project::graph_from_view_mappings::graph_from_view_mappings; -use crate::graph::models::CompiledProjectGraph; use crate::graph::models::GraphViewMappings; use crate::graph::models::KeyCache; use crate::graph::models::NativeProjectGraph; -use crate::planner_selection::_helpers::carrier::py_strings; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -const VIEW_OPTIONS: &[&str] = &[ +pub(crate) const VIEW_OPTIONS: &[&str] = &[ "all_keys", "upstream_deps", "downstream_deps", @@ -79,11 +71,11 @@ pub(crate) fn parsed_selector<'py>( } } -fn graph_names(graph: &ProjectGraph) -> Vec { +pub(crate) fn graph_names(graph: &ProjectGraph) -> Vec { graph.names.iter().map(|(name, _)| name.clone()).collect() } -fn selected<'py>( +pub(crate) fn selected<'py>( carrier: &Carrier<'py>, outcome: Result, SelectorError>, ) -> PyResult> { @@ -95,103 +87,18 @@ fn compiled_keys<'py>(py: Python<'py>, keys: &[ObjectKey]) -> PyResult( +pub(crate) fn resolved<'py>( graph: &ProjectGraph, (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { compiled_keys(select.py(), &resolve_graph_keys(graph, (select, exclude))?) } -/// Resolve graph keys for native planning without building a public frozenset. -pub(crate) fn resolve_graph_keys<'py>( - graph: &ProjectGraph, - (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = select.py(); - let select: Vec> = py_strings(select)?; - let exclude: Vec> = py_strings(exclude)?; - let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); - let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; - let (select, exclude) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); - let all: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, graph, &all, &[])?; - resolve_selectors(&graph, &select, &exclude).map_err(|error| carrier.error(error)) -} - -fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { +pub(crate) fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { graph_from_view_mappings(mappings) } -/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. -pub(crate) fn parse_project_selector<'py>( - raw: &Bound<'py, PyString>, -) -> PyResult> { - let carrier: Carrier<'py> = Carrier::single(raw)?; - let parsed: ParsedSelector = - parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; - parsed_selector(&carrier, parsed) -} - -/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. -pub(crate) fn resolve_graph_selection<'py>( - graph: PyRef<'py, NativeProjectGraph>, - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, -) -> PyResult> { - resolved(graph.graph(), (select, exclude)) -} - -/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. -pub(crate) fn select_project_graph<'py>( - graph: &Bound<'py, PyAny>, - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, -) -> PyResult> { - let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; - resolved( - graph.get().native(graph.py()).get().graph(), - (select, exclude), - ) -} - -/// Model, source, seed and function selectors resolved against compiled graph mappings. -pub(crate) fn resolve_project_selectors<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_project_selectors", - options, - (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), - )?; - let graph: NativeProjectGraph = arguments_graph(&arguments)?; - resolved( - graph.graph(), - (&arguments.value("select")?, &arguments.value("exclude")?), - ) -} - -/// The keys whitespace-separated selector tokens match, without required build resources. -pub(crate) fn resolve_project_selector_tokens<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_project_selector_tokens", - options, - (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), - )?; - let graph: NativeProjectGraph = arguments_graph(&arguments)?; - let texts: Vec> = py_strings(&arguments.value("selectors")?)?; - let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; - let selectors: Vec = carrier.send_all(&texts)?; - let graph = graph_with_jobs(py, graph.graph(), &selectors, &[])?; - selected(&carrier, resolve_selector_tokens(&graph, &selectors)) -} - -fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { +pub(crate) fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { view_graph(GraphViewMappings { all_keys: &arguments.value("all_keys")?, upstream: &arguments.value("upstream_deps")?, @@ -200,41 +107,3 @@ fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult( - parsed: &Bound<'py, PyAny>, - all_keys: &Bound<'py, PyAny>, - tag_index: &Bound<'py, PyAny>, - path_index: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = parsed.py(); - let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); - let graph: NativeProjectGraph = view_graph(GraphViewMappings { - all_keys, - upstream: &empty, - downstream: &empty, - tag_index, - path_index, - })?; - let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; - let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { - Ok(graph_names(graph.graph())) - })?; - let kind: String = native_field(parsed, "kind")?.str()?.extract()?; - let value: String = carrier.send(&value)?; - let graph = graph_with_jobs(py, graph.graph(), &[format!("{kind}:{value}")], &[])?; - selected( - &carrier, - match_selector(&graph, &kind, &value).map(|keys| keys.into_iter().collect()), - ) -} - -/// The upstream key closure of one compiled graph key. -pub(crate) fn expand_project_upstream_keys<'py>( - key: Bound<'py, PyAny>, - upstream_deps: &Bound<'py, PyAny>, -) -> PyResult> { - let visited = closure(vec![key], upstream_deps, (false, None))?; - PyFrozenSet::new(upstream_deps.py(), visited.iter()) -} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 0eaea0c386..600f939727 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -1,5 +1,6 @@ //! Unit-test selector splitting over discovered project names. +use crate::planner_selection::main::py_strings::py_strings; use std::collections::BTreeSet; use pyo3::IntoPyObject; @@ -13,7 +14,6 @@ use sqlbuild_analysis::planner_selection::models::{ }; use crate::compiled_project::main::job_names::job_names; -use crate::planner_selection::_helpers::carrier::py_strings; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -116,7 +116,7 @@ fn names_unknown(selectors: &[String], names: &UnitTestSelectorNames) -> bool { }) } -fn split_python<'py>( +pub(crate) fn split_python<'py>( (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), discovered: &Bound<'py, PyAny>, accepts_unit_tests: bool, @@ -176,31 +176,3 @@ fn split_python<'py>( ], ) } - -/// Move unit-test selectors out of resource selectors, or reject them for this command. -pub(crate) fn split_project_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, - accepts_unit_tests: bool, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, accepts_unit_tests) -} - -/// Split unit-test selectors from resource selectors for a command that runs unit tests. -pub(crate) fn split_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, true) -} - -/// Raise when a selector names a unit test; return the unchanged resource selectors. -pub(crate) fn reject_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, false) -} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs index 2487e94297..b55375ebfd 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs @@ -1,16 +1,14 @@ //! The upstream key closure of one compiled graph key. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::graph::main::nodes::closure::closure; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyFrozenSet, PySetMethods}; /// The upstream key closure of one compiled graph key. -#[inline] pub fn expand_project_upstream_keys<'py>( key: Bound<'py, PyAny>, upstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { - selectors::expand_project_upstream_keys(key, upstream_deps) + let visited = closure(vec![key], upstream_deps, (false, None))?; + PyFrozenSet::new(upstream_deps.py(), visited.iter()) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs index 61df662932..fd2804e314 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs @@ -1,18 +1,42 @@ //! The keys one parsed selector matches before `+` graph expansion. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::_helpers::selectors::selected; +use crate::planner_selection::_helpers::selectors::view_graph; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyString}; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The keys one parsed selector matches before `+` graph expansion. -#[inline] pub fn match_project_selector<'py>( parsed: &Bound<'py, PyAny>, all_keys: &Bound<'py, PyAny>, tag_index: &Bound<'py, PyAny>, path_index: &Bound<'py, PyAny>, ) -> PyResult> { - selectors::match_project_selector(parsed, all_keys, tag_index, path_index) + let py: Python<'py> = parsed.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let graph: NativeProjectGraph = view_graph(GraphViewMappings { + all_keys, + upstream: &empty, + downstream: &empty, + tag_index, + path_index, + })?; + let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { + Ok(graph_names(graph.graph())) + })?; + let kind: String = native_field(parsed, "kind")?.str()?.extract()?; + let value: String = carrier.send(&value)?; + let graph = graph_with_jobs(py, graph.graph(), &[format!("{kind}:{value}")], &[])?; + selected( + &carrier, + match_selector(&graph, &kind, &value).map(|keys| keys.into_iter().collect()), + ) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs index ff0c3b60e9..f59594d3c7 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs @@ -1,13 +1,16 @@ //! Parse one selector token into Python's `ParsedSelector` or `PathSelector`. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::selectors::parsed_selector; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyString; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; /// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. -#[inline] pub fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult> { - selectors::parse_project_selector(raw) + let carrier: Carrier<'py> = Carrier::single(raw)?; + let parsed: ParsedSelector = + parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; + parsed_selector(&carrier, parsed) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs index ec617699dc..9f1fc1e831 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs @@ -1,13 +1,12 @@ //! Python strings for each item of an iterable of `str`. -use crate::planner_selection::_helpers::carrier; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; /// Python strings for each item of an iterable of `str`. -#[inline] pub fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { - carrier::py_strings(values) + values + .try_iter()? + .map(|value| Ok(value?.downcast_into::()?)) + .collect() } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs index de754ed49f..0758ef5fcf 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs @@ -1,12 +1,11 @@ //! Register the model selectors functions and classes. -use crate::planner_selection::_helpers::model_selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_selection::main::split_selector_expansion::split_selector_expansion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model selectors functions and classes. -#[inline] pub fn register_model_selectors(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_selectors::register(module) + module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs index 3e321e2401..cc86e70da0 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs @@ -1,16 +1,13 @@ //! Raise when a selector names a unit test; return the unchanged resource selectors. -use crate::planner_selection::_helpers::unit_tests; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Raise when a selector names a unit test; return the unchanged resource selectors. -#[inline] pub fn reject_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - unit_tests::reject_unit_test_selectors(select, exclude, discovered_inputs) + split_python((select, exclude), discovered_inputs, false) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs index f648dc9061..c52bfcc3c6 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs @@ -1,17 +1,27 @@ //! Resolve graph keys for native planning without building a public frozenset. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyString; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; /// Resolve graph keys for native planning without building a public frozenset. -#[inline] pub fn resolve_graph_keys<'py>( graph: &ProjectGraph, (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - selectors::resolve_graph_keys(graph, (select, exclude)) + let py: Python<'py> = select.py(); + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; + let (select, exclude) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let all: Vec = select.iter().chain(&exclude).cloned().collect(); + let graph = graph_with_jobs(py, graph, &all, &[])?; + resolve_selectors(&graph, &select, &exclude).map_err(|error| carrier.error(error)) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs index 95d13f3949..8060ccc44a 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs @@ -1,19 +1,15 @@ //! `--select` minus `--exclude` on a native graph, adding the functions selected models need. use crate::graph::models::NativeProjectGraph; -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyRef; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyAny, PyRef, PyResult}; use pyo3::types::PyFrozenSet; /// `--select` minus `--exclude` on a native graph, adding the functions selected models need. -#[inline] pub fn resolve_graph_selection<'py>( graph: PyRef<'py, NativeProjectGraph>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, ) -> PyResult> { - selectors::resolve_graph_selection(graph, select, exclude) + resolved(graph.graph(), (select, exclude)) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs index 478a51e027..b480f0018a 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs @@ -1,17 +1,33 @@ //! The keys whitespace-separated selector tokens match, without required build resources. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyFrozenSet; +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; +use crate::planner_selection::_helpers::selectors::arguments_graph; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::_helpers::selectors::selected; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyString}; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// The keys whitespace-separated selector tokens match, without required build resources. -#[inline] pub fn resolve_project_selector_tokens<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - selectors::resolve_project_selector_tokens(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selector_tokens", + options, + (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + let texts: Vec> = py_strings(&arguments.value("selectors")?)?; + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; + let selectors: Vec = carrier.send_all(&texts)?; + let graph = graph_with_jobs(py, graph.graph(), &selectors, &[])?; + selected(&carrier, resolve_selector_tokens(&graph, &selectors)) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs index f49b926693..6e8d735b6d 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs @@ -1,17 +1,27 @@ //! Model, source, seed and function selectors resolved against compiled graph mappings. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyFrozenSet; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; +use crate::planner_selection::_helpers::selectors::arguments_graph; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Model, source, seed and function selectors resolved against compiled graph mappings. -#[inline] pub fn resolve_project_selectors<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - selectors::resolve_project_selectors(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selectors", + options, + (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + resolved( + graph.graph(), + (&arguments.value("select")?, &arguments.value("exclude")?), + ) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs index 3bfa72b628..5086df8520 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs @@ -1,17 +1,42 @@ //! One SQLBuild-owned model selector core's model names, and its path selector when translated. -use crate::planner_selection::_helpers::model_selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyString; -use pyo3::types::PyTuple; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::planner_selection::_helpers::model_selectors::selectable_model; +use crate::planner_selection::models::Carrier; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use sqlbuild_analysis::planner_selection::models::{ModelSelectorNames, SelectableModel}; /// One SQLBuild-owned model selector core's model names, and its path selector when translated. -#[inline] pub fn resolve_sqlbuild_model_selector_names<'py>( project: &Bound<'py, PyAny>, term: &Bound<'py, PyString>, ) -> PyResult> { - model_selectors::resolve_sqlbuild_model_selector_names(project, term) + let py: Python<'py> = project.py(); + let models: Vec = compiled_models(project)? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { + Ok(models + .iter() + .flat_map(|model| [model.name.clone(), model.folder.clone()]) + .chain(models.iter().flat_map(|model| model.tags.clone())) + .collect()) + })?; + let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); + let translated: Option> = resolved + .translated + .as_deref() + .map(|path| carrier.back(path)) + .transpose()?; + PyTuple::new( + py, + [ + PyTuple::new(py, resolved.names)?.into_any(), + translated.into_pyobject(py)?.into_any(), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs index 375d3757e4..6bf410c6e5 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs @@ -1,17 +1,19 @@ //! `--select` minus `--exclude` on a project graph, adding the functions selected models need. -use crate::planner_selection::_helpers::selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::graph::models::CompiledProjectGraph; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyFrozenSet; /// `--select` minus `--exclude` on a project graph, adding the functions selected models need. -#[inline] pub fn select_project_graph<'py>( graph: &Bound<'py, PyAny>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, ) -> PyResult> { - selectors::select_project_graph(graph, select, exclude) + let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + resolved( + graph.get().native(graph.py()).get().graph(), + (select, exclude), + ) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs index fee54a6e46..220e84d218 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs @@ -1,22 +1,14 @@ //! Move unit-test selectors out of resource selectors, or reject them for this command. -use crate::planner_selection::_helpers::unit_tests; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Move unit-test selectors out of resource selectors, or reject them for this command. -#[inline] pub fn split_project_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, accepts_unit_tests: bool, ) -> PyResult> { - unit_tests::split_project_unit_test_selectors( - select, - exclude, - discovered_inputs, - accepts_unit_tests, - ) + split_python((select, exclude), discovered_inputs, accepts_unit_tests) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs index f33c7faeab..1180cfa180 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs @@ -1,13 +1,40 @@ //! Split `+selector+` marker syntax from the selector core without resolving the core. -use crate::planner_selection::_helpers::model_selectors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::model_selectors::shared_input_error; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyBool; use pyo3::types::PyString; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; +use sqlbuild_analysis::planner_selection::models::SelectorExpansion; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Split `+selector+` marker syntax from the selector core without resolving the core. -#[inline] +#[pyfunction] pub fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult> { - model_selectors::split_selector_expansion(raw) + let py: Python<'py> = raw.py(); + let carrier: Carrier<'py> = Carrier::single(raw)?; + let expansion: SelectorExpansion = + split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; + new_record_with( + py, + RecordId::SelectorExpansion, + &[ + ("core", carrier.back(&expansion.core)?.into_any()), + ( + "upstream", + PyBool::new(py, expansion.upstream).to_owned().into_any(), + ), + ( + "downstream", + expansion + .downstream + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + ) } diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs index 751849b1cc..c40ac1494a 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs @@ -1,16 +1,13 @@ //! Split unit-test selectors from resource selectors for a command that runs unit tests. -use crate::planner_selection::_helpers::unit_tests; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Split unit-test selectors from resource selectors for a command that runs unit tests. -#[inline] pub fn split_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - unit_tests::split_unit_test_selectors(select, exclude, discovered_inputs) + split_python((select, exclude), discovered_inputs, true) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs index cf62a44560..6f5ff146a1 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs @@ -1,29 +1,23 @@ //! Relation fixture completion over the Rust fixture planning context. +use crate::planner_strategy::main::complete_fixtures::complete_fixtures; use crate::planner_strategy::models::FixtureCompletion; use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; use crate::planner_strategy::_helpers::fixtures::fixture_context::planning_context as project_planning_context; use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; -use crate::planner_strategy::_helpers::fixtures::fixture_preparation::prepare; -use crate::planner_strategy::_helpers::fixtures::fixture_reads::required_fixture_columns; -use crate::planner_strategy::_helpers::fixtures::fixture_selection::fixture_keys_requiring_analysis; use crate::planner_strategy::_helpers::fixtures::fixture_selection::unknown_fixture_column_diagnostics; -use crate::planner_strategy::_helpers::fixtures::fixture_sql::CompletionSql; -use crate::planner_strategy::_helpers::fixtures::fixture_sql::render_completed; -use crate::planner_strategy::_helpers::fixtures::fixture_types::expected_fixture_types; -use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; -use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixtures::fixture_values::label; -use crate::planner_strategy::_helpers::fixtures::fixture_values::record; -use crate::planner_strategy::models::ExpectedTypes; -use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::{CompletionSql, render_completed}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{ + columns, diagnostic, label, record, +}; +use crate::planner_strategy::models::{ExpectedTypes, RelationFixturePlanningContext}; use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::records::types::RecordId; @@ -38,21 +32,7 @@ impl ExpectedTypes<'_> { } } -/// Complete the fixture groups of one chain against the planning context. -pub(crate) fn complete_fixtures<'py, 'a>( - context: &'a RelationFixturePlanningContext, - adapter: &Bound<'py, PyAny>, - (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), -) -> PyResult> { - let py = adapter.py(); - let prepared = prepare(fixture_groups, context, adapter)?; - let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; - let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; - let expected = expected_fixture_types(context, py, ordered_model_names)?; - complete_prepared(prepared, (context, &required), expected, adapter) -} - -fn complete_prepared<'py, 'a>( +pub(crate) fn complete_prepared<'py, 'a>( mut prepared: PreparedFixtures<'py>, (context, required): ( &'a RelationFixturePlanningContext, @@ -193,7 +173,7 @@ fn quoted_names<'py>( /// Complete one chain's fixture groups, building the planning context when none is given. #[pyfunction] #[pyo3(signature = (*, project, adapter, request, planning_context = None))] -fn build_relation_fixture_completion<'py>( +pub(crate) fn build_relation_fixture_completion<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, request: (Vec, Bound<'py, PyAny>), @@ -224,7 +204,3 @@ fn build_relation_fixture_completion<'py>( kwargs.set_item("diagnostics", PyTuple::new(py, completion.diagnostics)?)?; record(py, RecordId::RelationFixtureCompletion, &kwargs) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_relation_fixture_completion, module)?) -} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs index 6da49ca819..e4772f64ab 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs @@ -3,10 +3,10 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyfunction, pymethods}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; @@ -19,17 +19,14 @@ use crate::native_host::models::ModelHandle; use crate::planner_strategy::_helpers::fixtures::fixture_metadata::model_fixture_relation; use crate::planner_strategy::_helpers::fixtures::fixture_reads::lineage_reads; use crate::planner_strategy::_helpers::fixtures::fixture_types::projections; -use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; -use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; -use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::planner_strategy::constants::SEED_REFERENCE_KIND; -use crate::planner_strategy::constants::SOURCE_REFERENCE_KIND; -use crate::planner_strategy::models::FixtureReadCacheHost; -use crate::planner_strategy::models::FixtureRelation; -use crate::planner_strategy::models::ModelFixtureFacts; -use crate::planner_strategy::models::RelationFixturePlanningContext; -use crate::planner_strategy::types::FixtureKey; -use crate::planner_strategy::types::FixtureRelationInput; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; +use crate::planner_strategy::constants::{ + MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, +}; +use crate::planner_strategy::models::{ + FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, +}; +use crate::planner_strategy::types::{FixtureKey, FixtureRelationInput}; use crate::resource_views::main::resource_reads::name::name; use crate::resource_views::main::seed_reads::schema_entry::schema_entry; use crate::resource_views::main::source_reads::source_entry::source_entry; @@ -37,12 +34,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; static FIXTURE_READS: PyOnceLock> = PyOnceLock::new(); -/// Count one batch resolution of SQL-test model chains. -pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { - work_counts(py)?.borrow_mut().chain_resolutions += 1; - Ok(()) -} - /// The extension-wide fixture read cache and planning work counters. pub(crate) fn work_counts(py: Python<'_>) -> PyResult> { Ok(FIXTURE_READS @@ -252,7 +243,7 @@ impl RelationFixturePlanningContext { /// Build the project-wide fixture planning context once for many SQL-test plans. #[pyfunction] #[pyo3(signature = (*, project, models_by_name = None))] -fn build_relation_fixture_context( +pub(crate) fn build_relation_fixture_context( project: &Bound<'_, PyAny>, models_by_name: Option>, ) -> PyResult { @@ -262,7 +253,7 @@ fn build_relation_fixture_context( /// Planning contexts built and SQL-test chain batches resolved, optionally resetting both. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { +pub(crate) fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { let host = work_counts(py)?; let mut host = host.borrow_mut(); let counts = (host.context_builds, host.chain_resolutions); @@ -272,9 +263,3 @@ fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64 } Ok(counts) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; - module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) -} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs index e874a437e8..fd0d0e684c 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs @@ -7,8 +7,7 @@ use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; use crate::compiled_views::main::compiled_models::schema_entry::schema_entry; use crate::compiled_views::main::compiled_models::star_flags::star_flags; -use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; -use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; use crate::planner_strategy::constants::ENFORCED_CONTRACT; use crate::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs index 48a75e89cd..9e313d23bc 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs @@ -1,12 +1,11 @@ //! Inference and schema preparation at the fixture Python boundary. +use crate::planner_strategy::main::complete_empty_fixture_sql::complete_empty_fixture_sql; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; -use crate::planner_strategy::_helpers::fixtures::fixture_sql::complete_empty_fixture_sql; -use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixtures::fixture_values::label; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{diagnostic, label}; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs index 89871c2c6c..8949fddebc 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs @@ -1,8 +1,8 @@ //! Boundary conversion for native fixture read analysis. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyFrozenSet; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ @@ -178,14 +178,10 @@ fn model_stars( .map_err(value_error) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) -} - /// Report resolver misses and retained entries, optionally clearing prior fixture reuse. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { +pub(crate) fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); if reset { diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs index c62bee7701..7ab6d934ca 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs @@ -10,9 +10,7 @@ use sqlbuild_analysis::planner_fixtures::main::{ }; use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; -use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; -use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; -use crate::planner_strategy::_helpers::fixtures::fixture_values::label; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, diagnostic, label}; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs index 13cdb549a5..f2ff9d0dba 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs @@ -3,14 +3,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::planner_fixtures::main::{ - empty_fixture_error::empty_fixture_error, empty_fixture_marker::empty_fixture_marker, - render_completed::render_completed as native_render_completed, render_empty::render_empty, + empty_fixture_marker::empty_fixture_marker, + render_completed::render_completed as native_render_completed, }; use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureSqlInput, SuppliedProjection, }; -use crate::planner_strategy::models::FixtureRelation; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -18,33 +17,6 @@ pub(crate) fn fixture_marker(sql: &str) -> PyResult { empty_fixture_marker(sql).map_err(value_error) } -/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. -pub(crate) fn complete_empty_fixture_sql( - sql: &str, - relation: Option<&FixtureRelation>, - adapter: &Bound<'_, PyAny>, -) -> PyResult<(String, Option)> { - if !fixture_marker(sql)? { - return Ok((sql.into(), None)); - } - let authoritative = relation.is_some_and(|relation| relation.authoritative); - let metadata: &[FixtureColumn] = match relation { - Some(relation) if authoritative => &relation.columns, - _ => &[], - }; - if let Some(error) = empty_fixture_error(authoritative, metadata) { - return Ok((sql.into(), Some(error))); - } - Ok(( - render_empty(metadata, |name| { - adapter - .call_method1("render_identifier", (name,))? - .extract::() - })?, - None, - )) -} - pub(crate) struct CompletionSql<'py, 'a> { pub(crate) sql: &'a str, pub(crate) inferred: Bound<'py, PyAny>, @@ -95,34 +67,3 @@ pub(crate) fn render_completed( }, ) } - -/// Fixture SQL with its untyped null columns cast to the expected types. -pub(crate) fn complete_typed_null_columns( - sql: &str, - inference: &Bound<'_, PyAny>, - expected_types: &[(String, String)], - adapter: &Bound<'_, PyAny>, -) -> PyResult { - let py = inference.py(); - let typed_nulls = PyDict::new(py); - for name in native_field(inference, "null_literal_names")?.try_iter()? { - let name = name?; - let folded: String = name.extract()?; - if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { - typed_nulls.set_item(name, expected)?; - } - } - if typed_nulls.is_empty() { - return Ok(sql.into()); - } - render_completed( - CompletionSql { - sql, - inferred: native_field(inference, "columns")?, - typed_nulls, - quoted_names: native_field(inference, "quoted_names")?, - completed: Vec::new(), - }, - adapter, - ) -} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs index 7a4ffe29ce..4222c87804 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs @@ -11,10 +11,8 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; use crate::compiled_views::models::LineageFactRow; -use crate::planner_strategy::constants::DIRECT_TRANSFORM; -use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::planner_strategy::models::ExpectedTypes; -use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::constants::{DIRECT_TRANSFORM, MODEL_REFERENCE_KIND}; +use crate::planner_strategy::models::{ExpectedTypes, RelationFixturePlanningContext}; use crate::planner_strategy::types::FixtureKey; /// Each chain model's known column types with direct lineage types propagated from its inputs. diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs index 406816e11c..fe7dffadd8 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs @@ -1,17 +1,36 @@ //! An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. -use crate::planner_strategy::_helpers::fixtures::fixture_sql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::fixture_marker; use crate::planner_strategy::models::FixtureRelation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_fixtures::main::{ + empty_fixture_error::empty_fixture_error, render_empty::render_empty, +}; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; /// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. -#[inline] pub fn complete_empty_fixture_sql( sql: &str, relation: Option<&FixtureRelation>, adapter: &Bound<'_, PyAny>, ) -> PyResult<(String, Option)> { - fixture_sql::complete_empty_fixture_sql(sql, relation, adapter) + if !fixture_marker(sql)? { + return Ok((sql.into(), None)); + } + let authoritative = relation.is_some_and(|relation| relation.authoritative); + let metadata: &[FixtureColumn] = match relation { + Some(relation) if authoritative => &relation.columns, + _ => &[], + }; + if let Some(error) = empty_fixture_error(authoritative, metadata) { + return Ok((sql.into(), Some(error))); + } + Ok(( + render_empty(metadata, |name| { + adapter + .call_method1("render_identifier", (name,))? + .extract::() + })?, + None, + )) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs index fd96d34ab4..2edf3b0b59 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs @@ -1,18 +1,24 @@ //! Complete the fixture groups of one chain against the planning context. -use crate::planner_strategy::_helpers::fixtures::fixture_completion; +use crate::planner_strategy::_helpers::fixtures::fixture_completion::complete_prepared; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::prepare; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::required_fixture_columns; +use crate::planner_strategy::_helpers::fixtures::fixture_selection::fixture_keys_requiring_analysis; +use crate::planner_strategy::_helpers::fixtures::fixture_types::expected_fixture_types; use crate::planner_strategy::models::FixtureCompletion; use crate::planner_strategy::models::RelationFixturePlanningContext; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Complete the fixture groups of one chain against the planning context. -#[inline] pub fn complete_fixtures<'py, 'a>( context: &'a RelationFixturePlanningContext, adapter: &Bound<'py, PyAny>, (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), ) -> PyResult> { - fixture_completion::complete_fixtures(context, adapter, (ordered_model_names, fixture_groups)) + let py = adapter.py(); + let prepared = prepare(fixture_groups, context, adapter)?; + let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; + let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; + let expected = expected_fixture_types(context, py, ordered_model_names)?; + complete_prepared(prepared, (context, &required), expected, adapter) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs index 6d24d3d769..99860782a3 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs @@ -1,17 +1,38 @@ //! Fixture SQL with its untyped null columns cast to the expected types. -use crate::planner_strategy::_helpers::fixtures::fixture_sql; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::CompletionSql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::render_completed; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Fixture SQL with its untyped null columns cast to the expected types. -#[inline] pub fn complete_typed_null_columns( sql: &str, inference: &Bound<'_, PyAny>, expected_types: &[(String, String)], adapter: &Bound<'_, PyAny>, ) -> PyResult { - fixture_sql::complete_typed_null_columns(sql, inference, expected_types, adapter) + let py = inference.py(); + let typed_nulls = PyDict::new(py); + for name in native_field(inference, "null_literal_names")?.try_iter()? { + let name = name?; + let folded: String = name.extract()?; + if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { + typed_nulls.set_item(name, expected)?; + } + } + if typed_nulls.is_empty() { + return Ok(sql.into()); + } + render_completed( + CompletionSql { + sql, + inferred: native_field(inference, "columns")?, + typed_nulls, + quoted_names: native_field(inference, "quoted_names")?, + completed: Vec::new(), + }, + adapter, + ) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs index cf884a46fd..d743a78161 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs @@ -1,11 +1,10 @@ //! Count one batch resolution of SQL-test model chains. -use crate::planner_strategy::_helpers::fixtures::fixture_context; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_strategy::_helpers::fixtures::fixture_context::work_counts; +use pyo3::prelude::{PyResult, Python}; /// Count one batch resolution of SQL-test model chains. -#[inline] pub fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { - fixture_context::count_chain_resolution(py) + work_counts(py)?.borrow_mut().chain_resolutions += 1; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs index 75b022ec9a..c9861c24fb 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs @@ -2,9 +2,7 @@ use crate::planner_strategy::_helpers::fixtures::fixture_context; use crate::planner_strategy::models::RelationFixturePlanningContext; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// The planning context of a project, over its models or the given name-to-model map. diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs index f60e68fdaf..d1a13f6ec3 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs @@ -1,12 +1,10 @@ //! Register the fixture completion functions and classes. -use crate::planner_strategy::_helpers::fixtures::fixture_completion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::fixtures::fixture_completion::build_relation_fixture_completion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fixture completion functions and classes. -#[inline] pub fn register_fixture_completion(module: &Bound<'_, PyModule>) -> PyResult<()> { - fixture_completion::register(module) + module.add_function(wrap_pyfunction!(build_relation_fixture_completion, module)?) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs index cb61136494..6107ba4a68 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs @@ -1,12 +1,14 @@ //! Register the fixture context functions and classes. -use crate::planner_strategy::_helpers::fixtures::fixture_context; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::fixtures::fixture_context::build_relation_fixture_context; +use crate::planner_strategy::_helpers::fixtures::fixture_context::fixture_planning_work_info; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fixture context functions and classes. -#[inline] pub fn register_fixture_context(module: &Bound<'_, PyModule>) -> PyResult<()> { - fixture_context::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; + module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs index 3328d77af0..d57bb7ebd9 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs @@ -1,12 +1,10 @@ //! Register the fixture reads functions and classes. -use crate::planner_strategy::_helpers::fixtures::fixture_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::fixture_read_cache_info; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fixture reads functions and classes. -#[inline] pub fn register_fixture_reads(module: &Bound<'_, PyModule>) -> PyResult<()> { - fixture_reads::register(module) + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/models.rs b/crates/sqlbuild-python-project/src/planner_strategy/models.rs index 8bd1580a41..7895145136 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/models.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/models.rs @@ -4,18 +4,15 @@ use crate::graph::models::NativeProjectGraph; use crate::native_host::models::ModelHandle; use crate::native_host::models::ModelLookup; use crate::planner_strategy::types::FixtureKey; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; +use pyo3::prelude::{Bound, Py}; use pyo3::pyclass; use pyo3::types::PyDict; -use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; -use sqlbuild_analysis::planner_fixtures::models::FixtureTypeProjection; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureReadCache, FixtureTypeProjection, +}; use sqlbuild_analysis::required_columns::models::ColumnRead; -use std::collections::BTreeSet; -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::Mutex; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; /// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. #[derive(Debug)] diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs index 050e2e0a07..d1491c4fe9 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs @@ -1,15 +1,12 @@ //! Project metadata stays native until its immutable snapshot is sent over HTTP. -use crate::project_snapshot::models::NativeOrchestrationSnapshot; use std::collections::BTreeMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::PyResult; use serde_json::Value; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::project_snapshot::models::NativeOrchestrationWorker; - /// Duplicate resource identities must fail, rather than silently replacing an earlier node. pub(crate) fn insert_node( mut nodes: BTreeMap, @@ -41,8 +38,3 @@ pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> "logical_schema": value.logical_schema, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs index 28838025f2..36c1166f24 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs @@ -1,5 +1,6 @@ //! Discovery files and authored test payloads remain the Python host's boundary. +use crate::project_snapshot::main::relative_path::relative_path; use std::collections::BTreeMap; use std::fs::File; use std::io::Read; @@ -15,12 +16,10 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot::destination; -use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; +use crate::project_snapshot::_helpers::snapshot::snapshot::{destination, insert_node}; use crate::resource_views::models::NativeCompiledSeed; use crate::resource_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::main::fields::path_posix::path_posix; /// Read only declared metadata fields, never callable attributes or connection settings. pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { @@ -31,10 +30,6 @@ pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult) -> PyResult { - path_posix(&field(value, "relative_path")?) -} - /// Source locations are local compiler diagnostics, not relocatable column metadata. pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { value diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index 5148b77f30..a628edd529 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -11,7 +11,6 @@ use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::orchestration_files::models::{ FilePosition, InitialStatus, NamedModule, OrchestrationInventory, RunFile, ScheduleSource, }; -use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ @@ -25,7 +24,6 @@ use crate::native_host::main::compile_calendars::compile_calendars; use crate::native_host::main::compile_sensors::compile_sensors; use crate::native_host::models::CalendarRequest; use sqlbuild_python_base::orchestration_reads::main::imported_declarations::imported_declarations; -use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; use sqlbuild_python_base::orchestration_reads::main::run_controls::run_controls; use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; use sqlbuild_python_base::orchestration_reads::models::ImportedDeclarations; @@ -33,25 +31,6 @@ use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_ use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. -pub(crate) fn python_declarations<'py>( - py: Python<'py>, - project_dir: &Path, - inventory: &OrchestrationInventory, -) -> PyResult>> { - let tree = ProjectTree::new(project_dir); - let mut declarations: Vec> = Vec::new(); - let mut seen: BTreeSet = BTreeSet::new(); - for module in inventory.calendars.iter().chain(&inventory.sensors) { - for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { - if seen.insert(declaration.as_ptr() as usize) { - declarations.push(declaration); - } - } - } - Ok(declarations) -} - /// Where a TOML job or group is declared: its file and the line of its `name`. fn file_site(path: &str, position: FilePosition) -> Option { Some(DeclaredAt { diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs index 8fdb23e755..6f70a31c5f 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs @@ -17,9 +17,9 @@ use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeProjectView; use crate::compiled_views::types::ModelInputRecord; -use crate::project_snapshot::_helpers::snapshot::snapshot::dependency_keys; -use crate::project_snapshot::_helpers::snapshot::snapshot::destination; -use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; +use crate::project_snapshot::_helpers::snapshot::snapshot::{ + dependency_keys, destination, insert_node, +}; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs index 7272d0416f..eb9baf9ece 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs @@ -1,5 +1,6 @@ //! Sources, functions and audits are read from their retained native records. +use crate::project_snapshot::main::relative_path::relative_path; use std::collections::BTreeMap; use pyo3::prelude::{PyAnyMethods, PyResult, Python}; @@ -10,13 +11,12 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot::dependency_keys; -use crate::project_snapshot::_helpers::snapshot::snapshot::destination; -use crate::project_snapshot::_helpers::snapshot::snapshot::insert_node; -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::boundary_nodes; -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::columns; -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::metadata; -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::relative_path; +use crate::project_snapshot::_helpers::snapshot::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::{ + boundary_nodes, columns, metadata, +}; use crate::resource_views::models::NativeCompiledFunction; pub(crate) fn resource_nodes( diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs index 0ffea4986f..79abee250c 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs @@ -1,43 +1,4 @@ //! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. -use std::path::Path; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::graph::models::ProjectGraph; -use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; - -use crate::graph::models::ProjectSelection; -use sqlbuild_python_base::native_host::main::write_console::write_console; - /// The project-root file that defines project roles. pub(crate) const ROLES_FILE: &str = "roles.toml"; - -pub(crate) fn compile_project_roles( - py: Python<'_>, - mut input: SnapshotInput, - (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), -) -> PyResult { - let path = project_dir.join(ROLES_FILE); - if !path.is_file() { - return Ok(input); - } - if input.documents.contains_key("roles") { - return Err(PyValueError::new_err( - "roles.toml cannot overwrite a snapshot roles document", - )); - } - let text = std::fs::read_to_string(&path) - .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; - let (document, warnings) = compile_roles(&text, |selector| { - let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); - selected.node_keys(graph, &tokens, &[]) - }) - .map_err(PyValueError::new_err)?; - for warning in warnings { - write_console(py, "stderr", &format!("Warning: {warning}\n"))?; - } - input.documents.insert("roles".into(), document); - Ok(input) -} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs index 3b8a69038c..de4e6710ad 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs @@ -2,10 +2,7 @@ use crate::graph::models::ProjectSelection; use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_orchestration::snapshots::models::SnapshotInput; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs index 753d6dc24e..64f3cbcaa2 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs @@ -1,19 +1,40 @@ //! Compile project roles. use crate::graph::models::ProjectSelection; -use crate::project_snapshot::_helpers::snapshot::snapshot_roles; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::project_snapshot::_helpers::snapshot::snapshot_roles::ROLES_FILE; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_python_base::native_host::main::write_console::write_console; use std::path::Path; /// Compile project roles. -#[inline] pub fn compile_project_roles( py: Python<'_>, - input: SnapshotInput, + mut input: SnapshotInput, (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), ) -> PyResult { - snapshot_roles::compile_project_roles(py, input, (project_dir, graph, selected)) + let path = project_dir.join(ROLES_FILE); + if !path.is_file() { + return Ok(input); + } + if input.documents.contains_key("roles") { + return Err(PyValueError::new_err( + "roles.toml cannot overwrite a snapshot roles document", + )); + } + let text = std::fs::read_to_string(&path) + .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; + let (document, warnings) = compile_roles(&text, |selector| { + let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); + selected.node_keys(graph, &tokens, &[]) + }) + .map_err(PyValueError::new_err)?; + for warning in warnings { + write_console(py, "stderr", &format!("Warning: {warning}\n"))?; + } + input.documents.insert("roles".into(), document); + Ok(input) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs index b911aeae8b..45583b6cfe 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs @@ -3,10 +3,8 @@ use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; use crate::project_snapshot::_helpers::snapshot::snapshot_models; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use serde_json::Map; -use serde_json::Value; +use pyo3::prelude::{PyResult, Python}; +use serde_json::{Map, Value}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs index bde5be109c..468fc93243 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs @@ -1,19 +1,27 @@ //! The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. -use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; +use std::collections::BTreeSet; use std::path::Path; /// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. -#[inline] pub fn python_declarations<'py>( py: Python<'py>, project_dir: &Path, inventory: &OrchestrationInventory, ) -> PyResult>> { - snapshot_declarations::python_declarations(py, project_dir, inventory) + let tree = ProjectTree::new(project_dir); + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for module in inventory.calendars.iter().chain(&inventory.sensors) { + for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { + if seen.insert(declaration.as_ptr() as usize) { + declarations.push(declaration); + } + } + } + Ok(declarations) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs index 2517ab3690..c699f58b14 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs @@ -1,12 +1,11 @@ //! Register the snapshot functions and classes. -use crate::project_snapshot::_helpers::snapshot::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::project_snapshot::models::NativeOrchestrationSnapshot; +use crate::project_snapshot::models::NativeOrchestrationWorker; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the snapshot functions and classes. -#[inline] pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { - snapshot::register(module) + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs index a0671cf6aa..8ca9b45a52 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs @@ -1,12 +1,10 @@ //! Relative path. -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::path_posix::path_posix; /// Relative path. -#[inline] pub fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { - snapshot_boundary::relative_path(value) + path_posix(&field(value, "relative_path")?) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs index 84701e6305..e14e160f77 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs @@ -3,8 +3,7 @@ use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; use crate::project_snapshot::_helpers::snapshot::snapshot_resources; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index 3ee0f3a367..d289897091 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -2,22 +2,16 @@ use crate::references::models::GateRow; use crate::references::models::HookRow; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use std::collections::{BTreeMap, BTreeSet}; use pyo3::exceptions::{PyOSError, PyTypeError}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PySet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; -use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; -use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; -use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; use sqlbuild_analysis::explicit_references::models::{ - ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, LiteralSql, - ReferenceReport, ReferenceRequest, SqlOwner, + GateEdge, HookEdge, HookType, LiteralSql, ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; @@ -27,8 +21,7 @@ use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; -use crate::references::_helpers::reference_facts::ProjectFacts; -use crate::references::_helpers::reference_facts::project_facts; +use crate::references::_helpers::reference_facts::{ProjectFacts, project_facts}; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -37,7 +30,6 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; @@ -396,57 +388,7 @@ fn python_literal_sql_calls( .collect() } -/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. -#[pyfunction] -#[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] -pub(crate) fn hard_coded_relation_remedy( - owner_kind: &str, - r#ref: &Bound<'_, PyAny>, - upstream_loader_by_source: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let target: ResourceKey = ( - native_field(r#ref, "kind")?.getattr("value")?.extract()?, - native_field(r#ref, "name")?.extract()?, - ); - let mut upstream_loaders: Vec<(String, String)> = Vec::new(); - if let Some(loaders) = upstream_loader_by_source.filter(|loaders| !loaders.is_none()) { - for item in mapping_items(loaders)?.try_iter()? { - upstream_loaders.push(item?.extract()?); - } - } - Ok(relation_remedy(owner_kind, &target, &upstream_loaders)) -} - -/// The P008 message for SQL naming a relation kept only as a compatibility view. -#[pyfunction] -#[pyo3(signature = (*, owner_label, written, method, model_name))] -pub(crate) fn old_name_reference_message( - owner_label: &str, - written: &str, - method: &str, - model_name: &str, -) -> String { - old_name_message(owner_label, written, method, model_name) -} - -/// The attached-audit gate edges and hook-read edges of one project, in Python's order. -pub(crate) fn native_execution_gate_edges( - project: &Bound<'_, PyAny>, -) -> PyResult<(Vec, Vec)> { - let facts: ProjectFacts = project_facts(project, false)?; - let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { - resources: facts.resources, - models: facts.models, - audits: facts.audits, - hooks: facts.hooks, - }); - Ok(( - edges.audit_gates.into_iter().map(gate_row).collect(), - edges.hook_reads.into_iter().map(hook_row).collect(), - )) -} - -fn gate_row(edge: GateEdge) -> GateRow { +pub(crate) fn gate_row(edge: GateEdge) -> GateRow { GateRow { audit_name: edge.audit_name, target: edge.target, @@ -455,7 +397,7 @@ fn gate_row(edge: GateEdge) -> GateRow { } } -fn hook_row(edge: HookEdge) -> HookRow { +pub(crate) fn hook_row(edge: HookEdge) -> HookRow { let hook_type: &'static str = match edge.hook_type { HookType::Python => "Python", HookType::Sql => "SQL", @@ -468,10 +410,3 @@ fn hook_row(edge: HookEdge) -> HookRow { hook_type, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; - module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; - module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/references/_helpers/relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs index ed7fc1f63b..16a3174444 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/relations.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs @@ -1,15 +1,14 @@ //! Relation names in SQL and the project relations they match, for hard-coded name checks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::ExecutorHost; use crate::analysis::main::extract_sql_relation_names::extract_sql_relation_names; use crate::analysis::types::RelationRow; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; const RUNTIME_RELATION_PREFIX: &str = "SELECT 1 FROM "; @@ -39,7 +38,7 @@ fn relation_name<'py>( /// Relations named in `sql` and temporary tables it creates, or None if unparseable. #[pyfunction] #[pyo3(signature = (*, sql, dialect))] -fn extract_relation_names<'py>( +pub(crate) fn extract_relation_names<'py>( py: Python<'py>, sql: &str, dialect: Option<&str>, @@ -61,50 +60,17 @@ fn extract_relation_names<'py>( } /// Whether two optional qualifier parts can name the same relation. -fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { +pub(crate) fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { match (left, right) { (Some(left), Some(right)) => python_casefold(left) == python_casefold(right), _ => true, } } -/// The project relation `relation` names, resolving missing qualifiers from the defaults. -#[pyfunction] -#[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] -pub(crate) fn match_project_relation<'py>( - index: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, - default_database: Option, - default_schema: Option, -) -> PyResult>> { - let name: String = python_casefold(&native_field(relation, "name")?.extract::()?); - let schema: Option = native_field(relation, "schema")? - .extract::>()? - .or(default_schema); - let database: Option = native_field(relation, "database")? - .extract::>()? - .or(default_database); - for candidate in native_field(index, "relations")?.try_iter()? { - let candidate: Bound<'py, PyAny> = candidate?; - let target: Bound<'py, PyAny> = native_field(&candidate, "relation")?; - if python_casefold(&native_field(&target, "name")?.extract::()?) != name { - continue; - } - let target_schema: Option = native_field(&target, "schema")?.extract()?; - let target_database: Option = native_field(&target, "database")?.extract()?; - if parts_compatible(schema.as_deref(), target_schema.as_deref()) - && parts_compatible(database.as_deref(), target_database.as_deref()) - { - return Ok(Some(candidate)); - } - } - Ok(None) -} - /// Project relations from adapter-rendered runtime relation names, deduplicated in order. #[pyfunction] #[pyo3(signature = (*, relations, dialect))] -fn runtime_project_relations<'py>( +pub(crate) fn runtime_project_relations<'py>( py: Python<'py>, relations: &Bound<'py, PyAny>, dialect: Option<&str>, @@ -134,10 +100,3 @@ fn runtime_project_relations<'py>( &[("relations", PyTuple::new(py, resolved.keys())?.into_any())], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(extract_relation_names, module)?)?; - module.add_function(wrap_pyfunction!(match_project_relation, module)?)?; - module.add_function(wrap_pyfunction!(runtime_project_relations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs index 37d3d0d621..0e2fe20777 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs @@ -2,17 +2,13 @@ use sqlbuild_host::entries::models::AdapterHost; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyErr, Python}; +use pyo3::types::PyString; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The SQL analysis dialect whose engines read replacing tables with `FINAL`. -const FINAL_READ_DIALECT: &str = "clickhouse"; +pub(crate) const FINAL_READ_DIALECT: &str = "clickhouse"; /// Error code for a `final` source on another adapter. const FINAL_UNSUPPORTED_CODE: &str = "A010"; /// Guidance for a `final` source on another adapter. @@ -23,7 +19,7 @@ const FINAL_UNSUPPORTED_HELP: &str = const QUERY_PREFIXES: [&str; 3] = ["select", "with", "values"]; /// A source expression as a table factor: queries parenthesized, relations as written. -fn expression_relation(expression: &str) -> String { +pub(crate) fn expression_relation(expression: &str) -> String { let trimmed: &str = expression.trim(); let stripped: &str = trimmed.strip_suffix(';').unwrap_or(trimmed).trim(); if stripped.starts_with('(') { @@ -39,74 +35,8 @@ fn expression_relation(expression: &str) -> String { stripped.to_owned() } -/// Render a source as a SQL table factor, through the adapter when one is given. -#[pyfunction] -#[pyo3(signature = (*, entry, adapter = None))] -pub(crate) fn render_source_relation<'py>( - entry: &Bound<'py, PyAny>, - adapter: Option<&Bound<'py, PyAny>>, -) -> PyResult { - let adapter: Option<&Bound<'py, PyAny>> = adapter.filter(|adapter| !adapter.is_none()); - let expression: Option = native_field(entry, "expression")?.extract()?; - if let Some(expression) = expression { - return match adapter { - Some(adapter) => { - let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); - keywords.set_item("expression", expression)?; - adapter - .call_method("render_source_expression_relation", (), Some(&keywords))? - .extract() - } - None => Ok(expression_relation(&expression)), - }; - } - let table: Option = native_field(entry, "table")?.extract()?; - let table: String = match table { - Some(table) => table, - None => native_field(entry, "name")?.extract()?, - }; - let database: Option = native_field(entry, "database")?.extract()?; - let schema: Option = native_field(entry, "schema")?.extract()?; - if let Some(adapter) = adapter { - let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); - keywords.set_item("database", &database)?; - keywords.set_item("schema", &schema)?; - keywords.set_item("name", &table)?; - let rendered: Option = adapter - .call_method("render_qualified_name", (), Some(&keywords))? - .extract()?; - if let Some(rendered) = rendered { - return Ok(rendered); - } - } - Ok([database, schema, Some(table)] - .into_iter() - .flatten() - .collect::>() - .join(".")) -} - -/// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. -pub(crate) fn render_source_read<'py>( - entry: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, -) -> PyResult { - let relation: String = render_source_relation(entry, Some(adapter))?; - if !native_field(entry, "final")?.is_truthy()? { - return Ok(relation); - } - let dialect: Option = native_field(adapter, "sql_analysis_dialect_name")?.extract()?; - if dialect.as_deref() != Some(FINAL_READ_DIALECT) { - return Err(final_unsupported( - entry.py(), - &native_field(adapter, "adapter_name")?.str()?.to_string(), - )); - } - Ok(format!("(SELECT * FROM {relation} FINAL)")) -} - /// The coded error for a `final` source on an adapter that cannot read with `FINAL`. -fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { +pub(crate) fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { match host_entry(py, AdapterHost::AdapterUserError).and_then(|class| { call_with( &class, @@ -130,8 +60,3 @@ fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { Err(error) => error, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_source_relation, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs index e63e34c0ab..13df509e75 100644 --- a/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs +++ b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs @@ -1,10 +1,7 @@ //! Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. use crate::references::_helpers::explicit_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. #[inline] diff --git a/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs index 8f4bb62207..ea9eef60f1 100644 --- a/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs +++ b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs @@ -1,16 +1,29 @@ //! How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. -use crate::references::_helpers::explicit_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; +use sqlbuild_analysis::explicit_references::types::ResourceKey; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] pub fn hard_coded_relation_remedy( owner_kind: &str, r#ref: &Bound<'_, PyAny>, upstream_loader_by_source: Option<&Bound<'_, PyAny>>, ) -> PyResult { - explicit_references::hard_coded_relation_remedy(owner_kind, r#ref, upstream_loader_by_source) + let target: ResourceKey = ( + native_field(r#ref, "kind")?.getattr("value")?.extract()?, + native_field(r#ref, "name")?.extract()?, + ); + let mut upstream_loaders: Vec<(String, String)> = Vec::new(); + if let Some(loaders) = upstream_loader_by_source.filter(|loaders| !loaders.is_none()) { + for item in mapping_items(loaders)?.try_iter()? { + upstream_loaders.push(item?.extract()?); + } + } + Ok(relation_remedy(owner_kind, &target, &upstream_loaders)) } diff --git a/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs index 3c3d8277c5..d73d4e3b92 100644 --- a/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs +++ b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs @@ -1,17 +1,40 @@ //! The project relation `relation` names, resolving missing qualifiers from the defaults. -use crate::references::_helpers::relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::references::_helpers::relations::parts_compatible; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project relation `relation` names, resolving missing qualifiers from the defaults. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] pub fn match_project_relation<'py>( index: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, default_database: Option, default_schema: Option, ) -> PyResult>> { - relations::match_project_relation(index, relation, default_database, default_schema) + let name: String = python_casefold(&native_field(relation, "name")?.extract::()?); + let schema: Option = native_field(relation, "schema")? + .extract::>()? + .or(default_schema); + let database: Option = native_field(relation, "database")? + .extract::>()? + .or(default_database); + for candidate in native_field(index, "relations")?.try_iter()? { + let candidate: Bound<'py, PyAny> = candidate?; + let target: Bound<'py, PyAny> = native_field(&candidate, "relation")?; + if python_casefold(&native_field(&target, "name")?.extract::()?) != name { + continue; + } + let target_schema: Option = native_field(&target, "schema")?.extract()?; + let target_database: Option = native_field(&target, "database")?.extract()?; + if parts_compatible(schema.as_deref(), target_schema.as_deref()) + && parts_compatible(database.as_deref(), target_database.as_deref()) + { + return Ok(Some(candidate)); + } + } + Ok(None) } diff --git a/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs index 21f552da41..d3e025b284 100644 --- a/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs +++ b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs @@ -1,16 +1,27 @@ //! The attached-audit gate edges and hook-read edges of one project, in Python's order. -use crate::references::_helpers::explicit_references; +use crate::references::_helpers::explicit_references::gate_row; +use crate::references::_helpers::explicit_references::hook_row; +use crate::references::_helpers::reference_facts::{ProjectFacts, project_facts}; use crate::references::models::GateRow; use crate::references::models::HookRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::models::{ExecutionGateEdges, ExecutionGateRequest}; /// The attached-audit gate edges and hook-read edges of one project, in Python's order. -#[inline] pub fn native_execution_gate_edges( project: &Bound<'_, PyAny>, ) -> PyResult<(Vec, Vec)> { - explicit_references::native_execution_gate_edges(project) + let facts: ProjectFacts = project_facts(project, false)?; + let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { + resources: facts.resources, + models: facts.models, + audits: facts.audits, + hooks: facts.hooks, + }); + Ok(( + edges.audit_gates.into_iter().map(gate_row).collect(), + edges.hook_reads.into_iter().map(hook_row).collect(), + )) } diff --git a/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs index 196476fd04..b95e474fd5 100644 --- a/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs +++ b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs @@ -1,14 +1,16 @@ //! The P008 message for SQL naming a relation kept only as a compatibility view. -use crate::references::_helpers::explicit_references; +use pyo3::pyfunction; +use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; /// The P008 message for SQL naming a relation kept only as a compatibility view. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, owner_label, written, method, model_name))] pub fn old_name_reference_message( owner_label: &str, written: &str, method: &str, model_name: &str, ) -> String { - explicit_references::old_name_reference_message(owner_label, written, method, model_name) + old_name_message(owner_label, written, method, model_name) } diff --git a/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs index e880d11aa1..429725cf14 100644 --- a/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs @@ -1,12 +1,15 @@ //! Register the explicit references functions and classes. -use crate::references::_helpers::explicit_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::references::_helpers::explicit_references::explicit_reference_report; +use crate::references::main::hard_coded_relation_remedy::hard_coded_relation_remedy; +use crate::references::main::old_name_reference_message::old_name_reference_message; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the explicit references functions and classes. -#[inline] pub fn register_explicit_references(module: &Bound<'_, PyModule>) -> PyResult<()> { - explicit_references::register(module) + module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; + module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; + module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/references/main/register_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_relations.rs index 97bfedcc25..ec601f0f32 100644 --- a/crates/sqlbuild-python-project/src/references/main/register_relations.rs +++ b/crates/sqlbuild-python-project/src/references/main/register_relations.rs @@ -1,12 +1,15 @@ //! Register the relations functions and classes. -use crate::references::_helpers::relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::references::_helpers::relations::extract_relation_names; +use crate::references::_helpers::relations::runtime_project_relations; +use crate::references::main::match_project_relation::match_project_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the relations functions and classes. -#[inline] pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { - relations::register(module) + module.add_function(wrap_pyfunction!(extract_relation_names, module)?)?; + module.add_function(wrap_pyfunction!(match_project_relation, module)?)?; + module.add_function(wrap_pyfunction!(runtime_project_relations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs index a287a54c5a..8b7aeb064b 100644 --- a/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs +++ b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs @@ -1,12 +1,11 @@ //! Register the source relations functions and classes. -use crate::references::_helpers::source_relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the source relations functions and classes. -#[inline] pub fn register_source_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { - source_relations::register(module) + module.add_function(wrap_pyfunction!(render_source_relation, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_read.rs b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs index 166eebbde5..a3269c5408 100644 --- a/crates/sqlbuild-python-project/src/references/main/render_source_read.rs +++ b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs @@ -1,15 +1,26 @@ //! A source as the table factor that reads it, deduplicated with `FINAL` when `final`. -use crate::references::_helpers::source_relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::references::_helpers::source_relations::FINAL_READ_DIALECT; +use crate::references::_helpers::source_relations::final_unsupported; +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. -#[inline] pub fn render_source_read<'py>( entry: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, ) -> PyResult { - source_relations::render_source_read(entry, adapter) + let relation: String = render_source_relation(entry, Some(adapter))?; + if !native_field(entry, "final")?.is_truthy()? { + return Ok(relation); + } + let dialect: Option = native_field(adapter, "sql_analysis_dialect_name")?.extract()?; + if dialect.as_deref() != Some(FINAL_READ_DIALECT) { + return Err(final_unsupported( + entry.py(), + &native_field(adapter, "adapter_name")?.str()?.to_string(), + )); + } + Ok(format!("(SELECT * FROM {relation} FINAL)")) } diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs index 98103c0c3f..2c3dee7663 100644 --- a/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs +++ b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs @@ -1,15 +1,54 @@ //! Render a source as a SQL table factor, through the adapter when one is given. -use crate::references::_helpers::source_relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::references::_helpers::source_relations::expression_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Render a source as a SQL table factor, through the adapter when one is given. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, entry, adapter = None))] pub fn render_source_relation<'py>( entry: &Bound<'py, PyAny>, adapter: Option<&Bound<'py, PyAny>>, ) -> PyResult { - source_relations::render_source_relation(entry, adapter) + let adapter: Option<&Bound<'py, PyAny>> = adapter.filter(|adapter| !adapter.is_none()); + let expression: Option = native_field(entry, "expression")?.extract()?; + if let Some(expression) = expression { + return match adapter { + Some(adapter) => { + let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); + keywords.set_item("expression", expression)?; + adapter + .call_method("render_source_expression_relation", (), Some(&keywords))? + .extract() + } + None => Ok(expression_relation(&expression)), + }; + } + let table: Option = native_field(entry, "table")?.extract()?; + let table: String = match table { + Some(table) => table, + None => native_field(entry, "name")?.extract()?, + }; + let database: Option = native_field(entry, "database")?.extract()?; + let schema: Option = native_field(entry, "schema")?.extract()?; + if let Some(adapter) = adapter { + let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); + keywords.set_item("database", &database)?; + keywords.set_item("schema", &schema)?; + keywords.set_item("name", &table)?; + let rendered: Option = adapter + .call_method("render_qualified_name", (), Some(&keywords))? + .extract()?; + if let Some(rendered) = rendered { + return Ok(rendered); + } + } + Ok([database, schema, Some(table)] + .into_iter() + .flatten() + .collect::>() + .join(".")) } diff --git a/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs index 97f7cf9351..1fd704bbf6 100644 --- a/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs +++ b/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs @@ -1,23 +1,18 @@ //! Generated relation names fitted within identifier limits, raising `SharedInputError`. -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use crate::relation_naming::main::fitted_logical_name::fitted_logical_name; +use crate::relation_naming::main::shared_error::shared_error; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; -use sqlbuild_host::entries::models::FacadeHost; -use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::boundary::models::Request; -pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { - host_error(py, FacadeHost::SharedInputError, &message) -} - /// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. #[pyfunction] #[pyo3(signature = (**options))] -fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("fit_artifact_logical_name", options)?; fitted_logical_name( request.py(), @@ -30,21 +25,10 @@ fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult, - names: (&str, &str), - identifier_limit: i64, - artifact_label: &str, -) -> PyResult { - fit_logical(names, identifier_limit, artifact_label) - .map_err(|message| shared_error(py, message)) -} - /// `base_name + suffix` fitted within the limit, keeping the suffix and a base hash. #[pyfunction] #[pyo3(signature = (*, base_name, suffix, identifier_limit))] -fn fit_auxiliary_relation_name( +pub(crate) fn fit_auxiliary_relation_name( py: Python<'_>, base_name: &str, suffix: &str, @@ -52,9 +36,3 @@ fn fit_auxiliary_relation_name( ) -> PyResult { fit_auxiliary(base_name, suffix, identifier_limit).map_err(|message| shared_error(py, message)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; - module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs index 0373160453..cd82d91552 100644 --- a/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs +++ b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs @@ -1,16 +1,16 @@ //! `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. -use crate::relation_naming::_helpers::fitting; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::relation_naming::main::shared_error::shared_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; /// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. -#[inline] pub fn fitted_logical_name( py: Python<'_>, names: (&str, &str), identifier_limit: i64, artifact_label: &str, ) -> PyResult { - fitting::fitted_logical_name(py, names, identifier_limit, artifact_label) + fit_logical(names, identifier_limit, artifact_label) + .map_err(|message| shared_error(py, message)) } diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs index f029d8a2a7..3660c8579c 100644 --- a/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs +++ b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs @@ -1,12 +1,13 @@ //! Register the fitting functions and classes. -use crate::relation_naming::_helpers::fitting; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::relation_naming::_helpers::fitting::fit_artifact_logical_name; +use crate::relation_naming::_helpers::fitting::fit_auxiliary_relation_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fitting functions and classes. -#[inline] pub fn register_fitting(module: &Bound<'_, PyModule>) -> PyResult<()> { - fitting::register(module) + module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; + module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs index a2c7f5b483..ac8d6d4c04 100644 --- a/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs +++ b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs @@ -1,11 +1,10 @@ //! Shared error. -use crate::relation_naming::_helpers::fitting; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; /// Shared error. -#[inline] pub fn shared_error(py: Python<'_>, message: String) -> PyErr { - fitting::shared_error(py, message) + host_error(py, FacadeHost::SharedInputError, &message) } diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs index 454c3aa050..ba9a66a5db 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs @@ -1,233 +1,19 @@ //! A compiled audit's attributes the planner reads, from its native view. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use crate::resource_views::models::NativeCompiledAudit; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind an audit object; any other object is a Python-built record, counted. #[inline] -fn native_audit<'a>(audit: &'a Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn native_audit<'a>( + audit: &'a Bound<'_, PyAny>, +) -> PyResult> { if let Ok(view) = audit.cast::() { return Ok(Some(view.get())); } count_stage(audit.py(), COMPILED_VIEW_FALLBACKS, 1)?; Ok(None) } - -/// The compiled audit's `always_run`, as its Python attribute reads it. -pub(crate) fn always_run<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .always_run(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "always_run") -} - -/// The compiled audit's `attached_column_name`, as its Python attribute reads it. -pub(crate) fn attached_column_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_column_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_column_name") -} - -/// The compiled audit's `attached_target_kind`, as its Python attribute reads it. -pub(crate) fn attached_target_kind<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_target_kind(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_target_kind") -} - -/// The compiled audit's `attached_target_name`, as its Python attribute reads it. -pub(crate) fn attached_target_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_target_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_target_name") -} - -/// The compiled audit's `definition_name`, as its Python attribute reads it. -pub(crate) fn definition_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .definition_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "definition_name") -} - -/// The compiled audit's `description`, as its Python attribute reads it. -pub(crate) fn description<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .description(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "description") -} - -/// The compiled audit's `evaluation_mode`, as its Python attribute reads it. -pub(crate) fn evaluation_mode<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evaluation_mode(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evaluation_mode") -} - -/// The compiled audit's `evidence_limit`, as its Python attribute reads it. -pub(crate) fn evidence_limit<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evidence_limit(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evidence_limit") -} - -/// The compiled audit's `evidence_sql`, as its Python attribute reads it. -pub(crate) fn evidence_sql<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evidence_sql(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evidence_sql") -} - -/// The compiled audit's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "key") -} - -/// The compiled audit's `measurement_contract`, as its Python attribute reads it. -pub(crate) fn measurement_contract<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .measurement_contract(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "measurement_contract") -} - -/// The compiled audit's `meta`, as its Python attribute reads it. -pub(crate) fn meta<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.meta(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "meta") -} - -/// The compiled audit's `minimum_samples`, as its Python attribute reads it. -pub(crate) fn minimum_samples<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .minimum_samples(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "minimum_samples") -} - -/// The compiled audit's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "name") -} - -/// The compiled audit's `references`, as its Python attribute reads it. -pub(crate) fn references<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "references") -} - -/// The compiled audit's `run_scope`, as its Python attribute reads it. -pub(crate) fn run_scope<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .run_scope(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "run_scope") -} - -/// The compiled audit's `scope_deps`, as its Python attribute reads it. -pub(crate) fn scope_deps<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .scope_deps(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "scope_deps") -} - -/// The compiled audit's `severity`, as its Python attribute reads it. -pub(crate) fn severity<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .severity(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "severity") -} - -/// The compiled audit's `sql_body`, as its Python attribute reads it. -pub(crate) fn sql_body<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .sql_body(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "sql_body") -} - -/// The compiled audit's `thresholds`, as its Python attribute reads it. -pub(crate) fn thresholds<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .thresholds(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "thresholds") -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs index fc7996ef74..5eafc3e79a 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs @@ -1,13 +1,11 @@ //! Audits attached by the native loop, held by the compiled project, and their views. -use crate::resource_views::models::AuditProjections; use crate::resource_views::models::NativeAuditInput; use crate::resource_views::models::NativeCompiledAudit; use crate::resource_views::types::AttachedAudit; use crate::resource_views::types::AuditRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; use pyo3::{FromPyObject, pymethods}; @@ -40,19 +38,7 @@ pub(crate) type AuditAssemblyRow = ( Option<(String, String)>, ); -/// The audit record of an attached audit, before assembly resolves its scope dependencies. -pub(crate) fn audit_record(attached: AttachedAudit) -> AuditRecord { - AuditRecord { - attached, - scope_deps: None, - host: AuditProjections { - references: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - -fn read_audit<'py, T>( +pub(crate) fn read_audit<'py, T>( owner: &Py, index: usize, py: Python<'py>, @@ -338,21 +324,3 @@ audit_getters!(NativeCompiledAudit, { }) } }); - -/// A native audit view's reference rows, or `None` for a hand-built audit record. -pub(crate) fn audit_reference_rows( - audit: &Bound<'_, PyAny>, -) -> PyResult>> { - let py: Python<'_> = audit.py(); - let (owner, index): (&Py, usize) = - if let Ok(view) = audit.cast::() { - (&view.get().owner, view.get().index) - } else if let Ok(view) = audit.cast::() { - (&view.get().owner, view.get().index) - } else { - return Ok(None); - }; - read_audit(owner, index, py, |record, _| { - Ok(Some(record.attached.references.clone())) - }) -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs index 03a64e433e..0584e54ae8 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs @@ -1,16 +1,14 @@ //! A compiled function's attributes the planner reads, from its native view. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use crate::resource_views::models::NativeCompiledFunction; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a function object; any other object is a Python-built record, counted. #[inline] -fn native_function<'a>( +pub(crate) fn native_function<'a>( function: &'a Bound<'_, PyAny>, ) -> PyResult> { if let Ok(view) = function.cast::() { @@ -19,164 +17,3 @@ fn native_function<'a>( count_stage(function.py(), COMPILED_VIEW_FALLBACKS, 1)?; Ok(None) } - -/// The compiled function's `arguments`, as its Python attribute reads it. -pub(crate) fn arguments<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .arguments(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "arguments") -} - -/// The compiled function's `body_sql`, as its Python attribute reads it. -pub(crate) fn body_sql<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .body_sql(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "body_sql") -} - -/// The compiled function's `deps`, as its Python attribute reads it. -pub(crate) fn deps<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.deps(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "deps") -} - -/// The compiled function's `destination`, as its Python attribute reads it. -pub(crate) fn destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "destination") -} - -/// The compiled function's `entry_point`, as its Python attribute reads it. -pub(crate) fn entry_point<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .entry_point(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "entry_point") -} - -/// The compiled function's `fingerprint_destination`, as its Python attribute reads it. -pub(crate) fn fingerprint_destination<'py>( - function: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .fingerprint_destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "fingerprint_destination") -} - -/// The compiled function's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "key") -} - -/// The compiled function's `language`, as its Python attribute reads it. -pub(crate) fn language<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .language(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "language") -} - -/// The compiled function's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "name") -} - -/// The compiled function's `packages`, as its Python attribute reads it. -pub(crate) fn packages<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .packages(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "packages") -} - -/// The compiled function's `relative_path`, as its Python attribute reads it. -pub(crate) fn relative_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .relative_path(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "relative_path") -} - -/// The compiled function's `return_columns`, as its Python attribute reads it. -pub(crate) fn return_columns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .return_columns(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "return_columns") -} - -/// The compiled function's `returns`, as its Python attribute reads it. -pub(crate) fn returns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .returns(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "returns") -} - -/// The compiled function's `runtime_version`, as its Python attribute reads it. -pub(crate) fn runtime_version<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .runtime_version(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "runtime_version") -} - -/// The compiled function's `source_file_path`, as its Python attribute reads it. -pub(crate) fn source_file_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .source_file_path(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "source_file_path") -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs index cf6aa59013..f6e5c2a22c 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs @@ -1,6 +1,5 @@ //! Functions attached by the native loop, held by the compiled project, and their views. -use crate::resource_views::models::FunctionProjections; use crate::resource_views::models::NativeCompiledFunction; use crate::resource_views::models::NativeFunctionInput; use crate::resource_views::types::AttachedFunction; @@ -52,20 +51,6 @@ positional_record! { } } -/// The function record of an attached function, before assembly resolves its dependencies. -pub(crate) fn function_record(attached: AttachedFunction) -> FunctionRecord { - FunctionRecord { - attached, - deps: None, - host: FunctionProjections { - arguments: PyOnceLock::new(), - return_columns: PyOnceLock::new(), - references: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - /// `rows` as `class` records, built once into `cache`. fn named_types( rows: &[NamedTypeRow], diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs index c93a560bd0..fe40ad34dc 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod audit_reads; pub(crate) mod audit_views; pub(crate) mod function_reads; pub(crate) mod function_views; -pub(crate) mod resource_reads; pub(crate) mod seed_reads; pub(crate) mod seed_views; pub(crate) mod source_reads; diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs deleted file mode 100644 index 6d70b71eba..0000000000 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/resource_reads.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! A compiled resource's attributes read alike for models, functions, seeds and sources. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; - -use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; -use crate::compiled_views::models::NativeCompiledModel; -use crate::graph::main::nodes::native_key::native_key; -use crate::graph::models::ResourceRow; -use crate::resource_views::models::NativeCompiledFunction; -use crate::resource_views::models::NativeCompiledSeed; -use crate::resource_views::models::NativeCompiledSource; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// A resource's graph facts without constructing its public keys or dependency tuple. -pub(crate) fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { - let py = resource.py(); - if let Ok(view) = resource.cast::() { - return model_graph_row(view.get(), py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - Ok(ResourceRow { - key: native_key(&native_field(resource, "key")?)?, - deps: native_field(resource, "deps")? - .try_iter()? - .map(|key| native_key(&key?)) - .collect::>()?, - tags: Vec::new(), - folder: None, - }) -} - -/// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn destination<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "destination") -} - -/// A resource's `key` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn key<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSource = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "key") -} - -/// A resource's `name` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn name<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSource = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "name") -} - -/// A resource's `references` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn references<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "references") -} - -/// A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn schema_entry<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "schema_entry") -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs index f8b594f936..7bce632019 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs @@ -1,79 +1,19 @@ //! A compiled seed's attributes the planner reads, from its native view. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use crate::resource_views::models::NativeCompiledSeed; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a seed object; any other object is a Python-built record, counted. #[inline] -fn native_seed<'a>(seed: &'a Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn native_seed<'a>( + seed: &'a Bound<'_, PyAny>, +) -> PyResult> { if let Ok(view) = seed.cast::() { return Ok(Some(view.get())); } count_stage(seed.py(), COMPILED_VIEW_FALLBACKS, 1)?; Ok(None) } - -/// The compiled seed's `destination`, as its Python attribute reads it. -pub(crate) fn destination<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "destination") -} - -/// The compiled seed's `external`, as its Python attribute reads it. -pub(crate) fn external<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.is_external().into_bound_py_any(py); - } - native_field(seed, "external") -} - -/// The compiled seed's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "key") -} - -/// The compiled seed's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "name") -} - -/// The compiled seed's `schema_entry`, as its Python attribute reads it. -pub(crate) fn schema_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "schema_entry") -} - -/// The compiled seed's `seed_file`, as its Python attribute reads it. -pub(crate) fn seed_file<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .seed_file(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "seed_file") -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs index 17af7c6a95..cf99e00a71 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs @@ -1,79 +1,21 @@ //! Seeds paired with their declarations, held by the compiled project, and their views. -use crate::resource_views::models::SeedHandles; use crate::resource_views::models::SeedNamespace; use crate::resource_views::models::SeedParts; use crate::resource_views::types::SeedRecord; use crate::resource_views::types::SeedViews; use pyo3::exceptions::PyIndexError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; +use pyo3::prelude::{Py, PyResult, Python}; use crate::compiled_project::models::NativeCompiledProject; use sqlbuild_project::models::DestinationFact; -use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) const SEED_RESOURCE: &str = "seed"; pub(crate) const TAGS_ATTRIBUTE: &str = "tags"; -const NAME_ATTRIBUTE: &str = "name"; -const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const STEM_ATTRIBUTE: &str = "stem"; -const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; - -/// Pair each declaration with the CSV file of its name, raising for the first without one. -pub(crate) fn paired_seeds( - declarations: Vec<(Py, Py)>, - seed_files: &[Py], - compile_error: &Bound<'_, PyAny>, -) -> PyResult> { - let py: Python<'_> = compile_error.py(); - let mut names: Vec = Vec::with_capacity(declarations.len()); - for (entry, _) in &declarations { - names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); - } - let mut stems: Vec = Vec::with_capacity(seed_files.len()); - for seed_file in seed_files { - stems.push( - native_field( - &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, - STEM_ATTRIBUTE, - )? - .extract()?, - ); - } - let pairs: Vec = match pair_seed_declarations(&names, &stems) { - Ok(pairs) => pairs, - Err(missing) => { - let schema_file: &Py = &declarations[missing].1; - let relative_path: String = - native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .extract()?; - let message: String = format!( - "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", - names[missing] - ); - return Err(PyErr::from_value(compile_error.call1((message,))?)); - } - }; - let mut seeds: Vec = Vec::with_capacity(pairs.len()); - for (((schema_entry, schema_file), name), file_index) in - declarations.into_iter().zip(names).zip(pairs) - { - seeds.push(SeedRecord { - name, - destination: None, - host: SeedHandles { - seed_file: seed_files[file_index].clone_ref(py), - schema_entry, - schema_file, - key: PyOnceLock::new(), - }, - }); - } - Ok(seeds) -} +pub(crate) const NAME_ATTRIBUTE: &str = "name"; +pub(crate) const FILE_PATH_ATTRIBUTE: &str = "file_path"; +pub(crate) const STEM_ATTRIBUTE: &str = "stem"; +pub(crate) const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; impl SeedViews for SeedRecord { fn parts(&self) -> SeedParts<'_> { diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs index 306022c624..ee72b3e2a5 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs @@ -1,59 +1,19 @@ //! A compiled source's attributes the planner reads, from its native view. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use crate::resource_views::models::NativeCompiledSource; use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a source object; any other object is a Python-built record, counted. #[inline] -fn native_source<'a>(source: &'a Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn native_source<'a>( + source: &'a Bound<'_, PyAny>, +) -> PyResult> { if let Ok(view) = source.cast::() { return Ok(Some(view.get())); } count_stage(source.py(), COMPILED_VIEW_FALLBACKS, 1)?; Ok(None) } - -/// The compiled source's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "key") -} - -/// The compiled source's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "name") -} - -/// The compiled source's `source_entry`, as its Python attribute reads it. -pub(crate) fn source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view - .source_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "source_entry") -} - -/// The compiled source's `source_file`, as its Python attribute reads it. -pub(crate) fn source_file<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view - .source_file(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "source_file") -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs index 8e2c145071..b584ce0ed0 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs @@ -1,5 +1,6 @@ //! Sources attached by the native loop, held by the compiled project, and their views. +use crate::resource_views::main::source_views::replaced::replaced; use crate::resource_views::models::NativeCompiledSource; use crate::resource_views::models::NativeSourceInput; use crate::resource_views::models::SourceKeyFacts; @@ -16,7 +17,6 @@ use sqlbuild_python_base::positional_record; use crate::compiled_project::main::python_column_facts::python_column_facts; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::main::compiled_models::audit_key_rows::audit_key_rows; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; pub(crate) const SOURCE_RESOURCE: &str = "source"; @@ -92,15 +92,6 @@ impl SourceViews for SourceRecord { } } -/// `dataclasses.replace(value, **changes)`. -pub(crate) fn replaced( - _py: Python<'_>, - value: &Bound<'_, PyAny>, - changes: &Bound<'_, PyDict>, -) -> PyResult> { - Ok(replaced_keywords(value, changes)?.unbind()) -} - /// Read the source record behind a native source view, or `None` for any other object. pub(crate) fn read_source_view<'py, T>( view: &Bound<'py, PyAny>, @@ -131,20 +122,3 @@ pub(crate) fn read_source<'py, T>( .ok_or_else(|| PyIndexError::new_err(index))?; read(source, py) } - -/// Keep `attached` in `owner`; return each source's view. -pub(crate) fn retain_sources( - owner: &Bound<'_, NativeCompiledProject>, - attached: Vec, -) -> Vec { - let mut project = owner.borrow_mut(); - let first: usize = project.sources().len(); - let count: usize = attached.len(); - for source in attached { - project.retain_source(source_record(source)); - } - drop(project); - (first..first + count) - .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) - .collect() -} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs index 2a3678b8d1..f5c1f81f6b 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs @@ -1,12 +1,17 @@ //! The compiled audit's `always_run`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `always_run`, as its Python attribute reads it. -#[inline] pub fn always_run<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::always_run(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .always_run(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "always_run") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs index de5f871f1c..fbc352a063 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs @@ -1,12 +1,17 @@ //! The compiled audit's `attached_column_name`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `attached_column_name`, as its Python attribute reads it. -#[inline] pub fn attached_column_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::attached_column_name(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_column_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_column_name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs index 7c27b4d747..368a605af4 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs @@ -1,12 +1,17 @@ //! The compiled audit's `attached_target_kind`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `attached_target_kind`, as its Python attribute reads it. -#[inline] pub fn attached_target_kind<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::attached_target_kind(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_target_kind(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_target_kind") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs index 9c03088229..174817d0bf 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs @@ -1,12 +1,17 @@ //! The compiled audit's `attached_target_name`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `attached_target_name`, as its Python attribute reads it. -#[inline] pub fn attached_target_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::attached_target_name(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_target_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_target_name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs index 29bdf90c00..ba5aa9b32e 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs @@ -1,12 +1,17 @@ //! The compiled audit's `definition_name`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `definition_name`, as its Python attribute reads it. -#[inline] pub fn definition_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::definition_name(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .definition_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "definition_name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs index a7c6f46d88..c2015c53f8 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs @@ -1,12 +1,17 @@ //! The compiled audit's `description`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `description`, as its Python attribute reads it. -#[inline] pub fn description<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::description(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .description(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "description") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs index 65a0b29d98..9a0049b3ee 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs @@ -1,12 +1,17 @@ //! The compiled audit's `evaluation_mode`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `evaluation_mode`, as its Python attribute reads it. -#[inline] pub fn evaluation_mode<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::evaluation_mode(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evaluation_mode(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evaluation_mode") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs index 10db612acc..84f5115c47 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs @@ -1,12 +1,17 @@ //! The compiled audit's `evidence_limit`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `evidence_limit`, as its Python attribute reads it. -#[inline] pub fn evidence_limit<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::evidence_limit(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evidence_limit(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evidence_limit") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs index 96f2fe8ea8..4f4fcd981c 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs @@ -1,12 +1,17 @@ //! The compiled audit's `evidence_sql`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `evidence_sql`, as its Python attribute reads it. -#[inline] pub fn evidence_sql<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::evidence_sql(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evidence_sql(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evidence_sql") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs index a8c7ae0dd6..4fc7b3f646 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs @@ -1,12 +1,15 @@ //! The compiled audit's `key`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `key`, as its Python attribute reads it. -#[inline] pub fn key<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::key(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "key") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs index 318c99d9b4..c78b6edb17 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs @@ -1,12 +1,17 @@ //! The compiled audit's `measurement_contract`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `measurement_contract`, as its Python attribute reads it. -#[inline] pub fn measurement_contract<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::measurement_contract(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .measurement_contract(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "measurement_contract") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs index c27710e29f..5338add72a 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs @@ -1,12 +1,15 @@ //! The compiled audit's `meta`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `meta`, as its Python attribute reads it. -#[inline] pub fn meta<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::meta(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.meta(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "meta") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs index c2a6305db9..3c6543f8ab 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs @@ -1,12 +1,17 @@ //! The compiled audit's `minimum_samples`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `minimum_samples`, as its Python attribute reads it. -#[inline] pub fn minimum_samples<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::minimum_samples(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .minimum_samples(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "minimum_samples") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs index a42fcd4383..1b0f2e7c9b 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs @@ -1,12 +1,15 @@ //! The compiled audit's `name`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `name`, as its Python attribute reads it. -#[inline] pub fn name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::name(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs index 2014ed52d0..d5e72183f9 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs @@ -1,12 +1,17 @@ //! The compiled audit's `references`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `references`, as its Python attribute reads it. -#[inline] pub fn references<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::references(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "references") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs index 1118606722..625a5dc1be 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs @@ -1,12 +1,17 @@ //! The compiled audit's `run_scope`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `run_scope`, as its Python attribute reads it. -#[inline] pub fn run_scope<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::run_scope(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .run_scope(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "run_scope") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs index f06237c75c..942d44339e 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs @@ -1,12 +1,17 @@ //! The compiled audit's `scope_deps`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `scope_deps`, as its Python attribute reads it. -#[inline] pub fn scope_deps<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::scope_deps(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .scope_deps(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "scope_deps") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs index 8bbfc23c77..13d47a3965 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs @@ -1,12 +1,17 @@ //! The compiled audit's `severity`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `severity`, as its Python attribute reads it. -#[inline] pub fn severity<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::severity(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .severity(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "severity") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs index e873228d83..9d7c183eaf 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs @@ -1,12 +1,17 @@ //! The compiled audit's `sql_body`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `sql_body`, as its Python attribute reads it. -#[inline] pub fn sql_body<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::sql_body(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .sql_body(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "sql_body") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs index 20a1fabb63..3082b33e51 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs @@ -1,12 +1,17 @@ //! The compiled audit's `thresholds`, as its Python attribute reads it. -use crate::resource_views::_helpers::audit_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled audit's `thresholds`, as its Python attribute reads it. -#[inline] pub fn thresholds<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - audit_reads::thresholds(audit) + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .thresholds(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "thresholds") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs index 050b4d852f..33cefb79c0 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs @@ -1,11 +1,18 @@ //! The audit record of an attached audit, before assembly resolves its scope dependencies. -use crate::resource_views::_helpers::audit_views; +use crate::resource_views::models::AuditProjections; use crate::resource_views::types::AttachedAudit; use crate::resource_views::types::AuditRecord; +use pyo3::sync::PyOnceLock; /// The audit record of an attached audit, before assembly resolves its scope dependencies. -#[inline] pub fn audit_record(attached: AttachedAudit) -> AuditRecord { - audit_views::audit_record(attached) + AuditRecord { + attached, + scope_deps: None, + host: AuditProjections { + references: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs index b1013abedb..c3efd47c89 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs @@ -1,13 +1,24 @@ //! A native audit view's reference rows, or `None` for a hand-built audit record. -use crate::resource_views::_helpers::audit_views; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_project::models::NativeCompiledProject; +use crate::resource_views::_helpers::audit_views::read_audit; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use sqlbuild_project::models::ReferenceRow; /// A native audit view's reference rows, or `None` for a hand-built audit record. -#[inline] pub fn audit_reference_rows(audit: &Bound<'_, PyAny>) -> PyResult>> { - audit_views::audit_reference_rows(audit) + let py: Python<'_> = audit.py(); + let (owner, index): (&Py, usize) = + if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else { + return Ok(None); + }; + read_audit(owner, index, py, |record, _| { + Ok(Some(record.attached.references.clone())) + }) } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs index 2d3fff3dd8..5fb05b4c33 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs @@ -1,12 +1,17 @@ //! The compiled function's `arguments`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `arguments`, as its Python attribute reads it. -#[inline] pub fn arguments<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::arguments(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .arguments(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "arguments") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs index e2c27bfe80..19cbb14b89 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs @@ -1,12 +1,17 @@ //! The compiled function's `body_sql`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `body_sql`, as its Python attribute reads it. -#[inline] pub fn body_sql<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::body_sql(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .body_sql(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "body_sql") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs index 5fb585af71..cf40148ff3 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs @@ -1,12 +1,15 @@ //! The compiled function's `deps`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `deps`, as its Python attribute reads it. -#[inline] pub fn deps<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::deps(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.deps(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "deps") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs index 740a4f3c86..414d0c0217 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs @@ -1,12 +1,17 @@ //! The compiled function's `destination`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `destination`, as its Python attribute reads it. -#[inline] pub fn destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::destination(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "destination") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs index bb6e970aaf..4af4e43095 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs @@ -1,12 +1,17 @@ //! The compiled function's `entry_point`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `entry_point`, as its Python attribute reads it. -#[inline] pub fn entry_point<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::entry_point(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .entry_point(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "entry_point") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs index b97b6f2555..5841da1258 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs @@ -1,12 +1,17 @@ //! The compiled function's `fingerprint_destination`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `fingerprint_destination`, as its Python attribute reads it. -#[inline] pub fn fingerprint_destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::fingerprint_destination(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .fingerprint_destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "fingerprint_destination") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs index ccd9ffd5da..0b37891c16 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs @@ -1,12 +1,15 @@ //! The compiled function's `key`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `key`, as its Python attribute reads it. -#[inline] pub fn key<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::key(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "key") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs index 76865efa26..76218bd9fc 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs @@ -1,12 +1,17 @@ //! The compiled function's `language`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `language`, as its Python attribute reads it. -#[inline] pub fn language<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::language(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .language(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "language") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs index 95376c0b8a..a58608ab5e 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs @@ -1,12 +1,15 @@ //! The compiled function's `name`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `name`, as its Python attribute reads it. -#[inline] pub fn name<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::name(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs index d3ed3c6ab9..b019e85eff 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs @@ -1,12 +1,17 @@ //! The compiled function's `packages`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `packages`, as its Python attribute reads it. -#[inline] pub fn packages<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::packages(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .packages(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "packages") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs index f53adeed38..12ceaec713 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs @@ -1,12 +1,17 @@ //! The compiled function's `relative_path`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `relative_path`, as its Python attribute reads it. -#[inline] pub fn relative_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::relative_path(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .relative_path(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "relative_path") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs index ee0a326ec2..5e40a8f43f 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs @@ -1,12 +1,17 @@ //! The compiled function's `return_columns`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `return_columns`, as its Python attribute reads it. -#[inline] pub fn return_columns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::return_columns(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .return_columns(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "return_columns") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs index 758d378812..f3ce015997 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs @@ -1,12 +1,17 @@ //! The compiled function's `returns`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `returns`, as its Python attribute reads it. -#[inline] pub fn returns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::returns(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .returns(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "returns") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs index 888613af23..6206a31407 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs @@ -1,12 +1,17 @@ //! The compiled function's `runtime_version`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `runtime_version`, as its Python attribute reads it. -#[inline] pub fn runtime_version<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::runtime_version(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .runtime_version(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "runtime_version") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs index 4831656309..a5bcd85d13 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs @@ -1,12 +1,17 @@ //! The compiled function's `source_file_path`, as its Python attribute reads it. -use crate::resource_views::_helpers::function_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled function's `source_file_path`, as its Python attribute reads it. -#[inline] pub fn source_file_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - function_reads::source_file_path(function) + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .source_file_path(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "source_file_path") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs index a5388f4ead..1090fa0106 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs @@ -1,11 +1,20 @@ //! The function record of an attached function, before assembly resolves its dependencies. -use crate::resource_views::_helpers::function_views; +use crate::resource_views::models::FunctionProjections; use crate::resource_views::types::AttachedFunction; use crate::resource_views::types::FunctionRecord; +use pyo3::sync::PyOnceLock; /// The function record of an attached function, before assembly resolves its dependencies. -#[inline] pub fn function_record(attached: AttachedFunction) -> FunctionRecord { - function_views::function_record(attached) + FunctionRecord { + attached, + deps: None, + host: FunctionProjections { + arguments: PyOnceLock::new(), + return_columns: PyOnceLock::new(), + references: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs index 45331cee28..1ea70b4ed1 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs @@ -1,12 +1,35 @@ //! A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. -#[inline] pub fn destination<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - resource_reads::destination(resource) + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "destination") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs index de3003f408..a97c37e812 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs @@ -1,13 +1,40 @@ //! A resource's graph facts without constructing its public keys or dependency tuple. +use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; +use crate::compiled_views::models::NativeCompiledModel; +use crate::graph::main::nodes::native_key::native_key; use crate::graph::models::ResourceRow; -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's graph facts without constructing its public keys or dependency tuple. -#[inline] pub fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { - resource_reads::graph_row(resource) + let py = resource.py(); + if let Ok(view) = resource.cast::() { + return model_graph_row(view.get(), py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + Ok(ResourceRow { + key: native_key(&native_field(resource, "key")?)?, + deps: native_field(resource, "deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + tags: Vec::new(), + folder: None, + }) } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs index 2191f77479..00c533c132 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs @@ -1,12 +1,34 @@ //! A resource's `key` from whichever native view it is; any other object is read by attribute, counted. -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's `key` from whichever native view it is; any other object is read by attribute, counted. -#[inline] pub fn key<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - resource_reads::key(resource) + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSource = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "key") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs index eb4a2d5815..58377f87a9 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs @@ -1,12 +1,34 @@ //! A resource's `name` from whichever native view it is; any other object is read by attribute, counted. -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's `name` from whichever native view it is; any other object is read by attribute, counted. -#[inline] pub fn name<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - resource_reads::name(resource) + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSource = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs index 5187c2cdd8..b1e37c87ca 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs @@ -1,12 +1,28 @@ //! A resource's `references` from whichever native view it is; any other object is read by attribute, counted. -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's `references` from whichever native view it is; any other object is read by attribute, counted. -#[inline] pub fn references<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - resource_reads::references(resource) + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "references") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs index 902b8fba1a..3f99966e56 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs @@ -1,12 +1,28 @@ //! A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. -use crate::resource_views::_helpers::resource_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledSeed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. -#[inline] pub fn schema_entry<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - resource_reads::schema_entry(resource) + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "schema_entry") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs index 34ce77c8bf..c4e83d5011 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs @@ -1,12 +1,17 @@ //! The compiled seed's `destination`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `destination`, as its Python attribute reads it. -#[inline] pub fn destination<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::destination(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "destination") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs index 337c75c948..64413b77de 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs @@ -1,12 +1,15 @@ //! The compiled seed's `external`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `external`, as its Python attribute reads it. -#[inline] pub fn external<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::external(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.is_external().into_bound_py_any(py); + } + native_field(seed, "external") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs index dbf7df7677..2183d8bdb0 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs @@ -1,12 +1,15 @@ //! The compiled seed's `key`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `key`, as its Python attribute reads it. -#[inline] pub fn key<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::key(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "key") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs index 62318d1ce4..7aad821a4d 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs @@ -1,12 +1,15 @@ //! The compiled seed's `name`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `name`, as its Python attribute reads it. -#[inline] pub fn name<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::name(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs index f2f1546ba1..147af1cadf 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs @@ -1,12 +1,17 @@ //! The compiled seed's `schema_entry`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `schema_entry`, as its Python attribute reads it. -#[inline] pub fn schema_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::schema_entry(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "schema_entry") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs index 487f2ce6ca..d0ad10b9d9 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs @@ -1,12 +1,17 @@ //! The compiled seed's `seed_file`, as its Python attribute reads it. -use crate::resource_views::_helpers::seed_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled seed's `seed_file`, as its Python attribute reads it. -#[inline] pub fn seed_file<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - seed_reads::seed_file(seed) + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .seed_file(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "seed_file") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs index 872c61cfa3..18e88f7247 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs @@ -1,18 +1,66 @@ //! Pair each declaration with the CSV file of its name, raising for the first without one. -use crate::resource_views::_helpers::seed_views; +use crate::resource_views::_helpers::seed_views::FILE_PATH_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::NAME_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::RELATIVE_PATH_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::STEM_ATTRIBUTE; +use crate::resource_views::models::SeedHandles; use crate::resource_views::types::SeedRecord; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Pair each declaration with the CSV file of its name, raising for the first without one. -#[inline] pub fn paired_seeds( declarations: Vec<(Py, Py)>, seed_files: &[Py], compile_error: &Bound<'_, PyAny>, ) -> PyResult> { - seed_views::paired_seeds(declarations, seed_files, compile_error) + let py: Python<'_> = compile_error.py(); + let mut names: Vec = Vec::with_capacity(declarations.len()); + for (entry, _) in &declarations { + names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); + } + let mut stems: Vec = Vec::with_capacity(seed_files.len()); + for seed_file in seed_files { + stems.push( + native_field( + &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, + STEM_ATTRIBUTE, + )? + .extract()?, + ); + } + let pairs: Vec = match pair_seed_declarations(&names, &stems) { + Ok(pairs) => pairs, + Err(missing) => { + let schema_file: &Py = &declarations[missing].1; + let relative_path: String = + native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? + .str()? + .extract()?; + let message: String = format!( + "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", + names[missing] + ); + return Err(PyErr::from_value(compile_error.call1((message,))?)); + } + }; + let mut seeds: Vec = Vec::with_capacity(pairs.len()); + for (((schema_entry, schema_file), name), file_index) in + declarations.into_iter().zip(names).zip(pairs) + { + seeds.push(SeedRecord { + name, + destination: None, + host: SeedHandles { + seed_file: seed_files[file_index].clone_ref(py), + schema_entry, + schema_file, + key: PyOnceLock::new(), + }, + }); + } + Ok(seeds) } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs index 896a8333ac..ea4d396800 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs @@ -1,12 +1,15 @@ //! The compiled source's `key`, as its Python attribute reads it. -use crate::resource_views::_helpers::source_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled source's `key`, as its Python attribute reads it. -#[inline] pub fn key<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - source_reads::key(source) + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "key") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs index 7523cde498..1cb44fd30c 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs @@ -1,12 +1,15 @@ //! The compiled source's `name`, as its Python attribute reads it. -use crate::resource_views::_helpers::source_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled source's `name`, as its Python attribute reads it. -#[inline] pub fn name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - source_reads::name(source) + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "name") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs index 22529c585a..b3cb95895b 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs @@ -1,12 +1,17 @@ //! The compiled source's `source_entry`, as its Python attribute reads it. -use crate::resource_views::_helpers::source_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled source's `source_entry`, as its Python attribute reads it. -#[inline] pub fn source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - source_reads::source_entry(source) + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view + .source_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "source_entry") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs index 7b8893850a..f39b9ec4f8 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs @@ -1,12 +1,17 @@ //! The compiled source's `source_file`, as its Python attribute reads it. -use crate::resource_views::_helpers::source_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The compiled source's `source_file`, as its Python attribute reads it. -#[inline] pub fn source_file<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - source_reads::source_file(source) + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view + .source_file(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "source_file") } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs index de07103deb..8b9956d343 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs @@ -2,10 +2,7 @@ use crate::resource_views::_helpers::source_views; use crate::resource_views::types::SourceRecord; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// Read the source record behind a native source view, or `None` for any other object. #[inline] diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs index 56ed3255f9..f69e8e5220 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs @@ -1,19 +1,14 @@ //! `dataclasses.replace(value, **changes)`. -use crate::resource_views::_helpers::source_views; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; /// `dataclasses.replace(value, **changes)`. -#[inline] pub fn replaced( _py: Python<'_>, value: &Bound<'_, PyAny>, changes: &Bound<'_, PyDict>, ) -> PyResult> { - source_views::replaced(_py, value, changes) + Ok(replaced_keywords(value, changes)?.unbind()) } diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs index 8a026e5bfe..379e29e8ec 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs @@ -1,16 +1,24 @@ //! Keep `attached` in `owner`; return each source's view. use crate::compiled_project::models::NativeCompiledProject; -use crate::resource_views::_helpers::source_views; +use crate::resource_views::_helpers::source_views::source_record; use crate::resource_views::models::NativeSourceInput; use crate::resource_views::types::AttachedSource; use pyo3::prelude::Bound; /// Keep `attached` in `owner`; return each source's view. -#[inline] pub fn retain_sources( owner: &Bound<'_, NativeCompiledProject>, attached: Vec, ) -> Vec { - source_views::retain_sources(owner, attached) + let mut project = owner.borrow_mut(); + let first: usize = project.sources().len(); + let count: usize = attached.len(); + for source in attached { + project.retain_source(source_record(source)); + } + drop(project); + (first..first + count) + .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) + .collect() } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs index 719a832680..9b391baf7d 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs @@ -2,9 +2,9 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; use crate::compiled_views::main::compiled_models::star_flags::star_flags; @@ -111,7 +111,3 @@ fn own_reasons<'py>( } Ok(partial) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(semantic_coverage, module)?) -} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs index 28c19328b7..e203d01972 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs @@ -1,49 +1,3 @@ //! The function names and declared types native binding treats as known. -use std::collections::BTreeSet; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; - -const UNKNOWN: &str = "UNKNOWN"; - -/// Each declared function's name and its UDF and table-function analysis names, sorted. -pub(crate) fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { - let mut names: BTreeSet = BTreeSet::new(); - for function in functions.try_iter()? { - let name: String = native_field(&function?, "name")?.extract()?; - names.insert(format!("__sqlbuild_udf_{name}")); - names.insert(format!("__sqlbuild_table_function_{name}")); - names.insert(name); - } - Ok(names.into_iter().collect()) -} - -/// Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. -pub(crate) fn declared_types( - functions: &Bound<'_, PyAny>, - column_types: &Bound<'_, PyAny>, -) -> PyResult> { - let mut types: BTreeSet = BTreeSet::new(); - for columns in mapping_values(column_types)?.try_iter()? { - for column_type in mapping_values(&columns?)?.try_iter()? { - types.insert(column_type?.extract()?); - } - } - for function in functions.try_iter()? { - let function: Bound<'_, PyAny> = function?; - for argument in native_field(&function, "arguments")?.try_iter()? { - types.insert(native_field(&argument?, "type")?.extract()?); - } - let returns: Bound<'_, PyAny> = native_field(&function, "return_columns")?; - for column in returns.try_iter()? { - types.insert(native_field(&column?, "type")?.extract()?); - } - if !returns.is_truthy()? { - types.insert(native_field(&function, "returns")?.extract()?); - } - } - types.remove(UNKNOWN); - Ok(types.into_iter().collect()) -} +pub(crate) const UNKNOWN: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs index b92343e453..4d3c222ee1 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs @@ -1,16 +1,14 @@ //! Closed relation shapes of a compiled project, as binding and semantic checks read them. -use crate::semantic_shapes::constants::VALID_FROM_DEFAULT; -use crate::semantic_shapes::constants::VALID_TO_DEFAULT; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::semantic_shapes::main::snapshot_validity_columns::snapshot_validity_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; -use crate::analysis::models::BindingCatalog; -use crate::analysis::models::CatalogKey; +use crate::analysis::models::{BindingCatalog, CatalogKey}; use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; use crate::compiled_project::models::ReferenceTuple; use crate::compiled_views::main::compiled_models::declared_rows::declared_rows; @@ -25,7 +23,6 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; @@ -115,29 +112,6 @@ pub(crate) fn published_shape<'py>( Ok(shape) } -/// A snapshot model's validity column names, which the snapshot adds to its query's columns. -pub(crate) fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model.py(); - let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; - let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; - if !mapping_get(&values, "materialized")?.eq(snapshot)? { - return Ok(Vec::new()); - } - let mut names: Vec = Vec::with_capacity(2); - for (key, default) in [ - ("valid_from_column", VALID_FROM_DEFAULT), - ("valid_to_column", VALID_TO_DEFAULT), - ] { - let name: Bound<'_, PyAny> = mapping_get(&values, key)?; - names.push(if name.is_truthy()? { - name.str()?.extract()? - } else { - default.to_owned() - }); - } - Ok(names) -} - /// Inferred columns, re-quoted from authored SQL where the dialect keeps quoted case. fn binding_shape<'py>( model: &Bound<'py, PyAny>, @@ -388,14 +362,16 @@ fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { /// Closed relation shapes; `profile` defaults to the project's dialect. #[pyfunction] #[pyo3(signature = (**request))] -fn semantic_shapes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn semantic_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("semantic_shapes", request)?; project_shapes(&request.get("project")?, request.optional("profile")?) } /// The analysis names the references bind, UDFs excluded. #[pyfunction] -fn binding_relation_names<'py>( +pub(crate) fn binding_relation_names<'py>( references: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = references.py(); @@ -420,7 +396,7 @@ fn binding_relation_names<'py>( /// A model relation's published shape: `published_model_shape(*, model, columns, inputs, profile)`. #[pyfunction] #[pyo3(signature = (**request))] -fn published_model_shape<'py>( +pub(crate) fn published_model_shape<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("published_model_shape", request)?; @@ -431,9 +407,3 @@ fn published_model_shape<'py>( &request.get("profile")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(semantic_shapes, module)?)?; - module.add_function(wrap_pyfunction!(published_model_shape, module)?)?; - module.add_function(wrap_pyfunction!(binding_relation_names, module)?) -} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 1aeccb09a2..1b4a796e94 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -1,23 +1,20 @@ //! Rebind selected models to source columns the planner already inspected. +use crate::semantic_shapes::main::declared_types::declared_types; +use crate::semantic_shapes::main::function_names::function_names; use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; -use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::constants::{CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES}; use crate::analysis::main::schema_validation_request::schema_validation_request; use crate::analysis::main::schema_validations::schema_validations; use crate::analysis::models::BindingPositions; use crate::analysis::models::SchemaRequestParts; use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; -use crate::semantic_shapes::_helpers::declarations::declared_types; -use crate::semantic_shapes::_helpers::declarations::function_names; -use crate::semantic_shapes::_helpers::shapes::model_inputs; -use crate::semantic_shapes::_helpers::shapes::project_shapes; -use crate::semantic_shapes::_helpers::shapes::published_shape; +use crate::semantic_shapes::_helpers::shapes::{model_inputs, project_shapes, published_shape}; use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; @@ -497,7 +494,3 @@ fn call<'py>(callable: &Object<'py>, pairs: &[(&str, Object<'py>)]) -> PyResult< } callable.call((), Some(&arguments)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(source_binding_diagnostics, module)?) -} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs index 115a374c2e..435e8830ed 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs @@ -1,15 +1,35 @@ //! Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. -use crate::semantic_shapes::_helpers::declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::semantic_shapes::_helpers::declarations::UNKNOWN; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; +use std::collections::BTreeSet; /// Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. -#[inline] pub fn declared_types( functions: &Bound<'_, PyAny>, column_types: &Bound<'_, PyAny>, ) -> PyResult> { - declarations::declared_types(functions, column_types) + let mut types: BTreeSet = BTreeSet::new(); + for columns in mapping_values(column_types)?.try_iter()? { + for column_type in mapping_values(&columns?)?.try_iter()? { + types.insert(column_type?.extract()?); + } + } + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + for argument in native_field(&function, "arguments")?.try_iter()? { + types.insert(native_field(&argument?, "type")?.extract()?); + } + let returns: Bound<'_, PyAny> = native_field(&function, "return_columns")?; + for column in returns.try_iter()? { + types.insert(native_field(&column?, "type")?.extract()?); + } + if !returns.is_truthy()? { + types.insert(native_field(&function, "returns")?.extract()?); + } + } + types.remove(UNKNOWN); + Ok(types.into_iter().collect()) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs index 4b3c5a012f..6270554ccd 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs @@ -1,12 +1,17 @@ //! Each declared function's name and its UDF and table-function analysis names, sorted. -use crate::semantic_shapes::_helpers::declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::BTreeSet; /// Each declared function's name and its UDF and table-function analysis names, sorted. -#[inline] pub fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { - declarations::function_names(functions) + let mut names: BTreeSet = BTreeSet::new(); + for function in functions.try_iter()? { + let name: String = native_field(&function?, "name")?.extract()?; + names.insert(format!("__sqlbuild_udf_{name}")); + names.insert(format!("__sqlbuild_table_function_{name}")); + names.insert(name); + } + Ok(names.into_iter().collect()) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs index f854a3a172..11b86cb4a5 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs @@ -1,9 +1,7 @@ //! Closed shapes by relation name; absence means open, never an empty physical table. use crate::semantic_shapes::_helpers::shapes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Closed shapes by relation name; absence means open, never an empty physical table. diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs index 71208889db..a478031553 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs @@ -1,12 +1,10 @@ //! Register the coverage functions and classes. -use crate::semantic_shapes::_helpers::coverage; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::semantic_shapes::_helpers::coverage::semantic_coverage; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the coverage functions and classes. -#[inline] pub fn register_coverage(module: &Bound<'_, PyModule>) -> PyResult<()> { - coverage::register(module) + module.add_function(wrap_pyfunction!(semantic_coverage, module)?) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs index eb6a14d4a0..799c2deae1 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs @@ -1,12 +1,14 @@ //! Register the shapes functions and classes. -use crate::semantic_shapes::_helpers::shapes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::semantic_shapes::_helpers::shapes::binding_relation_names; +use crate::semantic_shapes::_helpers::shapes::published_model_shape; +use crate::semantic_shapes::_helpers::shapes::semantic_shapes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the shapes functions and classes. -#[inline] pub fn register_shapes(module: &Bound<'_, PyModule>) -> PyResult<()> { - shapes::register(module) + module.add_function(wrap_pyfunction!(semantic_shapes, module)?)?; + module.add_function(wrap_pyfunction!(published_model_shape, module)?)?; + module.add_function(wrap_pyfunction!(binding_relation_names, module)?) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs index 18d09fd054..045414aef1 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs @@ -1,12 +1,10 @@ //! Register the source bindings functions and classes. -use crate::semantic_shapes::_helpers::source_bindings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::semantic_shapes::_helpers::source_bindings::source_binding_diagnostics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the source bindings functions and classes. -#[inline] pub fn register_source_bindings(module: &Bound<'_, PyModule>) -> PyResult<()> { - source_bindings::register(module) + module.add_function(wrap_pyfunction!(source_binding_diagnostics, module)?) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs index 73b112fa0e..c9e63abd2d 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs @@ -1,9 +1,7 @@ //! Partial-check reasons by model name; empty when SQL analysis is off. use crate::semantic_shapes::_helpers::coverage; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Partial-check reasons by model name; empty when SQL analysis is off. diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs index bc4d7f1bd5..3f3105ad01 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs @@ -1,12 +1,32 @@ //! A snapshot model's validity column names, which the snapshot adds to its query's columns. -use crate::semantic_shapes::_helpers::shapes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::semantic_shapes::constants::VALID_FROM_DEFAULT; +use crate::semantic_shapes::constants::VALID_TO_DEFAULT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; /// A snapshot model's validity column names, which the snapshot adds to its query's columns. -#[inline] pub fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { - shapes::snapshot_validity_columns(model) + let py: Python<'_> = model.py(); + let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; + let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; + if !mapping_get(&values, "materialized")?.eq(snapshot)? { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::with_capacity(2); + for (key, default) in [ + ("valid_from_column", VALID_FROM_DEFAULT), + ("valid_to_column", VALID_TO_DEFAULT), + ] { + let name: Bound<'_, PyAny> = mapping_get(&values, key)?; + names.push(if name.is_truthy()? { + name.str()?.extract()? + } else { + default.to_owned() + }); + } + Ok(names) } diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs index 929ce461cb..aee9fd01dc 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs @@ -1,10 +1,8 @@ //! `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. use crate::semantic_shapes::_helpers::source_bindings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyTuple}; /// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. #[inline] diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs index 60d0a5ae42..067d5c84bc 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs @@ -10,15 +10,11 @@ use sqlbuild_cursor::algebra::models::{Grain, Scalar}; use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::compiled_views::main::compiled_models::query_sql::query_sql; -use crate::sql_test_glue::constants::CURSOR_ALGEBRA_EXCEPTIONS; -use crate::sql_test_glue::constants::CURSOR_END_KEY; -use crate::sql_test_glue::constants::CURSOR_START_KEY; -use crate::sql_test_glue::constants::DEFAULT_INTEGER_END; -use crate::sql_test_glue::constants::DEFAULT_INTEGER_START; -use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; -use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; -use crate::sql_test_glue::constants::INTEGER_CURSOR_TYPE; -use crate::sql_test_glue::constants::TIMESTAMP_CURSOR_TYPE; +use crate::sql_test_glue::constants::{ + CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, + DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, + TIMESTAMP_CURSOR_TYPE, +}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs index b68f080f10..554d9c1d1c 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs @@ -7,8 +7,7 @@ use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::planner_strategy::main::planning_context::planning_context; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::sql_test_glue::_helpers::plan::plan_driver::planned as planned_tests; -use crate::sql_test_glue::models::SqlTestAssembly; -use crate::sql_test_glue::models::SqlTestPlanOptions; +use crate::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -154,10 +153,3 @@ pub(crate) fn assemble_sql_test_plans<'py>( } PyTuple::new(py, results) } - -/// `plan_sql_tests` as a function of `module`. -pub(crate) fn plan_sql_tests_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_sql_tests, module) -} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs index d89c3cce89..f935a7abb8 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs @@ -1,19 +1,17 @@ //! Host SQL-test request preparation, native planning, and Python result conversion. +use crate::sql_test_glue::main::planned_rows::planned_rows; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; +use pyo3::types::{PyDict, PyTuple}; use crate::planner_phases::main::object::object; use crate::planner_strategy::main::count_chain_resolution::count_chain_resolution; use crate::sql_test_glue::_helpers::plan::plan_values::tuple; -use crate::sql_test_glue::_helpers::plan::planning::ChainRequest; -use crate::sql_test_glue::_helpers::plan::planning::PlanningRequest; -use crate::sql_test_glue::_helpers::plan::planning::plan_compiled_sql_tests; -use crate::sql_test_glue::_helpers::plan::planning::resolve_compiled_sql_test_chains; +use crate::sql_test_glue::_helpers::plan::planning::{ + ChainRequest, resolve_compiled_sql_test_chains, +}; use crate::sql_test_glue::models::PlanRow; use crate::sql_test_glue::models::SqlTestPlanOptions; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -80,87 +78,6 @@ pub(crate) fn resolve_sql_test_model_chains<'py>( chains(project, tests) } -/// Native plans of one SQL-test batch, with adapter rendering and timing accounted for. -pub(crate) fn planned_rows<'py>( - (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - tests: &Bound<'py, PyAny>, - options: &SqlTestPlanOptions<'py>, -) -> PyResult> { - let py = project.py(); - let artifact = options.artifact; - if !tests.is_truthy()? { - return Ok(Vec::new()); - } - let start: u64 = imported(py, "time", "perf_counter_ns")? - .call0()? - .extract()?; - let model_sql = crate::sql_test_glue::_helpers::plan::plan_rendering::models(project, adapter)?; - let functions = - crate::sql_test_glue::_helpers::plan::plan_rendering::functions(project, adapter)?; - let overrides = - crate::sql_test_glue::_helpers::plan::plan_rendering::overrides(project, tests, adapter)?; - let request = PlanningRequest { - models: native_field(project, "models")?.extract()?, - rendered_model_sql: model_sql.extract()?, - functions: functions.extract()?, - tests: tests.extract()?, - rendered_test_overrides: overrides.extract()?, - sql_analysis_enabled: options.sql_analysis_enabled.extract()?, - sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, - set_difference_operator: adapter - .call_method0("render_set_difference_operator")? - .extract()?, - requires_derived_table_aliases: adapter - .call_method0("requires_derived_table_aliases")? - .extract()?, - lexical_syntax: native_field( - &native_field(project, "sql_lexical_syntax")?, - "native_mapping", - )? - .extract()?, - render_sql: artifact || options.render_sql, - include_plan: !artifact && options.include_plan, - max_unit_test_real_models: native_field( - &native_field(project, "settings")?, - "unit_test_max_real_models", - )? - .extract()?, - }; - let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { - Ok(value) => value, - Err(error) if error.is_instance_of::(py) => { - return Err( - crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( - py, error, tests, - )?, - ); - } - Err(error) => return Err(error), - }; - let end: u64 = imported(py, "time", "perf_counter_ns")? - .call0()? - .extract()?; - for (phase, elapsed) in [ - ( - "test_planning_ms", - planning_ns - + end - .saturating_sub(start) - .saturating_sub(planning_ns) - .saturating_sub(rendering_ns), - ), - ("comparison_render_ms", rendering_ns), - ] { - add_compile_timing(phase, elapsed); - } - if artifact { - for row in &rows { - row.rendered_sql(py)?; - } - } - Ok(rows) -} - /// Plans of one SQL-test batch, or their compiled artifacts, for Python consumers. pub(crate) fn planned<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -269,47 +186,3 @@ pub(crate) fn sql_test_plan_error_messages<'py>( } PyTuple::new(warnings.py(), messages) } - -/// Return model closures resolved natively in one project batch, keyed by SQL test key. -#[pyfunction] -#[pyo3(signature = (*, project, tests))] -pub(crate) fn sql_test_model_chain_names_by_key<'py>( - project: &Bound<'py, PyAny>, - tests: &Bound<'py, PyAny>, -) -> PyResult> { - let py = project.py(); - let chains = resolve_sql_test_model_chains(project, tests)?; - let by_key = PyDict::new(py); - for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { - by_key.set_item(native_field(&test?, "key")?, chain?)?; - } - Ok(by_key) -} - -/// `plan_compiled_sql_test_artifacts` as a function of `module`. -pub(crate) fn plan_compiled_sql_test_artifacts_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_compiled_sql_test_artifacts, module) -} - -/// `plan_sql_tests_natively` as a function of `module`. -pub(crate) fn plan_sql_tests_natively_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_sql_tests_natively, module) -} - -/// `sql_test_model_chain_names_by_key` as a function of `module`. -pub(crate) fn sql_test_model_chain_names_by_key_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(sql_test_model_chain_names_by_key, module) -} - -/// `sql_test_plan_error_messages` as a function of `module`. -pub(crate) fn sql_test_plan_error_messages_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(sql_test_plan_error_messages, module) -} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs index f22b0d047b..8e2028dd65 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs @@ -9,8 +9,7 @@ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; -use crate::sql_test_glue::constants::TABLE_FN; -use crate::sql_test_glue::constants::UDF; +use crate::sql_test_glue::constants::{TABLE_FN, UDF}; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs index e02fee468d..5201e859d9 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs @@ -2,15 +2,14 @@ use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::compiled_views::main::compiled_models::query_sql::query_sql; -use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; -use crate::sql_test_glue::_helpers::plan::plan_values::string; +use crate::sql_test_glue::_helpers::plan::plan_values::{model_payload, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::sql_test_glue::_helpers::inputs::cursor_window::declared_window_cursor_models; -use crate::sql_test_glue::_helpers::inputs::cursor_window::declares_cursor_window; -use crate::sql_test_glue::_helpers::inputs::cursor_window::render_test_cursor_intrinsics; +use crate::sql_test_glue::_helpers::inputs::cursor_window::{ + declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, +}; use crate::sql_test_glue::constants::SENTINEL; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs index 42d02b9353..3a196ffab5 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs @@ -2,9 +2,7 @@ use crate::sql_test_glue::_helpers::plan::plan_assembly; use crate::sql_test_glue::models::SqlTestAssembly; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; /// Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs index d75b8a5177..cb78440ec2 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs @@ -1,11 +1,10 @@ //! `plan_compiled_sql_test_artifacts` as a function of `module`. -use crate::sql_test_glue::_helpers::plan::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver::plan_compiled_sql_test_artifacts; /// `plan_compiled_sql_test_artifacts` as a function of `module`. -#[inline] pub fn plan_compiled_sql_test_artifacts_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - plan_driver::plan_compiled_sql_test_artifacts_function(module) + pyo3::wrap_pyfunction!(plan_compiled_sql_test_artifacts, module) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs index 939389f748..123b1d6d63 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs @@ -1,11 +1,10 @@ //! `plan_sql_tests` as a function of `module`. -use crate::sql_test_glue::_helpers::plan::plan_assembly; +use crate::sql_test_glue::_helpers::plan::plan_assembly::plan_sql_tests; /// `plan_sql_tests` as a function of `module`. -#[inline] pub fn plan_sql_tests_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - plan_assembly::plan_sql_tests_function(module) + pyo3::wrap_pyfunction!(plan_sql_tests, module) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs index 43ad71a6a3..445d8bf4ec 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs @@ -1,11 +1,10 @@ //! `plan_sql_tests_natively` as a function of `module`. -use crate::sql_test_glue::_helpers::plan::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver::plan_sql_tests_natively; /// `plan_sql_tests_natively` as a function of `module`. -#[inline] pub fn plan_sql_tests_natively_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - plan_driver::plan_sql_tests_natively_function(module) + pyo3::wrap_pyfunction!(plan_sql_tests_natively, module) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs index d65eb82839..a5509174c7 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs @@ -1,18 +1,90 @@ //! Native plans of one SQL-test batch, with adapter rendering and timing accounted for. -use crate::sql_test_glue::_helpers::plan::plan_driver; +use crate::sql_test_glue::_helpers::plan::planning::{PlanningRequest, plan_compiled_sql_tests}; use crate::sql_test_glue::models::PlanRow; use crate::sql_test_glue::models::SqlTestPlanOptions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Native plans of one SQL-test batch, with adapter rendering and timing accounted for. -#[inline] pub fn planned_rows<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), tests: &Bound<'py, PyAny>, options: &SqlTestPlanOptions<'py>, ) -> PyResult> { - plan_driver::planned_rows((project, adapter), tests, options) + let py = project.py(); + let artifact = options.artifact; + if !tests.is_truthy()? { + return Ok(Vec::new()); + } + let start: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let model_sql = crate::sql_test_glue::_helpers::plan::plan_rendering::models(project, adapter)?; + let functions = + crate::sql_test_glue::_helpers::plan::plan_rendering::functions(project, adapter)?; + let overrides = + crate::sql_test_glue::_helpers::plan::plan_rendering::overrides(project, tests, adapter)?; + let request = PlanningRequest { + models: native_field(project, "models")?.extract()?, + rendered_model_sql: model_sql.extract()?, + functions: functions.extract()?, + tests: tests.extract()?, + rendered_test_overrides: overrides.extract()?, + sql_analysis_enabled: options.sql_analysis_enabled.extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + requires_derived_table_aliases: adapter + .call_method0("requires_derived_table_aliases")? + .extract()?, + lexical_syntax: native_field( + &native_field(project, "sql_lexical_syntax")?, + "native_mapping", + )? + .extract()?, + render_sql: artifact || options.render_sql, + include_plan: !artifact && options.include_plan, + max_unit_test_real_models: native_field( + &native_field(project, "settings")?, + "unit_test_max_real_models", + )? + .extract()?, + }; + let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err( + crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( + py, error, tests, + )?, + ); + } + Err(error) => return Err(error), + }; + let end: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + for (phase, elapsed) in [ + ( + "test_planning_ms", + planning_ns + + end + .saturating_sub(start) + .saturating_sub(planning_ns) + .saturating_sub(rendering_ns), + ), + ("comparison_render_ms", rendering_ns), + ] { + add_compile_timing(phase, elapsed); + } + if artifact { + for row in &rows { + row.rendered_sql(py)?; + } + } + Ok(rows) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs index 8114d1917f..d961845f01 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs @@ -1,16 +1,22 @@ //! Return model closures resolved natively in one project batch, keyed by SQL test key. -use crate::sql_test_glue::_helpers::plan::plan_driver; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::sql_test_glue::_helpers::plan::plan_driver::resolve_sql_test_model_chains; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Return model closures resolved natively in one project batch, keyed by SQL test key. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, tests))] pub fn sql_test_model_chain_names_by_key<'py>( project: &Bound<'py, PyAny>, tests: &Bound<'py, PyAny>, ) -> PyResult> { - plan_driver::sql_test_model_chain_names_by_key(project, tests) + let py = project.py(); + let chains = resolve_sql_test_model_chains(project, tests)?; + let by_key = PyDict::new(py); + for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { + by_key.set_item(native_field(&test?, "key")?, chain?)?; + } + Ok(by_key) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs index a0a3391ef5..19b88aa2de 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs @@ -1,11 +1,10 @@ //! `sql_test_model_chain_names_by_key` as a function of `module`. -use crate::sql_test_glue::_helpers::plan::plan_driver; +use crate::sql_test_glue::main::sql_test_model_chain_names_by_key::sql_test_model_chain_names_by_key; /// `sql_test_model_chain_names_by_key` as a function of `module`. -#[inline] pub fn sql_test_model_chain_names_by_key_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - plan_driver::sql_test_model_chain_names_by_key_function(module) + pyo3::wrap_pyfunction!(sql_test_model_chain_names_by_key, module) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs index bfb26a146c..66a3326ea3 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs @@ -1,11 +1,10 @@ //! `sql_test_plan_error_messages` as a function of `module`. -use crate::sql_test_glue::_helpers::plan::plan_driver; +use crate::sql_test_glue::_helpers::plan::plan_driver::sql_test_plan_error_messages; /// `sql_test_plan_error_messages` as a function of `module`. -#[inline] pub fn sql_test_plan_error_messages_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - plan_driver::sql_test_plan_error_messages_function(module) + pyo3::wrap_pyfunction!(sql_test_plan_error_messages, module) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/models.rs b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs index 06681b9e6b..2bfa2fd81c 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/models.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs @@ -1,7 +1,6 @@ //! Public records of sql test glue. -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; use sqlbuild_analysis::compiler::models::SqlTestPlan; /// How one batch of SQL tests is planned. diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/types.rs b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs index b8d22b2788..6660825598 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/types.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs @@ -1,7 +1,6 @@ //! Public traits and type aliases of sql test glue. -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; use std::collections::BTreeMap; pub(crate) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs b/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs index 978b62389c..4f80f8b17e 100644 --- a/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs +++ b/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs @@ -1,8 +1,8 @@ //! Adapter nullability rules over `InferredNullability` arguments, decided by the native rules. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyString; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; use sqlbuild_analysis::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; @@ -34,7 +34,9 @@ fn argument_values(args: &Bound<'_, PyAny>, count: usize) -> PyResult(args: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn infer_first_arg_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { if !args.is_truthy()? { return enum_member(args.py(), RecordId::InferredNullability, UNKNOWN_MEMBER); } @@ -43,7 +45,7 @@ fn infer_first_arg_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn infer_conditional_result_nullability<'py>( args: &Bound<'py, PyAny>, ) -> PyResult> { let values: Vec = argument_values(args, args.len()?)?; @@ -53,7 +55,9 @@ fn infer_conditional_result_nullability<'py>( /// SAFE_CAST/TRY_CAST nullability without assuming the conversion succeeds. #[pyfunction] -fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn infer_safe_cast_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { let values: Vec = if args.is_truthy()? { argument_values(args, 1)? } else { @@ -62,13 +66,3 @@ fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult = values.iter().map(String::as_str).collect(); member(args, safe_cast_nullability(&values)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; - module.add_function(wrap_pyfunction!( - infer_conditional_result_nullability, - module - )?)?; - module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs index 83c5332bda..867b57758e 100644 --- a/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs +++ b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs @@ -1,12 +1,18 @@ //! Register the nullability functions and classes. -use crate::type_system::_helpers::nullability; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::type_system::_helpers::nullability::infer_conditional_result_nullability; +use crate::type_system::_helpers::nullability::infer_first_arg_nullability; +use crate::type_system::_helpers::nullability::infer_safe_cast_nullability; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the nullability functions and classes. -#[inline] pub fn register_nullability(module: &Bound<'_, PyModule>) -> PyResult<()> { - nullability::register(module) + module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; + module.add_function(wrap_pyfunction!( + infer_conditional_result_nullability, + module + )?)?; + module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 40645126b2..0a6d690dd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -30,20 +30,16 @@ use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_di use sqlbuild_python_project::analysis_session::main::return_types::return_types; use sqlbuild_python_project::analysis_session::main::shape_rows::shape_rows; use sqlbuild_python_project::analysis_session::main::start_model_analysis_session::start_model_analysis_session; -use sqlbuild_python_project::analysis_session::models::CacheStatsRow; -use sqlbuild_python_project::analysis_session::models::FamilyRow; -use sqlbuild_python_project::analysis_session::models::ModelRow; -use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; -use sqlbuild_python_project::analysis_session::models::RequestRow; +use sqlbuild_python_project::analysis_session::models::{ + CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, +}; use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; use sqlbuild_python_project::compiled_views::main::model_inputs::input_analysis_sql::input_analysis_sql; use sqlbuild_python_project::compiled_views::main::model_inputs::input_model_name::input_model_name; use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; use sqlbuild_python_project::compiled_views::main::model_inputs::input_sql_macro_checks::input_sql_macro_checks; -use sqlbuild_python_project::compiled_views::models::AttachedSchema; -use sqlbuild_python_project::compiled_views::models::NativeModelInput; -use sqlbuild_python_project::semantic_shapes::constants::VALID_FROM_DEFAULT; -use sqlbuild_python_project::semantic_shapes::constants::VALID_TO_DEFAULT; +use sqlbuild_python_project::compiled_views::models::{AttachedSchema, NativeModelInput}; +use sqlbuild_python_project::semantic_shapes::constants::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; const LOGGING_MODULE: &str = "logging"; const DEBUG_LOGGER: &str = "sqlbuild.compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 7decd6713c..774472d267 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -39,8 +39,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; use sqlbuild_python_project::analysis::main::ignores_quoted_case::ignores_quoted_case; -use sqlbuild_python_project::analysis::models::BindingCatalog; -use sqlbuild_python_project::analysis::models::CatalogKey; +use sqlbuild_python_project::analysis::models::{BindingCatalog, CatalogKey}; use sqlbuild_python_project::analysis_session::constants::ADAPTER_RULE_ID; use sqlbuild_python_project::analysis_session::main::expression_source_shapes::expression_source_shapes; use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 465c19a371..22b0b991b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -20,9 +20,9 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::models::HeaderFamilies; use sqlbuild_python_project::analysis_session::main::dialect_or_generic::dialect_or_generic; use sqlbuild_python_project::analysis_session::main::prove_dynamic_column_contracts::prove_dynamic_column_contracts; -use sqlbuild_python_project::analysis_session::models::FamilyRow; -use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; -use sqlbuild_python_project::analysis_session::models::PivotRequestRow; +use sqlbuild_python_project::analysis_session::models::{ + FamilyRow, NativeModelAnalysisSession, PivotRequestRow, +}; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 5d97463a57..a1ae3cec86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -22,9 +22,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; use sqlbuild_python_front::compile_driver::models::ProjectConfiguration; -use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_ENV; -use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_SETTING; -use sqlbuild_python_front::discovery::constants::TRUE_ENV_VALUES; +use sqlbuild_python_front::discovery::constants::{ + ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, +}; use sqlbuild_python_front::discovery::main::project::load_local_config_file::load_local_config_file; use sqlbuild_python_front::discovery::main::project::load_project_config_file::load_project_config_file; use sqlbuild_python_front::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 08a549b104..c2ca68e69e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -11,8 +11,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::models::CompiledProjectGraph; -use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::graph::models::{CompiledProjectGraph, NativeProjectGraph}; /// Discovered Python node functions whose selectors need the Python graph. const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index 54655ad6ed..ecf1b8c59c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -24,9 +24,9 @@ use crate::bindings::_helpers::consumption::facts::{ BuildFacts, MODEL, SEED, build_facts, enum_text, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::ConsumptionConnection; -use sqlbuild_python_front::native_host::models::Location; -use sqlbuild_python_front::native_host::models::NodeCompleteHook; +use sqlbuild_python_front::native_host::models::{ + ConsumptionConnection, Location, NodeCompleteHook, +}; /// Succeeded rows are flushed once this many executions are pending. const FLUSH_EXECUTIONS: usize = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 1187ed8461..185030d8e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -47,11 +47,11 @@ use crate::bindings::_helpers::compile_delta::driver::{ use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; use crate::bindings::_helpers::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; -use sqlbuild_python_front::attachments::constants::MICROBATCH_END_SENTINEL; -use sqlbuild_python_front::attachments::constants::MICROBATCH_START_SENTINEL; +use sqlbuild_python_front::attachments::constants::{ + MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, +}; use sqlbuild_python_front::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; -use sqlbuild_python_project::semantic_shapes::constants::VALID_FROM_DEFAULT; -use sqlbuild_python_project::semantic_shapes::constants::VALID_TO_DEFAULT; +use sqlbuild_python_project::semantic_shapes::constants::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; /// Text constants exported under their Python names. pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index b32e6d6f06..f433c73f53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -38,14 +38,13 @@ use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::headers::thresholds::thresholds; -use sqlbuild_python_front::model_config::models::AuditInstance; -use sqlbuild_python_front::model_config::models::HeaderAudits; use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::model_config::models::PyNode; -use sqlbuild_python_project::compiled_project::models::AuditBlockRow; -use sqlbuild_python_project::compiled_project::models::AuditFileRow; -use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_front::model_config::models::{AuditInstance, HeaderAudits}; +use sqlbuild_python_project::compiled_project::models::{ + AuditBlockRow, AuditFileRow, AuditSettings, +}; use sqlbuild_python_project::resource_views::models::AuditHandles; use sqlbuild_python_project::resource_views::types::AttachedAudit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 7fd436c750..b0fa440e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -44,8 +44,7 @@ use sqlbuild_python_base::compile_context::main::delta::record_source_reads::rec use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; -use sqlbuild_python_project::compiled_project::models::FunctionRow; -use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::{FunctionRow, FunctionSettings}; use sqlbuild_python_project::resource_views::models::FunctionHandles; use sqlbuild_python_project::resource_views::types::AttachedFunction; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 92f6014339..f086f52c52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -78,8 +78,7 @@ use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_python_front::discovery::models::ModelFileTexts; -use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; use sqlbuild_python_front::model_config::models::ConfigClasses; use sqlbuild_python_front::model_config::models::ContractClasses; @@ -87,12 +86,11 @@ use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::render::main::calls::span_row::span_row; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; use sqlbuild_python_front::render::models::ConstantRendering; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; -use sqlbuild_python_front::render::models::SqlCallOrigin; -use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; +use sqlbuild_python_front::render::models::{SqlCallOrigin, SqlMacroSource}; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_batch; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs index 502891c5c4..76248a4473 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs @@ -18,11 +18,11 @@ use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; use sqlbuild_python_front::model_config::main::config::config_fields::config_fields; use sqlbuild_python_front::model_config::main::config::view::view; -use sqlbuild_python_front::model_config::models::BuiltConfig; -use sqlbuild_python_front::model_config::models::ConfigBuild; use sqlbuild_python_front::model_config::models::ConfigClasses; -use sqlbuild_python_front::model_config::models::NativeModelConfigBuilder; use sqlbuild_python_front::model_config::models::PyNode; +use sqlbuild_python_front::model_config::models::{ + BuiltConfig, ConfigBuild, NativeModelConfigBuilder, +}; use sqlbuild_python_front::model_config::types::BuiltConfigValues; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs index ed82993a64..b1e5f7f3f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs @@ -13,8 +13,7 @@ use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_node::config_node; -use sqlbuild_python_front::model_config::models::HeaderColumn; -use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::{HeaderColumn, HeaderColumns}; use sqlbuild_python_front::model_config::types::PyConfigNode; const CASES_ATTRIBUTE: &str = "cases"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index b4213a78c9..c342a1607f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -40,10 +40,8 @@ use sqlbuild_python_front::model_config::main::headers::native_schema_column::na use sqlbuild_python_front::model_config::main::headers::parsed_audit_identity::parsed_audit_identity; use sqlbuild_python_front::model_config::main::model_nodes::column_migration_error::column_migration_error; use sqlbuild_python_front::model_config::models::HeaderAudits; -use sqlbuild_python_front::model_config::models::HeaderColumn; -use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::HeaderFamilies; -use sqlbuild_python_front::model_config::models::NativeColumnSpec; +use sqlbuild_python_front::model_config::models::{HeaderColumn, HeaderColumns, NativeColumnSpec}; use sqlbuild_python_front::model_config::types::HeaderValue; use sqlbuild_python_project::compiled_project::main::family_fact::family_fact; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs index 5ac32c69f0..73a36178f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs @@ -38,11 +38,11 @@ use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::argument_pairs::argument_pairs; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index 5a10f397dc..add77b1fbe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -21,8 +21,7 @@ use sqlbuild_project::models::ExpansionRecord; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_front::discovery::models::ModelFileTexts; -use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; use sqlbuild_python_front::model_config::main::config::retain_values::retain_values; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; @@ -32,18 +31,16 @@ use sqlbuild_python_front::render::models::ExpandedDeclarations; use sqlbuild_python_front::render::models::ExpansionOutput; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::models::ScannedReference; -use sqlbuild_python_project::compiled_project::constants::CONTRACT_KEY; -use sqlbuild_python_project::compiled_project::constants::INCREMENTAL_MODE_KEY; +use sqlbuild_python_project::compiled_project::constants::{CONTRACT_KEY, INCREMENTAL_MODE_KEY}; use sqlbuild_python_project::compiled_project::main::config_values::config_values; use sqlbuild_python_project::compiled_project::main::declarations::declarations; use sqlbuild_python_project::compiled_project::main::materialization::materialization; use sqlbuild_python_project::compiled_project::main::text_value::text_value; use sqlbuild_python_project::compiled_project::models::DeclarationPy; -use sqlbuild_python_project::compiled_views::models::InputProjections; -use sqlbuild_python_project::compiled_views::models::ModelInputHandles; -use sqlbuild_python_project::compiled_views::models::SchemaHandles; -use sqlbuild_python_project::compiled_views::types::ModelInputRecord; -use sqlbuild_python_project::compiled_views::types::SchemaRecord; +use sqlbuild_python_project::compiled_views::models::{ + InputProjections, ModelInputHandles, SchemaHandles, +}; +use sqlbuild_python_project::compiled_views::types::{ModelInputRecord, SchemaRecord}; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs index 99dff69f7d..7db7b0df27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs @@ -17,9 +17,7 @@ use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::expansion::expanded::expanded; -use sqlbuild_python_front::render::models::ExpansionOutput; -use sqlbuild_python_front::render::models::NativeMacroExpansion; -use sqlbuild_python_front::render::models::StateInput; +use sqlbuild_python_front::render::models::{ExpansionOutput, NativeMacroExpansion, StateInput}; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; /// Fields of `_VisibleModelDeclarations` a `DeclarationResolutionContext` is built from. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index e6c890408a..55d0611ed3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -11,18 +11,14 @@ use crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::compiled_project::models::AuditFileRow; -use sqlbuild_python_project::compiled_project::models::AuditSettings; -use sqlbuild_python_project::compiled_project::models::FunctionRow; -use sqlbuild_python_project::compiled_project::models::FunctionSettings; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_project::models::ResourceAttachment; -use sqlbuild_python_project::compiled_project::models::ScenarioRow; -use sqlbuild_python_project::compiled_project::models::SqlTestSettings; -use sqlbuild_python_project::compiled_project::models::TestFileRow; -use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; -use sqlbuild_python_project::compiled_project::types::SourceRow; -use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_project::models::{ + AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, ResourceAttachment, ScenarioRow, + SqlTestSettings, TestFileRow, +}; +use sqlbuild_python_project::compiled_project::types::{ + AttachedSqlTests, SourceRow, SourceSettings, +}; use sqlbuild_python_project::resource_views::models::NativeAuditInput; use sqlbuild_python_project::resource_views::models::NativeFunctionInput; use sqlbuild_python_project::resource_views::models::NativeSourceInput; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index fe58536011..8678d572a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -29,13 +29,11 @@ use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::models::NativeModelFile; -use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; -use sqlbuild_python_project::compiled_project::models::HookDescriptionRow; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_project::models::PythonResourceRow; -use sqlbuild_python_project::compiled_project::models::ScenarioDescriptionRow; -use sqlbuild_python_project::compiled_project::types::LoaderRow; -use sqlbuild_python_project::compiled_project::types::ResourceDiagnostics; +use sqlbuild_python_project::compiled_project::models::{ + DiagnosticSettings, HookDescriptionRow, PythonResourceRow, ScenarioDescriptionRow, +}; +use sqlbuild_python_project::compiled_project::types::{LoaderRow, ResourceDiagnostics}; use sqlbuild_python_project::resource_views::models::SeedParts; use sqlbuild_python_project::resource_views::types::SeedViews; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index ad171a1351..484c2e0534 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -22,21 +22,18 @@ use sqlbuild_python_front::model_config::main::config::expanded::expanded; use sqlbuild_python_front::model_config::main::config::native_config_error::native_config_error; use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; use sqlbuild_python_front::model_config::main::config::template_error::template_error; -use sqlbuild_python_front::model_config::models::PythonHost; -use sqlbuild_python_front::model_config::models::Stop; -use sqlbuild_python_front::model_config::models::TemplateSources; +use sqlbuild_python_front::model_config::models::{PythonHost, Stop, TemplateSources}; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; -use sqlbuild_python_project::compiled_project::types::SourceRow; -use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_project::types::{SourceRow, SourceSettings}; use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; use sqlbuild_python_project::resource_views::models::SourceHandles; use sqlbuild_python_project::resource_views::types::AttachedSource; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 6cb701b356..9f0b8e0f3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -48,9 +48,9 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::sql_macros::without_with_list_calls::without_with_list_calls; -use sqlbuild_python_project::compiled_project::models::SqlTestSettings; -use sqlbuild_python_project::compiled_project::models::TestBlockRow; -use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::models::{ + SqlTestSettings, TestBlockRow, TestFileRow, +}; use sqlbuild_python_project::compiled_project::types::TestCaseRow; use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index d553a72682..84d46233af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -16,9 +16,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::append_observations::append_observations; -use sqlbuild_python_front::native_host::models::ConsumptionConnection; -use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; +use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 403ee43e17..91bedada89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -46,8 +46,7 @@ use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::micro use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::functions::functions; use sqlbuild_python_project::planner_phases::main::object::object; -use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; -use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; +use sqlbuild_python_project::planner_strategy::models::{CursorInputLookups, ModelEntryDecisions}; use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics::source_binding_diagnostics; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index e7f376f2ad..820ea2335c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -25,9 +25,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_front::state_rows::constants::COLUMN_KIND; -use sqlbuild_python_front::state_rows::constants::MODEL_KIND; -use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; +use sqlbuild_python_front::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index d8b51c4d17..db3da0dc18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -45,10 +45,9 @@ use sqlbuild_python_project::planner_identity::main::execution_order::execution_ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_strategy::main::complete_fixtures::complete_fixtures as complete_relation_fixtures; use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; -use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_END; -use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_START; -use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; -use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; +use sqlbuild_python_project::sql_test_glue::constants::{ + DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, +}; #[pyfunction] #[pyo3(signature = (**kwargs))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index b3d3ebd8e8..6617268137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -45,10 +45,9 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::planner_strategy::constants::AUDIT_TYPES_MODULE; -use sqlbuild_python_project::planner_strategy::constants::COMPILE_TYPES_MODULE; -use sqlbuild_python_project::planner_strategy::constants::MODEL_ATTACHMENT; -use sqlbuild_python_project::planner_strategy::constants::SOURCE_ATTACHMENT; +use sqlbuild_python_project::planner_strategy::constants::{ + AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, +}; use sqlbuild_python_project::planner_strategy::models::AuditPlanning; /// One attribute read of a compiled audit. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index a2dd9a7f6a..37b06bfb0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -25,12 +25,10 @@ use sqlbuild_python_front::model_config::main::config::config_value::config_valu use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; -use sqlbuild_python_project::planner_strategy::constants::INCREMENTAL_MATERIALIZATION; -use sqlbuild_python_project::planner_strategy::constants::MODEL_REFERENCE; -use sqlbuild_python_project::planner_strategy::constants::SEED_REFERENCE; -use sqlbuild_python_project::planner_strategy::constants::SOURCE_REFERENCE; -use sqlbuild_python_project::planner_strategy::constants::TIMESTAMP_CURSOR; +use sqlbuild_python_project::planner_strategy::constants::{ + ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, + SOURCE_REFERENCE, TIMESTAMP_CURSOR, +}; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 831cad5042..2e40fecc2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -51,13 +51,10 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::model_name:: use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::compiled_views::main::compiled_models::relative_path::relative_path; -use sqlbuild_python_project::planner_strategy::constants::CUSTOM_MATERIALIZATION; -use sqlbuild_python_project::planner_strategy::constants::DDL_SEPARATOR; -use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; -use sqlbuild_python_project::planner_strategy::constants::MICROBATCH_MODE; -use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_DIALECT; -use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_UNSUPPORTED_CODE; -use sqlbuild_python_project::planner_strategy::constants::TABLE_RECREATING_ACTIONS; +use sqlbuild_python_project::planner_strategy::constants::{ + CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, + PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, TABLE_RECREATING_ACTIONS, +}; use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; /// Model config keys copied to the plan entry as optional strings, with their entry fields. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index df99de24ec..57da7f3ed6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -16,8 +16,7 @@ use sqlbuild_python_project::compiled_project::main::python_column_facts::python use sqlbuild_python_project::compiled_views::main::model_inputs::attached_columns::attached_columns; use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; use sqlbuild_python_project::resource_views::main::source_views::read_source_view::read_source_view; -use sqlbuild_python_project::resource_views::types::AttachedSource; -use sqlbuild_python_project::resource_views::types::SourceViews; +use sqlbuild_python_project::resource_views::types::{AttachedSource, SourceViews}; const UNKNOWN_TYPE: &str = "UNKNOWN"; const NOT_NULL_AUDIT_NAME: &str = "not_null"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c7a16a07ce..6f6cd2a808 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -50,9 +50,9 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::compiled_models::output_proven::output_proven; use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; -use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; -use sqlbuild_python_project::compiled_views::models::SqlRuleInputs; -use sqlbuild_python_project::compiled_views::models::SqlRuleKey; +use sqlbuild_python_project::compiled_views::models::{ + NativeCompiledModel, SqlRuleInputs, SqlRuleKey, +}; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 69f9af83f1..58eb210cb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -30,10 +30,8 @@ use sqlbuild_python_front::scopes::main::reports::scope_list::scope_list; use sqlbuild_python_front::scopes::main::reports::scope_move_preview::scope_move_preview; use sqlbuild_python_front::scopes::main::reports::scope_report::scope_report; use sqlbuild_python_front::scopes::main::reports::serialize_scope_report::serialize_scope_report; -use sqlbuild_python_front::scopes::models::ExplainArguments; -use sqlbuild_python_front::scopes::models::ListArguments; -use sqlbuild_python_front::scopes::models::ReportArguments; use sqlbuild_python_front::scopes::models::ScopeClasses; +use sqlbuild_python_front::scopes::models::{ExplainArguments, ListArguments, ReportArguments}; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ diff --git a/fensu.toml b/fensu.toml index 500065cff9..726c74e582 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1452,10 +1452,6 @@ reason = "Each turns the one Python exception its own call can expect into None; paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs"] reason = "Each builds one staying-Python object from its own keyword set." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs", "crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs"] -reason = "Each calls a class from its own lookup, a native attribute or a host entry, with keyword pairs." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-base/src/records/main/construct/new_record.rs"] reason = "The public entry forwards to the crate-internal constructor that the crate's own helpers call." @@ -1676,7 +1672,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147431 } +thresholds = { max_crate_code_lines = 147462 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2128,85 +2124,120 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registration.rs=26", "crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs=2", "crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs=3", - "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=1", - "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=3", + "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=0", + "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=2", + "crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs=1", + "crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs=1", "crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs=1", "crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs=3", "crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs=3", - "crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs=10", + "crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs=4", + "crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs=1", + "crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs=5", "crates/sqlbuild-python-front/src/authored_values/models.rs=2", - "crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs=4", + "crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs=1", + "crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs=3", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs=2", "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=2", - "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=9", - "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=2", + "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=3", + "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs=1", + "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs=5", + "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=0", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", - "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=7", - "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs=13", + "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=0", + "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs=11", "crates/sqlbuild-python-front/src/cursor/_helpers/values.rs=4", + "crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs=1", + "crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs=1", + "crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs=3", + "crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs=4", + "crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs=2", "crates/sqlbuild-python-front/src/cursor/models.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs=9", "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs=1", "crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs=4", "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs=16", - "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs=3", - "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs=0", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs=14", - "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=9", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=7", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs=15", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=8", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=6", "crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs=2", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs=5", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs=2", "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=14", + "crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs=6", + "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs=2", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs=2", + "crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs=1", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs=2", - "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=26", - "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs=5", + "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=23", + "crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs=2", + "crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs=3", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs=4", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs=2", - "crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs=4", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs=0", "crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs=3", + "crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs=1", + "crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs=1", + "crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs=3", + "crates/sqlbuild-python-front/src/model_config/main/config/view.rs=1", "crates/sqlbuild-python-front/src/model_config/models.rs=2", "crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs=6", "crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs=1", - "crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs=11", + "crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs=8", + "crates/sqlbuild-python-front/src/refactor/main/model_facts.rs=3", "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs=4", "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs=1", "crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs=2", "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs=24", "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs=1", - "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs=12", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs=7", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs=14", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs=6", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs=4", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs=9", + "crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs=3", + "crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs=2", "crates/sqlbuild-python-front/src/render/models.rs=10", "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs=5", - "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs=1", "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs=5", "crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs=15", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs=2", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs=24", - "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs=8", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs=6", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs=18", "crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs=4", "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs=3", - "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs=23", - "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs=20", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs=1", + "crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs=1", + "crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs=2", + "crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs=3", + "crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs=1", "crates/sqlbuild-python-front/src/scopes/models.rs=13", "crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs=2", "crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs=3", @@ -2215,36 +2246,59 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/sqltext/models.rs=1", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs=1", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs=4", - "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs=4", - "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs=7", - "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=1", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs=6", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=0", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=5", "crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs=1", - "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs=2", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs=1", + "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", + "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", "crates/sqlbuild-python-project/src/analysis/models.rs=1", "crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs=3", - "crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs=1", - "crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs=1", + "crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs=0", + "crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs=1", + "crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs=0", + "crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs=1", "crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs=4", "crates/sqlbuild-python-project/src/compiled_project/models.rs=2", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs=9", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs=7", - "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs=14", - "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs=6", - "crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs=15", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=0", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs=9", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs=1", + "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs=3", + "crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs=3", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs=13", "crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs=1", "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=16", "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs=1", + "crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs=2", "crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs=3", - "crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs=3", - "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs=2", + "crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs=1", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs=0", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs=3", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs=5", + "crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs=1", + "crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs=1", + "crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs=2", "crates/sqlbuild-python-project/src/graph/models.rs=2", - "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=3", + "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=1", + "crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs=2", "crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs=1", - "crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs=2", + "crates/sqlbuild-python-project/src/planner_phases/main/member.rs=1", + "crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs=1", "crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs=2", "crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs=6", "crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs=3", @@ -2252,24 +2306,29 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/planner_selection/models.rs=3", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=2", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs=4", - "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", - "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=19", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=2", + "crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs=1", + "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=18", "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=5", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs=2", - "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs=2", + "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs=0", + "crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs=1", + "crates/sqlbuild-python-project/src/references/main/render_source_relation.rs=2", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs=2", - "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs=1", + "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs=0", + "crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs=1", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs=1", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs=3", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs=13", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs=4", - "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs=8", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs=1", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs=4", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs=2", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs=2", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=1", + "crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", @@ -3158,26 +3217,10 @@ reason = "Each reads its own Python mapping into its own native pairs." paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs"] reason = "Each reads its own Python record's fields into its own native record." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs", "crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] reason = "Each bindings module registers its own functions and classes with the native module." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs", "crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs", "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs", "crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs", "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] -reason = "Each reads its own Python record's fields into its own native record." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs"] reason = "Each bindings module registers its own functions and classes with the native module." @@ -3186,10 +3229,6 @@ reason = "Each bindings module registers its own functions and classes with the paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs", "crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-front/src/model_config/models.rs", "crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs"] reason = "Each host keeps its own first Python error and returns its own failure type; only the string read is alike." @@ -3218,10 +3257,6 @@ reason = "Each reads an optional text value from its own Python mapping; consoli paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] reason = "Each getter reads its own cached projection through the view's shared read helper." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/_helpers/nodes/keys.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs"] -reason = "Each reads a resource key's type and name into its own native key; consolidating the key readers across the binding crates is a separate change." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] reason = "Each records its own per-resource fact into its own resource list by index." @@ -3275,16 +3310,300 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] reason = "The compiled model and the model input each read the reference rows of their own record." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs", "crates/sqlbuild-python-project/src/references/main/match_project_relation.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs"] +reason = "Each returns its own stored Python handle bound to the interpreter; only the bind-and-clone is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs", "crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs"] reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_phases/_helpers/objects.rs", "crates/sqlbuild-python-project/src/planner_phases/main/construct.rs"] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs"] reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs"] -reason = "Each returns its own stored Python handle bound to the interpreter; only the bind-and-clone is alike." +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs"] +reason = "Each reads a resource key's type and name into its own native key; consolidating the key readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] +reason = "Each reads its own Python record's fields into its own native record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/text.rs"] +reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/construct.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs"] +reason = "Each calls a class from its own lookup, a native attribute or a host entry, with keyword pairs." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs", "crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs", "crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs", "crates/sqlbuild-python-project/src/references/main/register_relations.rs", "crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] +reason = "Each reads the installed package version for its own output with its own fallback; only the metadata lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/native_model.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs"] +reason = "Each casts to its own native view class and counts the fallback; only the cast-and-count shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs"] +reason = "Each runs its own cursor operation in a cursor session; only the session set-up and conversion are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs"] +reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs", "crates/sqlbuild-python-project/src/graph/main/project/register_views.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs"] +reason = "One converts a typed SQL value to plain Python values, the other to JSON-ready values; only the kind dispatch is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +reason = "Each reads graph edges into its own key type; only the mapping walk is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs", "crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs"] +reason = "Each answers its own scope report request from the shared lookup reports; only reading the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs"] +reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs", "crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs", "crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs", "crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs"] +reason = "Each reads its own record's arguments to fill its own fields; only the argument reading is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs"] +reason = "Each discovers its own kind of Python file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs", "crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs"] +reason = "Each answers its own scope report request from the shared lookup reports; only reading the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs", "crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs"] +reason = "Each finds its own newest migration event in the state events; only the event walk is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/values.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs", "crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs"] +reason = "Each Python entry reads its own keyword options and calls its own scope report." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs"] +reason = "Each scans its own SQL texts for its own scope facts; only the scan set-up is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs"] +reason = "Each parses its own kind of SQL test file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs"] +reason = "Each crate strips a source expression to its relation name; consolidating these copies across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs", "crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs"] +reason = "Each loads its own project file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs", "crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs", "crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs", "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs", "crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs"] +reason = "Each discovers its own kind of project file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index ad2ad7572a..26f2188a9b 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -149,6 +149,7 @@ crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs PreparedCo crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs PythonNodeGraph.selected_sql_refs crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs PlannerScopeResolution.stale_warning_scope crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs CollectionRendering.value +crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs CollectionRendering.value crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs ConnectionProgressReporter.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs DbtInitProgressReporter.complete @@ -382,7 +383,9 @@ crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs Column crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs SqlValueKind.value crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs DeclarationKind.value crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs CollectionRendering.value +crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs CollectionRendering.value crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs VisibilityReason.value +crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs VisibilityReason.value crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs DeclarationKind.value crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs NativeMacroExpansion.render_relation crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs ResourceKind.value @@ -414,7 +417,7 @@ crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDia crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDiagnosticSeverity.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeKind.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs UsageKind.value -crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs ScopeDiagnosticCode.value +crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs ScopeDiagnosticCode.value crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.configure_resources crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.consume_resource_terminal From 84f5b510badefffa887dce93347bd36c8a6a50a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:59:16 +0000 Subject: [PATCH 08/10] wip: keep moved pyfunctions without doc comments, as before --- .../src/discovery/main/failures/explicit_references_help.rs | 1 - .../src/discovery/main/failures/microbatch_guidance.rs | 1 - .../src/sqltext/main/parse_model_headers.rs | 1 - .../src/analysis/main/normalize_analysis_sql.rs | 1 - .../src/planner_scope/main/build_intermediate_source_map.rs | 1 - .../src/planner_scope/main/upstream_loader_dependency_names.rs | 1 - 6 files changed, 6 deletions(-) diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs index 7921f405d0..811c4e2f4d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs @@ -3,7 +3,6 @@ use pyo3::pyfunction; use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; -/// Explicit references help. #[pyfunction] #[pyo3(signature = (*, allowed))] pub fn explicit_references_help(allowed: &str) -> String { diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs index f46db01825..15aab07517 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs @@ -3,7 +3,6 @@ use pyo3::pyfunction; use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; -/// Microbatch guidance. #[pyfunction] pub fn microbatch_guidance() -> (String, String) { concurrency_guidance() diff --git a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs index cecbed086b..cfdf9a884f 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs @@ -7,7 +7,6 @@ use pyo3::pyfunction; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; -/// Parse model headers. #[pyfunction] pub fn parse_model_headers( py: Python<'_>, diff --git a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs index 37407c64b2..7aa5f7d17d 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs @@ -5,7 +5,6 @@ use pyo3::pyfunction; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; -/// Normalize analysis sql. #[pyfunction] pub fn normalize_analysis_sql( py: Python<'_>, diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs index 807222a240..307f646797 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs @@ -5,7 +5,6 @@ use crate::planner_scope::main::keys::keys; use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; use pyo3::types::PyDict; -/// Build intermediate source map. #[pyfunction] #[pyo3(signature = (*, project, selected_keys))] pub fn build_intermediate_source_map<'py>( diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs index 911338e101..b65801d5ca 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs @@ -4,7 +4,6 @@ use crate::planner_scope::_helpers::values::{dependency_names, loader_graph, loa use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; use pyo3::types::PyTuple; -/// Upstream loader dependency names. #[pyfunction] #[pyo3(signature = (*, loader_function, loader_functions))] pub fn upstream_loader_dependency_names<'py>( From 22b4eeea7d42366403e2585e1e0b47aad536f5d6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:14:25 +0000 Subject: [PATCH 09/10] wip: name the long record tuples and satisfy clippy in the project crate --- .../src/analysis_session/models.rs | 12 ++------ .../src/analysis_session/types.rs | 14 ++++++++++ .../src/compiled_project/models.rs | 23 ++++----------- .../src/compiled_project/types.rs | 21 ++++++++++++++ .../src/compiled_views/models.rs | 13 ++------- .../src/compiled_views/types.rs | 13 +++++++++ .../project_snapshot/_helpers/snapshot/mod.rs | 2 +- .../_helpers/snapshot/snapshot_boundary.rs | 2 +- .../_helpers/snapshot/snapshot_models.rs | 2 +- .../{snapshot.rs => snapshot_project.rs} | 0 .../_helpers/snapshot/snapshot_resources.rs | 6 ++-- fensu.toml | 28 +++++++++++++------ 12 files changed, 85 insertions(+), 51 deletions(-) rename crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/{snapshot.rs => snapshot_project.rs} (100%) diff --git a/crates/sqlbuild-python-project/src/analysis_session/models.rs b/crates/sqlbuild-python-project/src/analysis_session/models.rs index 011c23a073..2180433bb3 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/models.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/models.rs @@ -15,6 +15,7 @@ use crate::analysis_session::_helpers::session::internal_error; use crate::analysis_session::_helpers::session::outcome_row; use crate::analysis_session::_helpers::session::pivot_model; use crate::analysis_session::_helpers::session::step_row; +use crate::analysis_session::types::PivotRequestTuple; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; @@ -56,7 +57,7 @@ use std::sync::Mutex; /// Reuse inference for identical fixture SQL under an equal, catalog-free profile. #[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] +#[derive(Debug, Default)] pub struct FixtureColumnInferences { pub(crate) by_profile: Mutex, Py)>>, } @@ -262,14 +263,7 @@ impl<'py> pyo3::FromPyObject<'py> for PivotRequestRow { column_nullability, families_by_table, models, - ): ( - String, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, - ) = pyo3::types::PyAnyMethods::extract(value)?; + ): PivotRequestTuple = pyo3::types::PyAnyMethods::extract(value)?; Ok(Self { dialect, column_types, diff --git a/crates/sqlbuild-python-project/src/analysis_session/types.rs b/crates/sqlbuild-python-project/src/analysis_session/types.rs index 2441ae1ff7..8df6a0949c 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/types.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/types.rs @@ -1,4 +1,18 @@ //! Public traits and type aliases of analysis session. +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::models::FamilyRow; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; + /// `(succeeded, [(name, type, nullability)] or None, has star)`. pub type ColumnsRow = (bool, Option, String)>>, bool); + +/// The tuple Python sends for a `PivotRequestRow`, in field order. +pub type PivotRequestTuple = ( + String, + Shapes, + Shapes, + Shapes, + Vec<(String, Vec)>, + Vec, +); diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs index 55b3e55438..73eb20c543 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -11,10 +11,12 @@ use crate::compiled_project::_helpers::project::INPUT_CLASSES_UNSET; use crate::compiled_project::_helpers::project::ModelTextsPy; use crate::compiled_project::_helpers::project::NamedType; use crate::compiled_project::types::AttachedSqlTests; +use crate::compiled_project::types::FunctionSettingsTuple; use crate::compiled_project::types::LoaderRow; use crate::compiled_project::types::ResourceDiagnostics; use crate::compiled_project::types::SourceRow; use crate::compiled_project::types::SourceSettings; +use crate::compiled_project::types::TestBlockTuple; use crate::compiled_project::types::TestCaseRow; use crate::compiled_views::models::CompiledViewClasses; use crate::compiled_views::models::NativeCompiledModel; @@ -122,14 +124,8 @@ pub struct FunctionSettings { impl<'py> pyo3::FromPyObject<'py> for FunctionSettings { fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { - let (adapter_name, validate, defaults, target, inherit, kind): ( - String, - bool, - (Option, Option), - Option<(Option, Option)>, - bool, - Py, - ) = pyo3::types::PyAnyMethods::extract(value)?; + let (adapter_name, validate, defaults, target, inherit, kind): FunctionSettingsTuple = + pyo3::types::PyAnyMethods::extract(value)?; Ok(Self { adapter_name, validate, @@ -155,15 +151,8 @@ pub struct TestBlockRow { impl<'py> pyo3::FromPyObject<'py> for TestBlockRow { fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { - let (block, name, sql_body, mode, parameters, cases, sql_body_span): ( - Py, - Option, - String, - String, - Vec, - Vec, - Option<(usize, usize)>, - ) = pyo3::types::PyAnyMethods::extract(value)?; + let (block, name, sql_body, mode, parameters, cases, sql_body_span): TestBlockTuple = + pyo3::types::PyAnyMethods::extract(value)?; Ok(Self { block, name, diff --git a/crates/sqlbuild-python-project/src/compiled_project/types.rs b/crates/sqlbuild-python-project/src/compiled_project/types.rs index 2a472bb98f..9254d9b6ea 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/types.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/types.rs @@ -25,3 +25,24 @@ pub type ResourceDiagnostics<'py> = ( Vec>, Vec>, ); + +/// The tuple Python sends for a `FunctionSettings`, in field order. +pub type FunctionSettingsTuple = ( + String, + bool, + (Option, Option), + Option<(Option, Option)>, + bool, + Py, +); + +/// The tuple Python sends for a `TestBlockRow`, in field order. +pub type TestBlockTuple = ( + Py, + Option, + String, + String, + Vec, + Vec, + Option<(usize, usize)>, +); diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index f078f57117..28438e56d5 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -43,6 +43,7 @@ use crate::compiled_views::main::model_inputs::view_classes::view_classes; use crate::compiled_views::types::DeclaredRow; use crate::compiled_views::types::LineageInput; use crate::compiled_views::types::ModelInputRecord; +use crate::compiled_views::types::RelationKeyTuple; use crate::graph::main::project::project_graph::project_graph; use crate::graph::models::NativeProjectGraph; use crate::graph::models::ResourceRow; @@ -1656,17 +1657,7 @@ pub struct RelationKeyRow { impl<'py> pyo3::FromPyObject<'py> for RelationKeyRow { fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { - let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits): ( - String, - String, - String, - String, - String, - bool, - Option>, - Vec<(String, Vec, bool)>, - Vec<(String, Option)>, - ) = pyo3::types::PyAnyMethods::extract(value)?; + let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits): RelationKeyTuple = pyo3::types::PyAnyMethods::extract(value)?; Ok(Self { kind, name, diff --git a/crates/sqlbuild-python-project/src/compiled_views/types.rs b/crates/sqlbuild-python-project/src/compiled_views/types.rs index 5836b137ee..0bb954a875 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/types.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/types.rs @@ -17,3 +17,16 @@ pub type ModelInputRecord = project_models::ModelInputRecord; /// Declared keys by `kind:name` identity, in identity order. pub type RelationKeys = Vec<(String, Vec>)>; + +/// The tuple Python sends for a `RelationKeyRow`, in field order. +pub type RelationKeyTuple = ( + String, + String, + String, + String, + String, + bool, + Option>, + Vec<(String, Vec, bool)>, + Vec<(String, Option)>, +); diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs index d930a14b58..2f10cb6b41 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs @@ -1,8 +1,8 @@ -pub(crate) mod snapshot; pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_execution; pub(crate) mod snapshot_models; +pub(crate) mod snapshot_project; pub(crate) mod snapshot_resources; pub(crate) mod snapshot_roles; pub(crate) mod snapshot_worker; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs index 36c1166f24..38373ed435 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs @@ -16,7 +16,7 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot::{destination, insert_node}; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{destination, insert_node}; use crate::resource_views::models::NativeCompiledSeed; use crate::resource_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs index 6f70a31c5f..44aeaa81e2 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs @@ -17,7 +17,7 @@ use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeProjectView; use crate::compiled_views::types::ModelInputRecord; -use crate::project_snapshot::_helpers::snapshot::snapshot::{ +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ dependency_keys, destination, insert_node, }; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_project.rs similarity index 100% rename from crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_project.rs diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs index eb9baf9ece..c7791c304a 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs @@ -11,12 +11,12 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot::{ - dependency_keys, destination, insert_node, -}; use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::{ boundary_nodes, columns, metadata, }; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ + dependency_keys, destination, insert_node, +}; use crate::resource_views::models::NativeCompiledFunction; pub(crate) fn resource_nodes( diff --git a/fensu.toml b/fensu.toml index 726c74e582..03b7bf1ba6 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1790,16 +1790,34 @@ path = "crates/sqlbuild-python-front/src/render/types.rs" symbols = ["StateTuple"] reason = "Python passes the expansion state as one five-item tuple; StateInput reads it in a single tuple extraction and names each position, as the positional record it replaces did." +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/analysis_session/types.rs" +symbols = ["PivotRequestTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_project/types.rs" +symbols = ["FunctionSettingsTuple", "TestBlockTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_views/types.rs" +symbols = ["RelationKeyTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + [[targets.rust.rule_exceptions]] rule = "FPRSS012" path = "crates/sqlbuild-python-project/src/analysis_session/models.rs" -symbols = ["FamilyRow::extract_bound", "PivotRequestRow::extract_bound"] +symbols = ["FamilyRow::extract_bound"] reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." [[targets.rust.rule_exceptions]] rule = "FPRSS012" path = "crates/sqlbuild-python-project/src/compiled_project/models.rs" -symbols = ["FunctionRow::extract_bound", "FunctionSettings::extract_bound", "TestBlockRow::extract_bound", "TestFileRow::extract_bound", "ScenarioRow::extract_bound", "AuditBlockRow::extract_bound", "AuditFileRow::extract_bound", "PythonResourceRow::extract_bound", "ScenarioDescriptionRow::extract_bound", "HookDescriptionRow::extract_bound", "ProofPy::extract_bound", "AnalysisPy::extract_bound"] +symbols = ["FunctionRow::extract_bound", "TestFileRow::extract_bound", "ScenarioRow::extract_bound", "AuditBlockRow::extract_bound", "AuditFileRow::extract_bound", "PythonResourceRow::extract_bound", "ScenarioDescriptionRow::extract_bound", "HookDescriptionRow::extract_bound", "ProofPy::extract_bound", "AnalysisPy::extract_bound"] reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." [[targets.rust.rule_exceptions]] @@ -1808,12 +1826,6 @@ path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" symbols = ["AnalysisProjections", "CompiledProjections", "ModelInputHandles", "InputProjections", "NativeProjectView", "SchemaHandles"] reason = "AnalysisProjections, CompiledProjections, ModelInputHandles, InputProjections, NativeProjectView implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; SchemaHandles implements it manually so the user-authored audit arguments its specs carry stay out of logs; other models derive it." -[[targets.rust.rule_exceptions]] -rule = "FPRSS012" -path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" -symbols = ["RelationKeyRow::extract_bound"] -reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." - [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-python-project/src/graph/models.rs" From 2c6a81e295a2292b66000fd921586cdf887db716 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:14:50 +0000 Subject: [PATCH 10/10] wip: allowlist the shortened record conversions --- fensu.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fensu.toml b/fensu.toml index 03b7bf1ba6..2337af0e2c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3617,6 +3617,10 @@ reason = "Each converts its own input into its own native or Python value; only paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults."