From de6301bbda4eb7b3760c90e86df033c87f55b3b2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 04:14:09 +0000 Subject: [PATCH] refactor: move the shared runtime of the Python bindings into sqlbuild-python-base The native store, compile context, diagnostics, progress and terminal text, warehouse access, orchestration reads and the other infra-layer helper modules move into the base crate, one area per helper bucket. sqlbuild-python imports their entry points directly; the extension module surface is unchanged. --- Cargo.lock | 9 + crates/README.md | 6 +- crates/sqlbuild-python-base/Cargo.toml | 9 + .../src/boundary/_helpers/mod.rs | 1 + .../src/boundary/_helpers/text_units.rs | 18 + .../src/boundary/main/mod.rs | 1 + .../src/boundary/main/utf16_units.rs | 11 + .../src/build_command/_helpers/mod.rs | 1 + .../src/build_command/_helpers}/support.rs | 2 +- .../src/build_command/main/boolean.rs | 12 + .../src/build_command/main/display_text.rs | 12 + .../src/build_command/main/emit.rs | 12 + .../src/build_command/main/frozenset.rs | 16 + .../src/build_command/main/joined.rs | 12 + .../src/build_command/main/keyword_relay.rs | 15 + .../src/build_command/main/line_writer.rs | 12 + .../src/build_command/main/mod.rs | 12 + .../src/build_command/main/none.rs | 12 + .../src/build_command/main/or_none.rs | 15 + .../src/build_command/main/text.rs | 12 + .../src/build_command/main/tuple.rs | 13 + .../src/build_command/main/utc_now.rs | 13 + .../src/build_command/mod.rs | 4 + .../src/build_progress/_helpers/mod.rs | 4 + .../src/build_progress/_helpers}/progress.rs | 223 +----- .../src/build_progress/_helpers}/terminal.rs | 18 +- .../_helpers/transient_lines.rs | 32 + .../_helpers/transient_status.rs | 96 +++ .../src/build_progress/constants.rs | 14 + .../src/build_progress/main/mod.rs | 4 + .../current_native_progress_projector.rs | 13 + .../main/progress/current_projector.rs | 13 + .../main/progress/machine_output_mode.rs | 12 + .../src/build_progress/main/progress/mod.rs | 7 + .../progress/projector_operation_active.rs | 14 + .../main/progress/register_progress.rs | 12 + .../main/progress/terminal_request.rs | 16 + .../main/progress/write_persistent.rs | 13 + .../src/build_progress/main/terminal/claim.rs | 16 + .../main/terminal/consume_terminal.rs | 16 + .../src/build_progress/main/terminal/flush.rs | 12 + .../main/terminal/join_spinner.rs | 12 + .../src/build_progress/main/terminal/mod.rs | 12 + .../build_progress/main/terminal/projector.rs | 13 + .../main/terminal/register_terminal.rs | 12 + .../build_progress/main/terminal/release.rs | 12 + .../main/terminal/shared_lines.rs | 13 + .../main/terminal/start_spinner.rs | 14 + .../main/terminal/terminal_columns.rs | 11 + .../build_progress/main/terminal/tty_flag.rs | 12 + .../src/build_progress/main/terminal/write.rs | 12 + .../main/transient_lines/mod.rs | 2 + .../register_transient_lines.rs | 12 + .../shared_transient_line_coordinator.rs | 15 + .../main/transient_status/delayed_status.rs | 16 + .../main/transient_status/maybe_status.rs | 15 + .../main/transient_status/mod.rs | 5 + .../progress_spinners_disabled.rs | 9 + .../register_transient_status.rs | 12 + .../main/transient_status/status_reporter.rs | 15 + .../src/build_progress/mod.rs | 6 + .../src/build_progress/models.rs | 653 ++++++++++++++++++ .../src/cache/_helpers/mod.rs | 3 + .../src/cache/_helpers/native_parse_store.rs | 22 + .../src/cache/_helpers}/native_store.rs | 8 +- .../src/cache/_helpers/sql_test_scan_store.rs | 39 ++ .../src/cache/main/content_digest.rs | 9 + .../src/cache/main/current_code_identity.rs | 11 + .../src/cache/main/digest_files.rs | 12 + .../src/cache/main/mod.rs | 8 + .../src/cache/main/open_store.rs | 18 + .../src/cache/main/parse_key.rs | 11 + .../src/cache/main/register_native_store.rs | 12 + .../main/register_sql_test_scan_store.rs | 12 + .../src/cache/main/save_store.rs | 18 + crates/sqlbuild-python-base/src/cache/mod.rs | 5 + .../src/cache/models.rs} | 129 ++-- .../src/cli/_helpers}/launch_state.rs | 0 .../src/cli/_helpers/mod.rs | 1 + .../src/cli/main/launch_record_path.rs | 13 + .../sqlbuild-python-base/src/cli/main/mod.rs | 4 + .../src/cli/main/offer_launch_record.rs | 13 + .../src/cli/main/register_launch_state.rs | 12 + .../src/cli/main/withhold_launch_record.rs | 11 + crates/sqlbuild-python-base/src/cli/mod.rs | 4 + .../src/command_runtime/_helpers}/adapters.rs | 10 +- .../src/command_runtime/_helpers/mod.rs | 1 + .../src/command_runtime/main/mod.rs | 2 + .../command_runtime/main/register_adapters.rs | 12 + .../main/resolve_cli_adapter.rs | 15 + .../src/command_runtime/mod.rs | 4 + .../src/compile_context/_helpers}/coverage.rs | 2 +- .../src/compile_context/_helpers/delta.rs | 250 +++++++ .../_helpers}/diagnostic_collector.rs | 83 +-- .../src/compile_context/_helpers}/getters.rs | 22 +- .../compile_context/_helpers/image_codec.rs | 22 + .../compile_context/_helpers}/image_facts.rs | 5 +- .../compile_context/_helpers}/input_reads.rs | 54 +- .../src/compile_context/_helpers}/mod.rs | 3 - .../compile_context/_helpers}/yaml_cache.rs | 83 +-- .../src/compile_context/constants.rs | 23 + .../main/coverage/coverage_reasons.rs | 13 + .../main/coverage/merged_coverage.rs | 10 + .../main/coverage/merged_order.rs | 12 + .../src/compile_context/main/coverage/mod.rs | 3 + .../src/compile_context/main/delta/capture.rs | 10 + .../compile_context/main/delta/capturing.rs | 9 + .../main/delta/delta_unsupported.rs | 9 + .../src/compile_context/main/delta/guarded.rs | 10 + .../main/delta/inject_relations.rs | 10 + .../main/delta/injected_fact_digest.rs | 9 + .../compile_context/main/delta/injection.rs | 11 + .../src/compile_context/main/delta/mod.rs | 20 + .../main/delta/record_source_reads.rs | 9 + .../main/delta/record_test_source_reads.rs | 9 + .../src/compile_context/main/delta/reset.rs | 9 + .../main/delta/set_pending_image.rs | 10 + .../main/delta/set_record_merge.rs | 10 + .../main/delta/start_capture.rs | 9 + .../main/delta/stop_capture.rs | 9 + .../main/delta/stop_on_errors.rs | 9 + .../main/delta/take_capture.rs | 10 + .../main/delta/take_delta_unsupported.rs | 9 + .../main/delta/take_pending_image.rs | 10 + .../main/delta/take_record_merge.rs | 10 + .../main/delta/with_injection.rs | 13 + .../collect_compile_diagnostics.rs | 15 + .../main/diagnostic_collector/mod.rs | 4 + .../register_diagnostic_collector.rs | 12 + .../report_compile_diagnostic.rs | 15 + .../report_compile_diagnostic_function.rs | 13 + .../compile_context/main/image_facts/mod.rs | 1 + .../main/image_facts/record_model_usages.rs | 15 + .../main/input_reads/context_read.rs | 11 + .../main/input_reads/environment_read.rs | 11 + .../compile_context/main/input_reads/mod.rs | 5 + .../input_reads/record_compile_input_reads.rs | 10 + .../input_reads/record_settings_class_read.rs | 12 + .../main/input_reads/register_input_reads.rs | 12 + .../src/compile_context/main/mod.rs | 6 + .../compile_context/main/yaml_cache/mod.rs | 4 + .../main/yaml_cache/open_documents.rs | 10 + .../main/yaml_cache/retain_read_sources.rs | 11 + .../main/yaml_cache/save_documents.rs | 10 + .../main/yaml_cache/yaml_store_path.rs | 10 + .../src/compile_context/mod.rs | 10 + .../src/compile_context/models.rs | 549 +++++++++++++++ .../src/compile_context/tests.rs | 6 + .../src/compile_context/tests/helpers.rs | 6 + .../src/compile_context/tests/test_types.rs | 9 + .../compile_context}/tests/test_yaml_cache.rs | 6 +- .../src/compile_context/types.rs | 4 + .../src/compile_driver/_helpers/mod.rs | 1 + .../src/compile_driver/_helpers/stage_work.rs | 57 ++ .../src/compile_driver/constants.rs | 57 ++ .../src/compile_driver/main/count_stage.rs | 11 + .../src/compile_driver/main/mod.rs | 2 + .../main/register_stage_work.rs | 12 + .../src/compile_driver/mod.rs | 5 + .../compile_outputs/_helpers}/json_report.rs | 27 +- .../src/compile_outputs/_helpers/mod.rs | 1 + .../compile_outputs/main/emit_json_report.rs | 12 + .../main/emit_orjson_report.rs | 12 + .../src/compile_outputs/main/float_key.rs | 12 + .../src/compile_outputs/main/indented_json.rs | 12 + .../src/compile_outputs/main/mod.rs | 8 + .../src/compile_outputs/main/orjson_sorted.rs | 11 + .../src/compile_outputs/main/orjson_value.rs | 13 + .../src/compile_outputs/main/plain_json.rs | 12 + .../main/register_json_report.rs | 12 + .../src/compile_outputs/mod.rs | 4 + .../src/contracts/_helpers/mod.rs | 2 + .../_helpers}/resource_identities.rs | 4 +- .../src/contracts/_helpers}/state_literals.rs | 11 +- .../src/contracts/main/mod.rs | 4 + .../main/register_resource_identities.rs | 12 + .../contracts/main/register_state_literals.rs | 12 + .../main/render_state_sql_literal.rs | 18 + .../main/validate_resource_identity.rs | 18 + .../sqlbuild-python-base/src/contracts/mod.rs | 4 + .../diagnostics/_helpers}/console_records.rs | 2 +- .../src/diagnostics/_helpers}/getters.rs | 12 +- .../src/diagnostics/_helpers}/log_records.rs | 12 +- .../src/diagnostics/_helpers/mod.rs | 5 + .../src/diagnostics/_helpers}/redaction.rs | 4 +- .../src/diagnostics/_helpers}/trackers.rs | 111 +-- .../src/diagnostics/constants.rs | 12 + .../src/diagnostics/main/current_tracker.rs | 13 + .../src/diagnostics/main/mod.rs | 5 + .../main/register_console_records.rs | 12 + .../diagnostics/main/register_log_records.rs | 12 + .../diagnostics/main/register_redaction.rs | 12 + .../src/diagnostics/main/register_trackers.rs | 12 + .../src/diagnostics/mod.rs | 6 + .../src/diagnostics/models.rs | 102 +++ .../src/event_exporting/_helpers/mod.rs | 2 + .../src/event_exporting/_helpers/scope.rs | 237 +++++++ .../src/event_exporting/_helpers}/values.rs | 11 +- .../src/event_exporting/constants.rs | 38 + .../configure_discovered_event_exporters.rs | 12 + .../src/event_exporting/main/current_scope.rs | 13 + .../src/event_exporting/main/error_type_of.rs | 11 + .../main/event_exporter_command_scope.rs | 12 + .../src/event_exporting/main/export_policy.rs | 12 + .../src/event_exporting/main/input_error.rs | 11 + .../src/event_exporting/main/mod.rs | 10 + .../event_exporting/main/register_scope.rs | 12 + .../src/event_exporting/main/severity_from.rs | 13 + .../event_exporting/main/severity_object.rs | 14 + .../src/event_exporting/main/state_error.rs | 11 + .../src/event_exporting/mod.rs | 6 + .../src/event_exporting/models.rs | 645 +++++++++++++++++ .../src/frontier/_helpers/encoder.rs | 157 +++++ .../src/frontier/_helpers/mod.rs | 2 + .../src/frontier/_helpers}/python_types.rs | 7 +- .../src/frontier/main/emit_error.rs | 11 + .../src/frontier/main/mod.rs | 1 + .../sqlbuild-python-base/src/frontier/mod.rs | 5 + .../src/frontier/models.rs} | 248 ++----- crates/sqlbuild-python-base/src/lib.rs | 20 + .../src/native_host/_helpers}/connections.rs | 2 +- .../src/native_host/_helpers/host.rs | 7 + .../src/native_host/_helpers/interrupt.rs | 25 + .../src/native_host/_helpers/mod.rs | 3 + .../src/native_host/_helpers}/results.rs | 12 +- .../src/native_host/constants.rs | 29 + .../src/native_host/main/check_signals.rs | 11 + .../native_host/main/connection_failure.rs | 12 + .../src/native_host/main/host_error.rs | 12 + .../src/native_host/main/interrupted.rs | 9 + .../src/native_host/main/mod.rs | 4 + .../src/native_host/mod.rs | 3 + .../src/native_host/models.rs | 145 ++++ .../src/native_host/types.rs | 10 + .../native_threads/_helpers}/exit_signal.rs | 0 .../_helpers}/interruptible_join.rs | 11 +- .../src/native_threads/_helpers}/mod.rs | 1 - .../native_threads/_helpers/reentrant_lock.rs | 29 + .../src/native_threads/constants.rs | 4 + .../main/interpreter_exiting.rs | 9 + .../native_threads/main/join_interruptibly.rs | 16 + .../src/native_threads/main/mod.rs | 3 + .../main/signal_interpreter_exit.rs | 9 + .../src/native_threads/mod.rs | 6 + .../src/native_threads/models.rs | 57 ++ .../src/orchestration_reads/_helpers/mod.rs | 3 + .../_helpers}/orchestration_declarations.rs | 65 +- .../_helpers}/orchestration_environment.rs | 8 +- .../_helpers}/provider_settings.rs | 21 +- .../main/callback_code_hash.rs | 12 + .../orchestration_reads/main/declared_at.rs | 15 + .../main/environment_hash.rs | 16 + .../main/imported_declarations.rs | 17 + .../main/missing_secrets.rs | 12 + .../src/orchestration_reads/main/mod.rs | 14 + .../main/module_declarations.rs | 18 + .../main/orchestration_code_hash.rs | 11 + .../main/provider_class_settings.rs | 13 + .../main/provider_state.rs | 13 + .../register_orchestration_declarations.rs | 12 + .../orchestration_reads/main/run_controls.rs | 11 + .../orchestration_reads/main/run_resources.rs | 15 + .../main/secret_variables.rs | 12 + .../orchestration_reads/main/setting_rows.rs | 10 + .../src/orchestration_reads/mod.rs | 5 + .../src/orchestration_reads/models.rs | 67 ++ .../planner_identity/_helpers}/json_values.rs | 7 +- .../src/planner_identity/_helpers/mod.rs | 1 + .../src/planner_identity/constants.rs | 7 + .../src/planner_identity/main/mod.rs | 2 + .../src/planner_identity/main/sorted_dumps.rs | 16 + .../main/unicode_sorted_dumps.rs | 15 + .../src/planner_identity/mod.rs | 5 + .../presentation_text/_helpers}/document.rs | 2 +- .../presentation_text/_helpers}/functions.rs | 4 +- .../presentation_text/_helpers}/getters.rs | 12 +- .../src/presentation_text/_helpers/mod.rs | 5 + .../src/presentation_text/_helpers/style.rs | 8 + .../presentation_text/_helpers}/terminal.rs | 0 .../src/presentation_text/constants.rs | 12 + .../src/presentation_text/main/mod.rs | 6 + .../main/register_document.rs | 12 + .../main/register_functions.rs | 12 + .../presentation_text/main/register_style.rs | 12 + .../main/register_terminal.rs | 12 + .../presentation_text/main/stdout_is_tty.rs | 11 + .../main/stdout_supports_color.rs | 11 + .../src/presentation_text/mod.rs | 6 + .../src/presentation_text/models.rs | 130 ++++ .../src/python_values/_helpers}/mod.rs | 0 .../src/python_values/_helpers}/values.rs | 30 +- .../src/python_values/main/mod.rs | 1 + .../src/python_values/main/python_value.rs | 13 + .../src/python_values/mod.rs | 4 + .../_helpers/native_classes/construct.rs | 7 + .../src/records/main/construct/mod.rs | 1 + .../records/main/construct/new_plain_error.rs | 12 + .../_helpers}/integration_values.rs | 18 +- .../src/result_output/_helpers/mod.rs | 1 + .../result_output/main/bounded_metadata.rs | 13 + .../src/result_output/main/json_value.rs | 13 + .../src/result_output/main/mod.rs | 7 + .../src/result_output/main/optional_text.rs | 13 + .../src/result_output/main/required_json.rs | 13 + .../src/result_output/main/text.rs | 10 + .../src/result_output/main/text_or_empty.rs | 13 + .../result_output/main/validation_error.rs | 12 + .../src/result_output/mod.rs | 5 + .../src/result_output/types.rs | 7 + .../src/type_system/_helpers}/equality.rs | 8 +- .../src/type_system/_helpers/mod.rs | 2 + .../type_system/_helpers}/normalization.rs | 42 +- .../src/type_system/main/mod.rs | 3 + .../type_system/main/normalization_error.rs | 11 + .../main/register_normalization.rs | 12 + .../src/type_system/main/types_equal.rs | 12 + .../src/type_system/mod.rs | 4 + .../src/warehouse_access/_helpers/mod.rs | 5 + .../_helpers/sqlite_connection.rs | 49 ++ .../_helpers}/state_column_evolution.rs | 6 +- .../warehouse_access/_helpers/synced_copy.rs | 109 +++ .../_helpers}/warehouse_catalog.rs | 4 +- .../_helpers}/warehouse_statistics.rs | 4 +- .../main/ensure_event_id_column.rs | 26 + .../main/is_warehouse_access_refused.rs | 11 + .../src/warehouse_access/main/mod.rs | 6 + .../src/warehouse_access/main/read_catalog.rs | 19 + .../warehouse_access/main/read_statistics.rs | 19 + .../main/register_synced_copy.rs | 12 + .../warehouse_access/main/sqlite_params.rs | 12 + .../src/warehouse_access/mod.rs | 5 + .../src/warehouse_access/models.rs} | 436 ++++++++---- .../analysis_session/model_analyses.rs | 5 +- .../_helpers/analysis_session/session.rs | 17 +- .../analysis_session/stage_functions.rs | 6 +- .../_helpers/audit_command/command.rs | 4 +- .../bindings/_helpers/auditing/measurement.rs | 7 +- .../_helpers/authored_values/project_vars.rs | 7 +- .../_helpers/authored_values/python_nodes.rs | 6 +- .../_helpers/build_command/command.rs | 5 +- .../_helpers/build_command/completion.rs | 9 +- .../_helpers/build_command/execution.rs | 15 +- .../_helpers/build_command/invocation.rs | 8 +- .../_helpers/build_command/lifecycle.rs | 9 +- .../bindings/_helpers/build_command/mod.rs | 1 - .../_helpers/build_command/planning.rs | 8 +- .../_helpers/build_progress/audit_progress.rs | 15 +- .../build_progress/build_callbacks.rs | 21 +- .../_helpers/build_progress/getters.rs | 2 +- .../bindings/_helpers/build_progress/mod.rs | 4 - .../build_progress/nested_progress.rs | 21 +- .../build_progress/progress_reporters.rs | 9 +- .../_helpers/build_progress/progress_text.rs | 2 +- .../_helpers/build_progress/registration.rs | 12 +- .../_helpers/build_progress/result_text.rs | 4 +- .../build_progress/transient_lines.rs | 141 ---- .../build_progress/transient_status.rs | 384 ---------- .../src/bindings/_helpers/cache/mod.rs | 3 - .../_helpers/cache/native_parse_store.rs | 73 -- .../bindings/_helpers/cache/registration.rs | 7 +- .../change_selection/base_snapshot.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 4 +- .../bindings/_helpers/check/check_results.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 9 +- .../src/bindings/_helpers/cli/mod.rs | 1 - .../bindings/_helpers/cli/observability.rs | 12 +- .../src/bindings/_helpers/cli/registration.rs | 12 +- .../bindings/_helpers/clone/clone_command.rs | 6 +- .../_helpers/command_runtime/connection.rs | 13 +- .../_helpers/command_runtime/invocation.rs | 4 +- .../bindings/_helpers/command_runtime/mod.rs | 1 - .../_helpers/command_runtime/registration.rs | 5 +- .../command_runtime/warehouse_scope.rs | 17 +- .../compile_attachment/declaration_scope.rs | 5 +- .../compile_attachment/model_inputs.rs | 9 +- .../compile_command/prepared_artifacts.rs | 5 +- .../_helpers/compile_command/report.rs | 7 +- .../_helpers/compile_command/reuse_attempt.rs | 22 +- .../compile_command/reuse_recording.rs | 8 +- .../_helpers/compile_command/target_writer.rs | 16 +- .../compile_command/test_artifact_cache.rs | 3 +- .../compile_context/compile_timings.rs | 21 - .../_helpers/compile_context/delta.rs | 449 ------------ .../_helpers/compile_context/image_codec.rs | 165 ----- .../bindings/_helpers/compile_delta/driver.rs | 33 +- .../bindings/_helpers/compile_delta/image.rs | 16 +- .../_helpers/compile_delta/lint_hooks.rs | 7 +- .../_helpers/compile_delta/report_hooks.rs | 13 +- .../_helpers/compile_delta/rules_rows.rs | 7 +- .../_helpers/compile_driver/analysis.rs | 21 +- .../_helpers/compile_driver/analysis_cache.rs | 3 +- .../_helpers/compile_driver/assembly.rs | 2 +- .../compile_driver/assembly_analysis.rs | 8 +- .../_helpers/compile_driver/assembly_facts.rs | 6 +- .../compile_driver/assembly_project.rs | 5 +- .../compile_driver/assembly_sql_tests.rs | 5 +- .../_helpers/compile_driver/command.rs | 10 +- .../_helpers/compile_driver/deferral.rs | 7 +- .../_helpers/compile_driver/discovery.rs | 7 +- .../_helpers/compile_driver/inputs.rs | 2 +- .../bindings/_helpers/compile_driver/mod.rs | 1 - .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compile_driver/plan_result.rs | 2 +- .../_helpers/compile_driver/project.rs | 6 +- .../_helpers/compile_driver/registration.rs | 22 +- .../_helpers/compile_driver/stage_work.rs | 79 --- .../bindings/_helpers/compile_outputs/mod.rs | 1 - .../_helpers/compile_outputs/registration.rs | 5 +- .../_helpers/compile_outputs/reuse.rs | 4 +- .../compile_outputs/runtime_target.rs | 2 +- .../compiled_project/attachment_inputs.rs | 2 +- .../compiled_project/snapshot_declarations.rs | 8 +- .../_helpers/compiled_views/audit_reads.rs | 3 +- .../_helpers/compiled_views/function_reads.rs | 3 +- .../_helpers/compiled_views/model_reads.rs | 3 +- .../_helpers/compiled_views/project_reads.rs | 3 +- .../_helpers/compiled_views/project_view.rs | 3 +- .../_helpers/compiled_views/resource_reads.rs | 3 +- .../_helpers/compiled_views/seed_reads.rs | 3 +- .../_helpers/compiled_views/source_reads.rs | 3 +- .../_helpers/compute_logs/conversion.rs | 6 +- .../_helpers/contract_command/command.rs | 6 +- .../src/bindings/_helpers/contracts/mod.rs | 2 - .../_helpers/contracts/model_contracts.rs | 2 +- .../_helpers/contracts/registration.rs | 9 +- .../bindings/_helpers/cost_command/command.rs | 2 +- .../bindings/_helpers/cost_command/refresh.rs | 2 +- .../bindings/_helpers/cursor/conversion.rs | 7 +- .../bindings/_helpers/dag_artifact/rows.rs | 2 +- .../_helpers/dag_artifact/selection.rs | 8 +- .../_helpers/dbt_command/init_project.rs | 7 +- .../bindings/_helpers/dbt_command/support.rs | 4 +- .../_helpers/dbt_interop/event_stream.rs | 2 +- .../bindings/_helpers/dbt_interop/pipeline.rs | 4 +- .../_helpers/dbt_interop/pipeline_phases.rs | 6 +- .../_helpers/dbt_interop/plan_output.rs | 4 +- .../bindings/_helpers/dbt_interop/profile.rs | 8 +- .../dbt_interop/profile_connection.rs | 7 +- .../_helpers/dbt_interop/runtime_execution.rs | 4 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 14 +- .../_helpers/debug_command/run_debug.rs | 9 +- .../src/bindings/_helpers/diagnostics/mod.rs | 5 - .../_helpers/diagnostics/registration.rs | 13 +- .../bindings/_helpers/diff_report/command.rs | 4 +- .../bindings/_helpers/diff_report/output.rs | 4 +- .../bindings/_helpers/diff_report/query.rs | 2 +- .../_helpers/discovery/header_keys.rs | 3 +- .../_helpers/discovery/imported_modules.rs | 8 +- .../_helpers/discovery/imported_providers.rs | 7 +- .../_helpers/discovery/model_records.rs | 2 +- .../discovery/orchestration_values.rs | 2 +- .../_helpers/discovery/project_facts.rs | 2 +- .../_helpers/discovery/provider_secrets.rs | 2 +- .../bindings/_helpers/discovery/snapshot.rs | 16 +- .../_helpers/discovery/sql_test_records.rs | 2 +- .../_helpers/discovery/yaml_entries.rs | 4 +- .../_helpers/discovery/yaml_records.rs | 24 +- .../_helpers/event_exporting/constants.rs | 25 - .../_helpers/event_exporting/dispatcher.rs | 35 +- .../_helpers/event_exporting/getters.rs | 11 +- .../bindings/_helpers/event_exporting/mod.rs | 4 - .../_helpers/event_exporting/models.rs | 320 --------- .../_helpers/event_exporting/policy.rs | 6 +- .../_helpers/event_exporting/registration.rs | 14 +- .../_helpers/event_exporting/scope.rs | 521 -------------- .../_helpers/event_exporting/workers.rs | 15 +- .../_helpers/execution_history/conversion.rs | 6 +- .../_helpers/execution_history/sqlite.rs | 8 +- .../_helpers/execution_history/statements.rs | 7 +- .../_helpers/fingerprints/latest_rows.rs | 7 +- .../_helpers/fingerprints/storage_io.rs | 2 +- .../bindings/_helpers/format_command/cache.rs | 6 +- .../_helpers/format_command/command.rs | 6 +- .../bindings/_helpers/format_command/run.rs | 8 +- .../_helpers/format_command/selection.rs | 2 +- .../_helpers/freshness/freshness_command.rs | 6 +- .../bindings/_helpers/freshness/planning.rs | 4 +- .../bindings/_helpers/freshness/state_io.rs | 2 +- .../src/bindings/_helpers/frontier/mod.rs | 2 - .../_helpers/frontier/stage_capture.rs | 3 +- .../bindings/_helpers/init_command/command.rs | 2 +- .../_helpers/input_loops/audit_loop.rs | 2 +- .../_helpers/input_loops/function_loop.rs | 2 +- .../_helpers/input_loops/model_loop.rs | 2 +- .../_helpers/input_loops/model_loop_header.rs | 6 +- .../input_loops/model_loop_opt_out.rs | 3 +- .../input_loops/resource_diagnostics.rs | 7 +- .../_helpers/input_loops/scenario_loop.rs | 2 +- .../_helpers/input_loops/sql_test_loop.rs | 4 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/janitor/janitor_phases.rs | 7 +- .../_helpers/janitor/janitor_progress.rs | 7 +- .../_helpers/janitor/janitor_usage.rs | 2 +- .../_helpers/lifecycle_events/payloads.rs | 9 +- .../_helpers/lineage/lineage_command.rs | 6 +- .../_helpers/lineage/project_lineage.rs | 13 +- .../bindings/_helpers/load/dlt_destination.rs | 6 +- .../bindings/_helpers/load/ingestr_command.rs | 5 +- .../bindings/_helpers/load/load_command.rs | 13 +- .../bindings/_helpers/load/load_progress.rs | 6 +- .../_helpers/location_command/snapshot.rs | 4 +- .../location_command/snapshot_providers.rs | 6 +- .../_helpers/macro_bridge/first_compile.rs | 2 +- .../_helpers/macro_bridge/module_digests.rs | 4 +- .../_helpers/macro_bridge/store_keys.rs | 19 +- .../src/bindings/_helpers/manifest/rows.rs | 2 +- .../_helpers/microbatches/conversion.rs | 9 +- .../_helpers/microbatches/statements.rs | 4 +- .../src/bindings/_helpers/mod.rs | 3 - .../_helpers/model_config/header_audits.rs | 4 +- .../_helpers/model_config/header_families.rs | 2 +- .../_helpers/model_config/model_validator.rs | 4 +- .../_helpers/model_config/template_data.rs | 3 +- .../_helpers/model_config/typed_config.rs | 3 +- .../_helpers/native_host/constants.rs | 25 - .../_helpers/native_host/consumption_state.rs | 4 +- .../_helpers/native_host/header_projection.rs | 54 -- .../_helpers/native_host/interrupt.rs | 84 --- .../src/bindings/_helpers/native_host/mod.rs | 13 - .../native_host/orchestration_sensors.rs | 7 +- .../native_host/orchestration_triggers.rs | 16 +- .../_helpers/native_host/registration.rs | 11 +- .../_helpers/native_host/sqlite_connection.rs | 279 -------- .../native_host/warehouse_metadata.rs | 6 +- .../_helpers/native_threads/constants.rs | 4 - .../_helpers/native_threads/reentrant_lock.rs | 71 -- .../_helpers/output_capture/dispatcher.rs | 4 +- .../_helpers/output_capture/values.rs | 7 +- .../bindings/_helpers/plan_command/plan.rs | 6 +- .../bindings/_helpers/plan_command/rules.rs | 2 +- .../_helpers/plan_output/plan_json.rs | 2 +- .../_helpers/planner_changes/detect.rs | 2 +- .../_helpers/planner_changes/results.rs | 9 +- .../planner_entries/execution_plan.rs | 2 +- .../_helpers/planner_entries/test_planning.rs | 2 +- .../_helpers/planner_identity/graph.rs | 7 +- .../_helpers/planner_identity/metadata.rs | 4 +- .../bindings/_helpers/planner_identity/mod.rs | 1 - .../_helpers/planner_phases/retention.rs | 2 +- .../_helpers/planner_phases/runtime.rs | 2 +- .../planner_resolution/cursor_ranges.rs | 7 +- .../planner_resolution/cursor_safety.rs | 7 +- .../planner_resolution/orchestration.rs | 2 +- .../_helpers/planner_resolution/sources.rs | 2 +- .../_helpers/planner_warehouse/cursors.rs | 2 +- .../planner_warehouse/source_columns.rs | 2 +- .../planner_warehouse/source_freshness.rs | 2 +- .../planner_warehouse/source_tables.rs | 2 +- .../_helpers/presentation_text/mod.rs | 5 - .../presentation_text/registration.rs | 13 +- .../_helpers/presentation_text/style.rs | 129 ---- .../_helpers/query_command/project.rs | 2 +- .../bindings/_helpers/query_command/query.rs | 4 +- .../_helpers/query_command/text_output.rs | 4 +- .../_helpers/recall/clickhouse_calls.rs | 2 +- .../bindings/_helpers/recall/replacements.rs | 6 +- .../_helpers/record_behaviours/auditing.rs | 7 +- .../_helpers/record_behaviours/constants.rs | 14 +- .../_helpers/record_behaviours/planner.rs | 7 +- .../_helpers/refactor/contract_compare.rs | 2 +- .../_helpers/refactor/contract_edits.rs | 7 +- .../_helpers/refactor/refactor_command.rs | 4 +- .../_helpers/relation_naming/fitting.rs | 6 +- .../_helpers/render/declaration_indexes.rs | 7 +- .../_helpers/render/macro_call_memo.rs | 7 +- .../render/macro_generated_references.rs | 2 +- .../_helpers/result_output/documents.rs | 4 +- .../_helpers/result_output/event_writer.rs | 2 +- .../result_output/integration_envelope.rs | 3 +- .../result_output/integration_records.rs | 11 +- .../bindings/_helpers/result_output/mod.rs | 1 - .../_helpers/result_output/terminal_index.rs | 15 +- .../_helpers/review_command/comparison.rs | 3 +- .../_helpers/review_command/execution.rs | 3 +- .../_helpers/review_command/planning.rs | 5 +- .../_helpers/review_command/schema.rs | 4 +- .../_helpers/rules/custom_evaluation.rs | 18 +- .../bindings/_helpers/rules/custom_hosts.rs | 7 +- .../_helpers/rules/custom_rule_evidence.rs | 7 +- .../bindings/_helpers/rules/early_sql_lint.rs | 3 +- .../src/bindings/_helpers/rules/rows.rs | 13 +- .../src/bindings/_helpers/rules/rule_api.rs | 6 +- .../bindings/_helpers/rules/rule_dispatch.rs | 4 +- .../src/bindings/_helpers/rules/selection.rs | 8 +- .../src/bindings/_helpers/rules/sql_rules.rs | 4 +- .../_helpers/scenario_command/presentation.rs | 4 +- .../_helpers/scenario_command/setup.rs | 4 +- .../_helpers/scenario_command/test_command.rs | 2 +- .../_helpers/scopes/compile_usages.rs | 2 +- .../_helpers/scopes/relationship_scan.rs | 2 +- .../bindings/_helpers/scopes/scope_command.rs | 2 +- .../bindings/_helpers/scopes/scope_errors.rs | 2 +- .../_helpers/scopes/scope_validation.rs | 4 +- .../bindings/_helpers/seed_command/command.rs | 2 +- .../_helpers/semantic_checks/failures.rs | 2 +- .../_helpers/semantic_checks/type_recovery.rs | 2 +- .../_helpers/semantic_shapes/coverage.rs | 2 +- .../_helpers/semantic_shapes/delta_shapes.rs | 18 +- .../semantic_shapes/source_bindings.rs | 2 +- .../_helpers/sql_test_command/python_tests.rs | 3 +- .../_helpers/sql_test_command/test_command.rs | 6 +- .../_helpers/sqltext/sql_references.rs | 2 +- .../_helpers/state_rows/migration_objects.rs | 7 +- .../_helpers/state_rows/state_tables.rs | 2 +- .../src/bindings/_helpers/type_system/mod.rs | 2 - .../_helpers/type_system/registration.rs | 5 +- .../_helpers/workspace_commands/commands.rs | 2 +- crates/sqlbuild-python/src/bindings/tests.rs | 2 - .../src/bindings/tests/helpers.rs | 18 +- .../src/bindings/tests/test_coverage.rs | 8 +- .../src/bindings/tests/test_types.rs | 11 +- fensu.toml | 337 +++++---- scripts/fensu_policy/constants.py | 2 +- tests/native_field_fallbacks.txt | 107 +-- 615 files changed, 7899 insertions(+), 4973 deletions(-) create mode 100644 crates/sqlbuild-python-base/src/boundary/_helpers/text_units.rs create mode 100644 crates/sqlbuild-python-base/src/boundary/main/utf16_units.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/build_command => sqlbuild-python-base/src/build_command/_helpers}/support.rs (98%) create mode 100644 crates/sqlbuild-python-base/src/build_command/main/boolean.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/display_text.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/emit.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/frozenset.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/joined.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/keyword_relay.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/line_writer.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/none.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/or_none.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/text.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/tuple.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/main/utc_now.rs create mode 100644 crates/sqlbuild-python-base/src/build_command/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/build_progress => sqlbuild-python-base/src/build_progress/_helpers}/progress.rs (50%) rename crates/{sqlbuild-python/src/bindings/_helpers/build_progress => sqlbuild-python-base/src/build_progress/_helpers}/terminal.rs (88%) create mode 100644 crates/sqlbuild-python-base/src/build_progress/_helpers/transient_lines.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/constants.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/current_native_progress_projector.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/current_projector.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/machine_output_mode.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/projector_operation_active.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/register_progress.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/terminal_request.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/progress/write_persistent.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/claim.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/consume_terminal.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/flush.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/join_spinner.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/projector.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/register_terminal.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/release.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/shared_lines.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/start_spinner.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/terminal_columns.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/tty_flag.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/terminal/write.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_lines/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_lines/register_transient_lines.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_lines/shared_transient_line_coordinator.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/progress_spinners_disabled.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/register_transient_status.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/main/transient_status/status_reporter.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/mod.rs create mode 100644 crates/sqlbuild-python-base/src/build_progress/models.rs create mode 100644 crates/sqlbuild-python-base/src/cache/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/cache/_helpers/native_parse_store.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cache => sqlbuild-python-base/src/cache/_helpers}/native_store.rs (96%) create mode 100644 crates/sqlbuild-python-base/src/cache/_helpers/sql_test_scan_store.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/content_digest.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/current_code_identity.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/digest_files.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/open_store.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/parse_key.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/register_native_store.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/register_sql_test_scan_store.rs create mode 100644 crates/sqlbuild-python-base/src/cache/main/save_store.rs create mode 100644 crates/sqlbuild-python-base/src/cache/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs => sqlbuild-python-base/src/cache/models.rs} (58%) rename crates/{sqlbuild-python/src/bindings/_helpers/cli => sqlbuild-python-base/src/cli/_helpers}/launch_state.rs (100%) create mode 100644 crates/sqlbuild-python-base/src/cli/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/cli/main/launch_record_path.rs create mode 100644 crates/sqlbuild-python-base/src/cli/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/cli/main/offer_launch_record.rs create mode 100644 crates/sqlbuild-python-base/src/cli/main/register_launch_state.rs create mode 100644 crates/sqlbuild-python-base/src/cli/main/withhold_launch_record.rs create mode 100644 crates/sqlbuild-python-base/src/cli/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/command_runtime => sqlbuild-python-base/src/command_runtime/_helpers}/adapters.rs (87%) create mode 100644 crates/sqlbuild-python-base/src/command_runtime/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/command_runtime/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/command_runtime/main/register_adapters.rs create mode 100644 crates/sqlbuild-python-base/src/command_runtime/main/resolve_cli_adapter.rs create mode 100644 crates/sqlbuild-python-base/src/command_runtime/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/coverage.rs (98%) create mode 100644 crates/sqlbuild-python-base/src/compile_context/_helpers/delta.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/diagnostic_collector.rs (57%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/getters.rs (64%) create mode 100644 crates/sqlbuild-python-base/src/compile_context/_helpers/image_codec.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/image_facts.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/input_reads.rs (75%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/mod.rs (64%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_context => sqlbuild-python-base/src/compile_context/_helpers}/yaml_cache.rs (78%) create mode 100644 crates/sqlbuild-python-base/src/compile_context/constants.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/coverage/coverage_reasons.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_coverage.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_order.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/coverage/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/capture.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/capturing.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/delta_unsupported.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/guarded.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/inject_relations.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/injected_fact_digest.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/injection.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/record_source_reads.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/record_test_source_reads.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/reset.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/set_pending_image.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/set_record_merge.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/start_capture.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/stop_capture.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/stop_on_errors.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/take_capture.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/take_delta_unsupported.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/take_pending_image.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/take_record_merge.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/delta/with_injection.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/collect_compile_diagnostics.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/register_diagnostic_collector.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic_function.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/image_facts/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/image_facts/record_model_usages.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/context_read.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/environment_read.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_compile_input_reads.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_settings_class_read.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/input_reads/register_input_reads.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/open_documents.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/retain_read_sources.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/save_documents.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/yaml_store_path.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/models.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/tests.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/tests/helpers.rs create mode 100644 crates/sqlbuild-python-base/src/compile_context/tests/test_types.rs rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-base/src/compile_context}/tests/test_yaml_cache.rs (94%) create mode 100644 crates/sqlbuild-python-base/src/compile_context/types.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/_helpers/stage_work.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/constants.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/main/count_stage.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/main/register_stage_work.rs create mode 100644 crates/sqlbuild-python-base/src/compile_driver/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_outputs => sqlbuild-python-base/src/compile_outputs/_helpers}/json_report.rs (91%) create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/emit_json_report.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/emit_orjson_report.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/float_key.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/indented_json.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/orjson_sorted.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/orjson_value.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/plain_json.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/main/register_json_report.rs create mode 100644 crates/sqlbuild-python-base/src/compile_outputs/mod.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-base/src/contracts/_helpers}/resource_identities.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/contracts => sqlbuild-python-base/src/contracts/_helpers}/state_literals.rs (93%) create mode 100644 crates/sqlbuild-python-base/src/contracts/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/main/register_resource_identities.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/main/register_state_literals.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/main/render_state_sql_literal.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/main/validate_resource_identity.rs create mode 100644 crates/sqlbuild-python-base/src/contracts/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/diagnostics => sqlbuild-python-base/src/diagnostics/_helpers}/console_records.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/diagnostics => sqlbuild-python-base/src/diagnostics/_helpers}/getters.rs (70%) rename crates/{sqlbuild-python/src/bindings/_helpers/diagnostics => sqlbuild-python-base/src/diagnostics/_helpers}/log_records.rs (97%) create mode 100644 crates/sqlbuild-python-base/src/diagnostics/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/diagnostics => sqlbuild-python-base/src/diagnostics/_helpers}/redaction.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/diagnostics => sqlbuild-python-base/src/diagnostics/_helpers}/trackers.rs (53%) create mode 100644 crates/sqlbuild-python-base/src/diagnostics/constants.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/current_tracker.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/register_console_records.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/register_log_records.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/register_redaction.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/main/register_trackers.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/mod.rs create mode 100644 crates/sqlbuild-python-base/src/diagnostics/models.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs rename crates/{sqlbuild-python/src/bindings/_helpers/event_exporting => sqlbuild-python-base/src/event_exporting/_helpers}/values.rs (94%) create mode 100644 crates/sqlbuild-python-base/src/event_exporting/constants.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/configure_discovered_event_exporters.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/current_scope.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/error_type_of.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/event_exporter_command_scope.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/export_policy.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/input_error.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/register_scope.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/severity_from.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/severity_object.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/main/state_error.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/mod.rs create mode 100644 crates/sqlbuild-python-base/src/event_exporting/models.rs create mode 100644 crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs create mode 100644 crates/sqlbuild-python-base/src/frontier/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/frontier => sqlbuild-python-base/src/frontier/_helpers}/python_types.rs (96%) create mode 100644 crates/sqlbuild-python-base/src/frontier/main/emit_error.rs create mode 100644 crates/sqlbuild-python-base/src/frontier/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/frontier/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs => sqlbuild-python-base/src/frontier/models.rs} (56%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/native_host/_helpers}/connections.rs (92%) create mode 100644 crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/native_host/_helpers}/results.rs (97%) create mode 100644 crates/sqlbuild-python-base/src/native_host/constants.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/main/check_signals.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/main/connection_failure.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/main/host_error.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/main/interrupted.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/models.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_threads => sqlbuild-python-base/src/native_threads/_helpers}/exit_signal.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_threads => sqlbuild-python-base/src/native_threads/_helpers}/interruptible_join.rs (75%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_threads => sqlbuild-python-base/src/native_threads/_helpers}/mod.rs (78%) create mode 100644 crates/sqlbuild-python-base/src/native_threads/_helpers/reentrant_lock.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/constants.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/interpreter_exiting.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/join_interruptibly.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/signal_interpreter_exit.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/mod.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/models.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/orchestration_reads/_helpers}/orchestration_declarations.rs (79%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/orchestration_reads/_helpers}/orchestration_environment.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/orchestration_reads/_helpers}/provider_settings.rs (88%) create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/callback_code_hash.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/declared_at.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/environment_hash.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/imported_declarations.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/missing_secrets.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/module_declarations.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/orchestration_code_hash.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/provider_class_settings.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/provider_state.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/register_orchestration_declarations.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/run_controls.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/run_resources.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/secret_variables.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/setting_rows.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/mod.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-base/src/planner_identity/_helpers}/json_values.rs (95%) create mode 100644 crates/sqlbuild-python-base/src/planner_identity/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/planner_identity/constants.rs create mode 100644 crates/sqlbuild-python-base/src/planner_identity/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/planner_identity/main/sorted_dumps.rs create mode 100644 crates/sqlbuild-python-base/src/planner_identity/main/unicode_sorted_dumps.rs create mode 100644 crates/sqlbuild-python-base/src/planner_identity/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/presentation_text => sqlbuild-python-base/src/presentation_text/_helpers}/document.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/presentation_text => sqlbuild-python-base/src/presentation_text/_helpers}/functions.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/presentation_text => sqlbuild-python-base/src/presentation_text/_helpers}/getters.rs (58%) create mode 100644 crates/sqlbuild-python-base/src/presentation_text/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/_helpers/style.rs rename crates/{sqlbuild-python/src/bindings/_helpers/presentation_text => sqlbuild-python-base/src/presentation_text/_helpers}/terminal.rs (100%) create mode 100644 crates/sqlbuild-python-base/src/presentation_text/constants.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/register_document.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/register_functions.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/register_style.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/register_terminal.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/stdout_is_tty.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/main/stdout_supports_color.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/mod.rs create mode 100644 crates/sqlbuild-python-base/src/presentation_text/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/python_values => sqlbuild-python-base/src/python_values/_helpers}/mod.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/python_values => sqlbuild-python-base/src/python_values/_helpers}/values.rs (83%) create mode 100644 crates/sqlbuild-python-base/src/python_values/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/python_values/main/python_value.rs create mode 100644 crates/sqlbuild-python-base/src/python_values/mod.rs create mode 100644 crates/sqlbuild-python-base/src/records/main/construct/new_plain_error.rs rename crates/{sqlbuild-python/src/bindings/_helpers/result_output => sqlbuild-python-base/src/result_output/_helpers}/integration_values.rs (89%) create mode 100644 crates/sqlbuild-python-base/src/result_output/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/bounded_metadata.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/json_value.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/optional_text.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/required_json.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/text.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/text_or_empty.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/main/validation_error.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/mod.rs create mode 100644 crates/sqlbuild-python-base/src/result_output/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/type_system => sqlbuild-python-base/src/type_system/_helpers}/equality.rs (91%) create mode 100644 crates/sqlbuild-python-base/src/type_system/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/type_system => sqlbuild-python-base/src/type_system/_helpers}/normalization.rs (65%) create mode 100644 crates/sqlbuild-python-base/src/type_system/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/type_system/main/normalization_error.rs create mode 100644 crates/sqlbuild-python-base/src/type_system/main/register_normalization.rs create mode 100644 crates/sqlbuild-python-base/src/type_system/main/types_equal.rs create mode 100644 crates/sqlbuild-python-base/src/type_system/mod.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/_helpers/sqlite_connection.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/warehouse_access/_helpers}/state_column_evolution.rs (93%) create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/warehouse_access/_helpers}/warehouse_catalog.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-base/src/warehouse_access/_helpers}/warehouse_statistics.rs (94%) create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/ensure_event_id_column.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/sqlite_params.rs create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs => sqlbuild-python-base/src/warehouse_access/models.rs} (58%) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cache/native_parse_store.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_codec.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_threads/reentrant_lock.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs diff --git a/Cargo.lock b/Cargo.lock index 329b76ce3d..f502b30315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4477,15 +4477,24 @@ dependencies = [ name = "sqlbuild-python-base" version = "0.0.0" dependencies = [ + "chrono", "mimalloc", "pyo3", + "rusqlite", + "rustc-hash", + "serde", "serde_json", + "sha2 0.10.9", "sqlbuild-analysis", + "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-rules", + "sqlbuild-state", ] [[package]] diff --git a/crates/README.md b/crates/README.md index 72b7ed43eb..c0b283aff3 100644 --- a/crates/README.md +++ b/crates/README.md @@ -28,7 +28,7 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | | `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | -| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; and the process allocator. | +| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | | `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-base`. | ## Layering @@ -67,7 +67,7 @@ entry carries a version other than its manifest's. `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`. Inside `sqlbuild-python`, the custom rule XSB075 orders the `bindings/_helpers` modules in layers -that follow the planned crate split: shared Python plumbing (`infra`, part of which is already +that follow the planned crate split: shared Python plumbing (`infra`, which is almost all in `sqlbuild-python-base`, which Cargo keeps below every other layer), then the glue of the lower crates (`front`), the compiled project (`project`), planning and compilation as siblings (`plan`, `compile`), command output (`output`), command handlers (`commands`) and registration @@ -89,7 +89,7 @@ empty: every import points down, and a new upward import fails the check. - module paths are never built at runtime: a `format!` or `concat!` template may not start with `sqlbuild.` or with a `{..PACKAGE..}.`/`{..MODULE..}.` constant. Write the complete module path as a constant instead; -- every row of the host table (`bindings/_helpers/native_host/`) must resolve as an attribute of +- every row of the host table (`entries` in `sqlbuild-host`) must resolve as an attribute of `sqlbuild._native_host`. Staying-Python entry points (adapter, user code, executor and runtime, third-party and public diff --git a/crates/sqlbuild-python-base/Cargo.toml b/crates/sqlbuild-python-base/Cargo.toml index 58ec6a9a87..aa9f59b91e 100644 --- a/crates/sqlbuild-python-base/Cargo.toml +++ b/crates/sqlbuild-python-base/Cargo.toml @@ -8,15 +8,24 @@ publish.workspace = true rust-version.workspace = true [dependencies] +chrono.workspace = true mimalloc.workspace = true pyo3.workspace = true +rusqlite.workspace = true +rustc-hash.workspace = true +serde.workspace = true serde_json.workspace = true +sha2.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-rules.workspace = true +sqlbuild-state.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python-base/src/boundary/_helpers/mod.rs b/crates/sqlbuild-python-base/src/boundary/_helpers/mod.rs index afb7571d09..1f3f70646e 100644 --- a/crates/sqlbuild-python-base/src/boundary/_helpers/mod.rs +++ b/crates/sqlbuild-python-base/src/boundary/_helpers/mod.rs @@ -9,3 +9,4 @@ pub(crate) mod oracles; pub(crate) mod panics; mod positional_records; pub(crate) mod python_calls; +pub(crate) mod text_units; diff --git a/crates/sqlbuild-python-base/src/boundary/_helpers/text_units.rs b/crates/sqlbuild-python-base/src/boundary/_helpers/text_units.rs new file mode 100644 index 0000000000..389d37b46e --- /dev/null +++ b/crates/sqlbuild-python-base/src/boundary/_helpers/text_units.rs @@ -0,0 +1,18 @@ +//! A Python `str` as UTF-16 code units, which keeps lone surrogates `to_str` rejects. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyBytes, PyBytesMethods, PyString}; + +/// The UTF-16 code units of `text`, lone surrogates included. +pub(crate) fn utf16_units(text: &Bound<'_, PyString>) -> PyResult> { + let encoded: Bound<'_, PyBytes> = text + .call_method1("encode", ("utf-16-le", "surrogatepass"))? + .downcast_into::()?; + Ok(encoded + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect()) +} diff --git a/crates/sqlbuild-python-base/src/boundary/main/mod.rs b/crates/sqlbuild-python-base/src/boundary/main/mod.rs index 689ee6debd..0386ce48a9 100644 --- a/crates/sqlbuild-python-base/src/boundary/main/mod.rs +++ b/crates/sqlbuild-python-base/src/boundary/main/mod.rs @@ -11,5 +11,6 @@ pub mod native_compiler_error; pub mod register_expected_errors; pub mod register_native_compiler_error; pub mod register_oracles; +pub mod utf16_units; pub mod value_error; pub mod within; diff --git a/crates/sqlbuild-python-base/src/boundary/main/utf16_units.rs b/crates/sqlbuild-python-base/src/boundary/main/utf16_units.rs new file mode 100644 index 0000000000..9c5b647e77 --- /dev/null +++ b/crates/sqlbuild-python-base/src/boundary/main/utf16_units.rs @@ -0,0 +1,11 @@ +//! The UTF-16 code units of a Python `str`, lone surrogates included. + +use crate::boundary::_helpers::text_units; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::PyString; + +/// The UTF-16 code units of a Python `str`, lone surrogates included. +#[inline] +pub fn utf16_units(text: &Bound<'_, PyString>) -> PyResult> { + text_units::utf16_units(text) +} diff --git a/crates/sqlbuild-python-base/src/build_command/_helpers/mod.rs b/crates/sqlbuild-python-base/src/build_command/_helpers/mod.rs new file mode 100644 index 0000000000..d38764abd9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs b/crates/sqlbuild-python-base/src/build_command/_helpers/support.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs rename to crates/sqlbuild-python-base/src/build_command/_helpers/support.rs index 840b51bdfb..801837f257 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs +++ b/crates/sqlbuild-python-base/src/build_command/_helpers/support.rs @@ -6,7 +6,7 @@ use pyo3::types::{ }; use pyo3::{IntoPyObject, pyclass, pymethods}; -use sqlbuild_python_base::boundary::main::imported::imported; +use crate::boundary::main::imported::imported; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python-base/src/build_command/main/boolean.rs b/crates/sqlbuild-python-base/src/build_command/main/boolean.rs new file mode 100644 index 0000000000..8844619f9b --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/boolean.rs @@ -0,0 +1,12 @@ +//! Boolean. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::Python; + +/// Boolean. +#[inline] +pub fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + support::boolean(py, value) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/display_text.rs b/crates/sqlbuild-python-base/src/build_command/main/display_text.rs new file mode 100644 index 0000000000..24e22d101e --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/display_text.rs @@ -0,0 +1,12 @@ +//! A string, integer or float shown as text; anything else is unset. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A string, integer or float shown as text; anything else is unset. +#[inline] +pub fn display_text(value: &Bound<'_, PyAny>) -> PyResult> { + support::display_text(value) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/emit.rs b/crates/sqlbuild-python-base/src/build_command/main/emit.rs new file mode 100644 index 0000000000..fe6156b3a3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/emit.rs @@ -0,0 +1,12 @@ +//! Write `value` to the stream and flush it. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Write `value` to the stream and flush it. +#[inline] +pub fn emit(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { + support::emit(stream, value) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/frozenset.rs b/crates/sqlbuild-python-base/src/build_command/main/frozenset.rs new file mode 100644 index 0000000000..2fa1844e29 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/frozenset.rs @@ -0,0 +1,16 @@ +//! Frozenset. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Frozenset. +#[inline] +pub fn frozenset<'py>( + py: Python<'py>, + values: &[Bound<'py, PyAny>], +) -> PyResult> { + support::frozenset(py, values) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/joined.rs b/crates/sqlbuild-python-base/src/build_command/main/joined.rs new file mode 100644 index 0000000000..3697ed6209 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/joined.rs @@ -0,0 +1,12 @@ +//! `base / part`. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `base / part`. +#[inline] +pub fn joined<'py>(base: &Bound<'py, PyAny>, parts: &[&str]) -> PyResult> { + support::joined(base, parts) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/keyword_relay.rs b/crates/sqlbuild-python-base/src/build_command/main/keyword_relay.rs new file mode 100644 index 0000000000..d56a2584a2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/keyword_relay.rs @@ -0,0 +1,15 @@ +//! A relay calling `target` with `names` as keywords. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A relay calling `target` with `names` as keywords. +#[inline] +pub fn keyword_relay<'py>( + target: Bound<'py, PyAny>, + names: &[&'static str], +) -> PyResult> { + support::keyword_relay(target, names) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/line_writer.rs b/crates/sqlbuild-python-base/src/build_command/main/line_writer.rs new file mode 100644 index 0000000000..3c2e848efa --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/line_writer.rs @@ -0,0 +1,12 @@ +//! A callable writing each message as a line on `stream`. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A callable writing each message as a line on `stream`. +#[inline] +pub fn line_writer<'py>(stream: &Bound<'py, PyAny>) -> PyResult> { + support::line_writer(stream) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/mod.rs b/crates/sqlbuild-python-base/src/build_command/main/mod.rs new file mode 100644 index 0000000000..6b6cb0e0f5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/mod.rs @@ -0,0 +1,12 @@ +pub mod boolean; +pub mod display_text; +pub mod emit; +pub mod frozenset; +pub mod joined; +pub mod keyword_relay; +pub mod line_writer; +pub mod none; +pub mod or_none; +pub mod text; +pub mod tuple; +pub mod utc_now; diff --git a/crates/sqlbuild-python-base/src/build_command/main/none.rs b/crates/sqlbuild-python-base/src/build_command/main/none.rs new file mode 100644 index 0000000000..8aa905c650 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/none.rs @@ -0,0 +1,12 @@ +//! None. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::Python; + +/// None. +#[inline] +pub fn none(py: Python<'_>) -> Bound<'_, PyAny> { + support::none(py) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/or_none.rs b/crates/sqlbuild-python-base/src/build_command/main/or_none.rs new file mode 100644 index 0000000000..1fe45c55b1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/or_none.rs @@ -0,0 +1,15 @@ +//! The value, or `default` when it is `None`. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The value, or `default` when it is `None`. +#[inline] +pub fn or_none<'py>( + value: Bound<'py, PyAny>, + default: impl FnOnce() -> PyResult>, +) -> PyResult> { + support::or_none(value, default) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/text.rs b/crates/sqlbuild-python-base/src/build_command/main/text.rs new file mode 100644 index 0000000000..e894fd837d --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/text.rs @@ -0,0 +1,12 @@ +//! Text. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::Python; + +/// Text. +#[inline] +pub fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + support::text(py, value) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/tuple.rs b/crates/sqlbuild-python-base/src/build_command/main/tuple.rs new file mode 100644 index 0000000000..a49aea19b2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/tuple.rs @@ -0,0 +1,13 @@ +//! Tuple. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Tuple. +#[inline] +pub fn tuple<'py>(py: Python<'py>, values: &[Bound<'py, PyAny>]) -> PyResult> { + support::tuple(py, values) +} diff --git a/crates/sqlbuild-python-base/src/build_command/main/utc_now.rs b/crates/sqlbuild-python-base/src/build_command/main/utc_now.rs new file mode 100644 index 0000000000..e3a97666cd --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/main/utc_now.rs @@ -0,0 +1,13 @@ +//! `datetime.now(UTC)`. + +use crate::build_command::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `datetime.now(UTC)`. +#[inline] +pub fn utc_now(py: Python<'_>) -> PyResult> { + support::utc_now(py) +} diff --git a/crates/sqlbuild-python-base/src/build_command/mod.rs b/crates/sqlbuild-python-base/src/build_command/mod.rs new file mode 100644 index 0000000000..4196a6bfc1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_command/mod.rs @@ -0,0 +1,4 @@ +//! Python value helpers and callback relays the build command shares. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-base/src/build_progress/_helpers/mod.rs b/crates/sqlbuild-python-base/src/build_progress/_helpers/mod.rs new file mode 100644 index 0000000000..f1ce1e5fac --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod progress; +pub(crate) mod terminal; +pub(crate) mod transient_lines; +pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs b/crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs similarity index 50% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs rename to crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs index 02db6f3d95..0101a25803 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs +++ b/crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs @@ -1,28 +1,20 @@ //! The invocation-scoped native CLI projection of canonical lifecycle facts. -use std::sync::{Mutex, MutexGuard, PoisonError}; +use crate::build_progress::models::NativeProgressProjector; -use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyMapping, PyString}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_core::progress::main::claim_progress_terminal::claim_progress_terminal; -use sqlbuild_core::progress::main::close_progress_projection::close_progress_projection; -use sqlbuild_core::progress::main::consume_progress_event::consume_progress_event; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::progress::main::machine_output_mode::machine_output_mode as output_mode; -use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal_duration; -use sqlbuild_core::progress::models::{ - OutputModeFacts, ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, -}; +use sqlbuild_core::progress::models::{OutputModeFacts, ProgressEvent, ProgressPayload}; use sqlbuild_core::terminal_events::models::TerminalRequest; -use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use crate::boundary::models::KeywordArguments; +use crate::build_progress::_helpers::transient_lines::shared_transient_line_coordinator; +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_get::mapping_get; +use crate::records::main::fields::mapping_get_or::mapping_get_or; /// The module attribute holding the `ContextVar` of the installed projector. const CONTEXT_ATTRIBUTE: &str = "_native_progress_projector_context"; @@ -37,16 +29,6 @@ const CLAIM_OPTIONS: (&[&str], &[&str]) = ( const DEFAULT_INTENT: &str = "execute"; const DEFAULT_STATEMENT_KIND: &str = "UNKNOWN"; -/// Project canonical starts and correlate enriched terminal presentation. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeProgressProjector { - state: Mutex, - stream: Py, - coordinator: Py, - /// The lines held back instead of written, while something holds them. - held: Mutex>>, -} - fn payload_value<'py>(payload: &Bound<'py, PyAny>, key: &str) -> PyResult> { mapping_get(payload, key) } @@ -128,7 +110,7 @@ fn optional_text(event: &Bound<'_, PyAny>, name: &str) -> PyResult) -> PyResult { +pub(crate) fn progress_event(event: &Bound<'_, PyAny>) -> PyResult { Ok(ProgressEvent { event_id: native_field(event, "event_id")?.extract()?, event_type: native_field(event, "event_type")?.extract()?, @@ -146,7 +128,11 @@ pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py)) } -fn persist(coordinator: &Bound<'_, PyAny>, stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { +pub(crate) fn persist( + coordinator: &Bound<'_, PyAny>, + stream: &Bound<'_, PyAny>, + text: &str, +) -> PyResult<()> { let keywords: Bound<'_, PyDict> = PyDict::new(coordinator.py()); keywords.set_item("stream", stream)?; keywords.set_item("text", text)?; @@ -170,77 +156,8 @@ pub(crate) fn terminal_stream(stream: &Bound<'_, PyAny>) -> PyResult { stream.call_method0("isatty")?.is_truthy() } -impl NativeProgressProjector { - fn held(&self) -> MutexGuard<'_, Option>> { - self.held.lock().unwrap_or_else(PoisonError::into_inner) - } - - /// Hold back every line from now on instead of writing it. - pub(crate) fn hold_lines(&self) { - *self.held() = Some(Vec::new()); - } - - /// Replace the lines held back so far with `lines`, returning them; nothing while not holding. - pub(crate) fn exchange_held_lines(&self, lines: Vec) -> Vec { - match self.held().as_mut() { - Some(held) => std::mem::replace(held, lines), - None => Vec::new(), - } - } - - /// Write lines as they come again, returning those held back since `hold_lines`. - pub(crate) fn release_lines(&self) -> Vec { - self.held().take().unwrap_or_default() - } - - /// Write lines held back earlier, as they would have been written. - pub(crate) fn write_held(&self, py: Python<'_>, lines: &[String]) -> PyResult<()> { - let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); - for line in lines { - persist(coordinator, self.stream.bind(py), line)?; - } - Ok(()) - } - - fn locked(&self) -> PyResult> { - self.state - .lock() - .map_err(|_| PyRuntimeError::new_err("native progress projector lock is poisoned")) - } - - /// Run `update` and write its lines while holding the shared terminal line lock. - fn write_with( - &self, - py: Python<'_>, - update: impl FnOnce(ProgressProjection) -> (ProgressProjection, Vec, T), - ) -> PyResult { - let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); - let lock: Bound<'_, PyAny> = native_field(coordinator, "lock")?; - lock.call_method0("acquire")?; - let result: PyResult = (|| { - let (lines, value) = { - let mut state: MutexGuard<'_, ProgressProjection> = self.locked()?; - let (projection, lines, value) = update(std::mem::take(&mut *state)); - *state = projection; - (lines, value) - }; - for line in lines { - let text: String = format!("{line}\n"); - if let Some(held) = self.held().as_mut() { - held.push(text); - continue; - } - persist(coordinator, self.stream.bind(py), &text)?; - } - Ok(value) - })(); - lock.call_method0("release")?; - result - } -} - /// The terminal a result callback names through its keyword arguments. -fn terminal_request( +pub(crate) fn terminal_request( py: Python<'_>, function: &str, options: Option>, @@ -255,115 +172,7 @@ fn terminal_request( }) } -#[pymethods] -impl NativeProgressProjector { - #[new] - #[pyo3(signature = (*, stream, use_color, debug = false))] - pub(crate) fn new( - py: Python<'_>, - stream: Bound<'_, PyAny>, - use_color: bool, - debug: bool, - ) -> PyResult { - let coordinator: Bound<'_, PyAny> = shared_coordinator(py)?; - Ok(Self { - state: Mutex::new(ProgressProjection::new(ProgressSettings { - use_color, - debug, - is_tty: terminal_stream(&stream)?, - })), - stream: stream.unbind(), - coordinator: coordinator.unbind(), - held: Mutex::new(None), - }) - } - - /// Install this projector for presentation adapters in the invocation context. - fn install<'py>(slf: &Bound<'py, Self>) -> PyResult> { - context_variable(slf.py())?.call_method1("set", (slf,)) - } - - /// Restore the previous invocation projector. - fn restore(&self, py: Python<'_>, token: &Bound<'_, PyAny>) -> PyResult<()> { - context_variable(py)?.call_method1("reset", (token,))?; - Ok(()) - } - - /// Add safe display order metadata without changing canonical facts. - #[pyo3(signature = (*, ordinals, total))] - fn configure_resources(&self, ordinals: &Bound<'_, PyAny>, total: i64) -> PyResult<()> { - let mut entries: Vec<(String, i64)> = Vec::new(); - for item in mapping_items(ordinals)?.try_iter()? { - entries.push(item?.extract()?); - } - self.locked()?.configure_resources(entries, total); - Ok(()) - } - - /// Declare that an existing result callback will enrich this resource terminal. - #[pyo3(signature = (*, resource_name))] - fn expect_resource_enrichment(&self, resource_name: &str) -> PyResult<()> { - self.locked()?.expect_resource_enrichment(resource_name); - Ok(()) - } - - /// Record how one audit attempt ended, so its terminal row shows the outcome. - #[pyo3(signature = (*, resource_attempt_id, outcome))] - fn record_audit_outcome(&self, resource_attempt_id: &str, outcome: &str) -> PyResult<()> { - self.locked()? - .record_audit_outcome(resource_attempt_id, outcome); - Ok(()) - } - - /// Return whether a visible canonical operation currently owns progress. - #[pyo3(name = "is_operation_active", signature = (*, operation_name))] - pub(crate) fn operation_active(&self, operation_name: &str) -> PyResult { - Ok(self.locked()?.is_operation_active(operation_name)) - } - - /// Consume one known lifecycle fact idempotently. - fn consume(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - let event: ProgressEvent = progress_event(event)?; - self.write_with(py, |projection| { - let (projection, lines) = consume_progress_event(projection, &event); - (projection, lines, ()) - }) - } - - /// Claim the targeted or latest terminal for one final resource result. - #[pyo3(signature = (**options))] - fn consume_resource_terminal( - &self, - py: Python<'_>, - options: Option>, - ) -> PyResult> { - let request: TerminalRequest = terminal_request(py, "consume_resource_terminal", options)?; - self.write_with(py, |projection| { - claim_progress_terminal(projection, &request) - }) - } - - /// Return a terminal duration without claiming its presentation. - #[pyo3(signature = (**options))] - fn resource_terminal_duration( - &self, - py: Python<'_>, - options: Option>, - ) -> PyResult> { - let request: TerminalRequest = terminal_request(py, "resource_terminal_duration", options)?; - Ok(progress_terminal_duration(&*self.locked()?, &request)) - } - - /// Render resource terminals no result callback claimed. - fn close(&self, py: Python<'_>) -> PyResult<()> { - self.write_with(py, |projection| { - let (projection, lines) = close_progress_projection(projection); - (projection, lines, ()) - }) - } -} - -fn context_variable(py: Python<'_>) -> PyResult> { +pub(crate) fn context_variable(py: Python<'_>) -> PyResult> { Ok(CONTEXT .get_or_try_init(py, || { let keywords: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs rename to crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs index f71692995f..44901d1910 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs +++ b/crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs @@ -4,25 +4,17 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; -use crate::bindings::_helpers::build_progress::transient_lines::shared_coordinator; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::imported::imported; +use crate::build_progress::_helpers::progress::current_native_progress_projector; +use crate::build_progress::_helpers::transient_lines::shared_coordinator; +use crate::records::main::fields::field::field as native_field; const DEFAULT_TERMINAL_SIZE: (i64, i64) = (120, 24); const SPINNER_JOIN_SECONDS: f64 = 0.2; const EXIT_JOIN_SECONDS: f64 = 1.0; const SPINNER_THREAD_NAME: &str = "sqlbuild-progress-spinner"; const STOP_EVENT_ATTRIBUTE: &str = "_sqlbuild_stop_event"; -pub(crate) const SPINNER_TICK_SECONDS: f64 = 0.1; -pub(crate) const CLEAR_LINE: &str = "\r\u{1b}[K"; -pub(crate) const HIDE_CURSOR: &str = "\u{1b}[?25l"; -pub(crate) const SHOW_CURSOR: &str = "\u{1b}[?25h"; -pub(crate) const SPINNER_FRAMES: [&str; 10] = [ - "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", - "\u{2807}", "\u{280f}", -]; /// The current terminal width, `shutil.get_terminal_size(fallback=(120, 24)).columns`. pub(crate) fn terminal_columns(py: Python<'_>) -> PyResult { diff --git a/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_lines.rs b/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_lines.rs new file mode 100644 index 0000000000..3a82ba6583 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_lines.rs @@ -0,0 +1,32 @@ +//! The coordinator of one live transient terminal line and persistent writes. + +use crate::build_progress::models::TransientLineCoordinator; + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// The stream a transient line is drawn on and the renderer that owns it. +pub(crate) type ActiveLine = (Py, Py); + +static SHARED_COORDINATOR: PyOnceLock> = PyOnceLock::new(); + +/// The coordinator shared by every terminal writer in this process. +pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { + SHARED_COORDINATOR + .get_or_try_init(py, || Py::new(py, TransientLineCoordinator::new(py)?)) + .map(|coordinator| coordinator.bind(py).clone()) +} + +/// Return the coordinator shared by every terminal writer in this process. +#[pyfunction] +pub(crate) fn shared_transient_line_coordinator( + py: Python<'_>, +) -> PyResult> { + shared_coordinator(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(shared_transient_line_coordinator, module)?) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs b/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs new file mode 100644 index 0000000000..ea5ad9013d --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs @@ -0,0 +1,96 @@ +//! One transient status line with a spinner on a terminal and plain lines elsewhere. + +use crate::build_progress::models::TransientStatusReporter; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::time::Duration; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; + +use crate::build_progress::_helpers::terminal::tty_flag; + +const NO_PROGRESS_VARIABLE: &str = "SQLBUILD_NO_PROGRESS"; +const PROGRESS_DISABLED_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; +pub(crate) const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Debug, Default)] +pub(crate) struct StatusState { + pub(crate) message: Option, + pub(crate) active: bool, + pub(crate) drawn: bool, + pub(crate) frame: usize, + pub(crate) stop_event: Option>, + pub(crate) threads: Vec>, + pub(crate) exit_hook_registered: bool, +} + +/// Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. +pub(crate) fn progress_spinners_disabled() -> bool { + std::env::var(NO_PROGRESS_VARIABLE) + .is_ok_and(|value| PROGRESS_DISABLED_VALUES.contains(&value.trim().to_lowercase().as_str())) +} + +/// A spinner reporter on stderr, or `None` when stderr is not a terminal or progress is off. +fn stderr_reporter(py: Python<'_>) -> PyResult>> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + if progress_spinners_disabled() || !tty_flag(&stderr)? { + return Ok(None); + } + Py::new(py, TransientStatusReporter::new(py, &stderr, false, true)?).map(Some) +} + +/// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. +pub(crate) fn maybe_status<'py, T>( + py: Python<'py>, + (message, enabled): (&str, bool), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let Some(reporter) = enabled.then(|| stderr_reporter(py)).transpose()?.flatten() else { + return body(); + }; + let reporter: Bound<'py, TransientStatusReporter> = reporter.into_bound(py); + TransientStatusReporter::start(&reporter, message.to_owned())?; + let result: PyResult = body(); + TransientStatusReporter::close(&reporter)?; + result +} + +/// Run `body`, drawing a stderr spinner named `message` only once it has run for `delay`. +pub(crate) fn delayed_status<'py, T>( + py: Python<'py>, + (message, delay): (&str, Duration), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let Some(reporter) = stderr_reporter(py)? else { + return body(); + }; + let (finished, waiting) = mpsc::channel::<()>(); + let pending: Py = reporter.clone_ref(py); + let message: String = message.to_owned(); + let timer = std::thread::spawn(move || -> PyResult<()> { + if waiting.recv_timeout(delay) != Err(RecvTimeoutError::Timeout) { + return Ok(()); + } + Python::attach(|py| TransientStatusReporter::start(pending.bind(py), message)) + }); + let result: PyResult = body(); + let _ = finished.send(()); + let started: PyResult<()> = py.detach(|| timer.join()).unwrap_or_else(|_| Ok(())); + TransientStatusReporter::close(reporter.bind(py))?; + started?; + result +} + +/// A native status reporter writing to `stream`. +pub(crate) fn status_reporter<'py>( + stream: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let reporter: TransientStatusReporter = + TransientStatusReporter::new(stream.py(), stream, use_color, true)?; + Ok(Bound::new(stream.py(), reporter)?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-base/src/build_progress/constants.rs b/crates/sqlbuild-python-base/src/build_progress/constants.rs new file mode 100644 index 0000000000..9d199b4111 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/constants.rs @@ -0,0 +1,14 @@ +//! Public constants of build progress. + +pub const SPINNER_TICK_SECONDS: f64 = 0.1; + +pub const CLEAR_LINE: &str = "\r\u{1b}[K"; + +pub const HIDE_CURSOR: &str = "\u{1b}[?25l"; + +pub const SHOW_CURSOR: &str = "\u{1b}[?25h"; + +pub const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/mod.rs b/crates/sqlbuild-python-base/src/build_progress/main/mod.rs new file mode 100644 index 0000000000..12e75642d1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/mod.rs @@ -0,0 +1,4 @@ +pub mod progress; +pub mod terminal; +pub mod transient_lines; +pub mod transient_status; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/current_native_progress_projector.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/current_native_progress_projector.rs new file mode 100644 index 0000000000..affa1af70e --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/current_native_progress_projector.rs @@ -0,0 +1,13 @@ +//! Return the projector installed for the current CLI invocation. + +use crate::build_progress::_helpers::progress; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Return the projector installed for the current CLI invocation. +#[inline] +pub fn current_native_progress_projector(py: Python<'_>) -> PyResult> { + progress::current_native_progress_projector(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/current_projector.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/current_projector.rs new file mode 100644 index 0000000000..f1c1efb4d0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/current_projector.rs @@ -0,0 +1,13 @@ +//! The projector installed for this invocation, when there is one. + +use crate::build_progress::_helpers::progress; +use crate::build_progress::models::NativeProgressProjector; +use pyo3::prelude::Py; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The projector installed for this invocation, when there is one. +#[inline] +pub fn current_projector(py: Python<'_>) -> PyResult>> { + progress::current_projector(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/machine_output_mode.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/machine_output_mode.rs new file mode 100644 index 0000000000..acba5fc566 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/machine_output_mode.rs @@ -0,0 +1,12 @@ +//! Whether the parsed command writes machine output to stdout, so progress must use stderr. + +use crate::build_progress::_helpers::progress; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether the parsed command writes machine output to stdout, so progress must use stderr. +#[inline] +pub fn machine_output_mode(args: &Bound<'_, PyAny>) -> PyResult { + progress::machine_output_mode(args) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/mod.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/mod.rs new file mode 100644 index 0000000000..7d11a9eca4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/mod.rs @@ -0,0 +1,7 @@ +pub mod current_native_progress_projector; +pub mod current_projector; +pub mod machine_output_mode; +pub mod projector_operation_active; +pub mod register_progress; +pub mod terminal_request; +pub mod write_persistent; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/projector_operation_active.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/projector_operation_active.rs new file mode 100644 index 0000000000..c4481d613e --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/projector_operation_active.rs @@ -0,0 +1,14 @@ +//! Whether an installed projector shows a visible operation with this name. + +use crate::build_progress::_helpers::progress; +use crate::build_progress::models::NativeProgressProjector; +use pyo3::prelude::PyResult; + +/// Whether an installed projector shows a visible operation with this name. +#[inline] +pub fn projector_operation_active( + projector: &NativeProgressProjector, + operation_name: &str, +) -> PyResult { + progress::projector_operation_active(projector, operation_name) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/register_progress.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/register_progress.rs new file mode 100644 index 0000000000..09fdfbabcd --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/register_progress.rs @@ -0,0 +1,12 @@ +//! Register the progress functions and classes. + +use crate::build_progress::_helpers::progress; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the progress functions and classes. +#[inline] +pub fn register_progress(module: &Bound<'_, PyModule>) -> PyResult<()> { + progress::register(module) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/terminal_request.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/terminal_request.rs new file mode 100644 index 0000000000..a9d4dc620f --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/terminal_request.rs @@ -0,0 +1,16 @@ +//! The terminal claim keywords of `function`: resource name, id, attempt id and attempt number. + +use crate::build_progress::_helpers::progress; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_core::terminal_events::models::TerminalRequest; + +/// The terminal claim keywords of `function`: resource name, id, attempt id and attempt number. +#[inline] +pub fn terminal_request( + py: Python<'_>, + function: &str, + options: Option>, +) -> PyResult { + progress::terminal_request(py, function, options) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/progress/write_persistent.rs b/crates/sqlbuild-python-base/src/build_progress/main/progress/write_persistent.rs new file mode 100644 index 0000000000..fbbdca98f7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/progress/write_persistent.rs @@ -0,0 +1,13 @@ +//! Write lasting text, clearing and then redrawing any live transient line. + +use crate::build_progress::_helpers::progress; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Write lasting text, clearing and then redrawing any live transient line. +#[inline] +pub fn write_persistent(py: Python<'_>, stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + progress::write_persistent(py, stream, text) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/claim.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/claim.rs new file mode 100644 index 0000000000..f3c36726a1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/claim.rs @@ -0,0 +1,16 @@ +//! Claim or release the live line for `owner`. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Claim or release the live line for `owner`. +#[inline] +pub fn claim( + lines: &Bound<'_, PyAny>, + stream: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, +) -> PyResult<()> { + terminal::claim(lines, stream, owner) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/consume_terminal.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/consume_terminal.rs new file mode 100644 index 0000000000..f49271a27c --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/consume_terminal.rs @@ -0,0 +1,16 @@ +//! The projector's canonical duration of a finished resource, `None` while it is pending. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The projector's canonical duration of a finished resource, `None` while it is pending. +#[inline] +pub fn consume_terminal( + projector: &Bound<'_, PyAny>, + resource_name: &str, + resource_id: Option<&str>, +) -> PyResult> { + terminal::consume_terminal(projector, resource_name, resource_id) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/flush.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/flush.rs new file mode 100644 index 0000000000..4cf49161e6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/flush.rs @@ -0,0 +1,12 @@ +//! Flush. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Flush. +#[inline] +pub fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { + terminal::flush(stream) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/join_spinner.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/join_spinner.rs new file mode 100644 index 0000000000..ea10106303 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/join_spinner.rs @@ -0,0 +1,12 @@ +//! Join a stopped spinner thread briefly if it is still alive. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Join a stopped spinner thread briefly if it is still alive. +#[inline] +pub fn join_spinner(thread: &Bound<'_, PyAny>) -> PyResult<()> { + terminal::join_spinner(thread) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/mod.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/mod.rs new file mode 100644 index 0000000000..6bed587a1c --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/mod.rs @@ -0,0 +1,12 @@ +pub mod claim; +pub mod consume_terminal; +pub mod flush; +pub mod join_spinner; +pub mod projector; +pub mod register_terminal; +pub mod release; +pub mod shared_lines; +pub mod start_spinner; +pub mod terminal_columns; +pub mod tty_flag; +pub mod write; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/projector.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/projector.rs new file mode 100644 index 0000000000..ac0f1ed26f --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/projector.rs @@ -0,0 +1,13 @@ +//! The current native progress projector, if a projection is active. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The current native progress projector, if a projection is active. +#[inline] +pub fn projector(py: Python<'_>) -> PyResult>> { + terminal::projector(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/register_terminal.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/register_terminal.rs new file mode 100644 index 0000000000..777fde6cbc --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/register_terminal.rs @@ -0,0 +1,12 @@ +//! Add `stop_progress_spinners` and run it at exit, before finalization ends daemon threads. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Add `stop_progress_spinners` and run it at exit, before finalization ends daemon threads. +#[inline] +pub fn register_terminal(module: &Bound<'_, PyModule>) -> PyResult<()> { + terminal::register(module) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/release.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/release.rs new file mode 100644 index 0000000000..13dec51307 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/release.rs @@ -0,0 +1,12 @@ +//! Release. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Release. +#[inline] +pub fn release(lines: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { + terminal::release(lines, owner) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/shared_lines.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/shared_lines.rs new file mode 100644 index 0000000000..a640627894 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/shared_lines.rs @@ -0,0 +1,13 @@ +//! The process-wide coordinator of the one live transient line. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The process-wide coordinator of the one live transient line. +#[inline] +pub fn shared_lines(py: Python<'_>) -> PyResult> { + terminal::shared_lines(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/start_spinner.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/start_spinner.rs new file mode 100644 index 0000000000..b956297aa2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/start_spinner.rs @@ -0,0 +1,14 @@ +//! Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. +#[inline] +pub fn start_spinner<'py>( + owner: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + terminal::start_spinner(owner) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/terminal_columns.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/terminal_columns.rs new file mode 100644 index 0000000000..19d55a61af --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/terminal_columns.rs @@ -0,0 +1,11 @@ +//! The current terminal width, `shutil.get_terminal_size(fallback=(120, 24)).columns`. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The current terminal width, `shutil.get_terminal_size(fallback=(120, 24)).columns`. +#[inline] +pub fn terminal_columns(py: Python<'_>) -> PyResult { + terminal::terminal_columns(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/tty_flag.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/tty_flag.rs new file mode 100644 index 0000000000..8fde40727b --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/tty_flag.rs @@ -0,0 +1,12 @@ +//! Whether `stream` reports a terminal. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether `stream` reports a terminal. +#[inline] +pub fn tty_flag(stream: &Bound<'_, PyAny>) -> PyResult { + terminal::tty_flag(stream) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/terminal/write.rs b/crates/sqlbuild-python-base/src/build_progress/main/terminal/write.rs new file mode 100644 index 0000000000..67068403ee --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/terminal/write.rs @@ -0,0 +1,12 @@ +//! Write. + +use crate::build_progress::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Write. +#[inline] +pub fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + terminal::write(stream, text) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/mod.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/mod.rs new file mode 100644 index 0000000000..d994103c74 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/mod.rs @@ -0,0 +1,2 @@ +pub mod register_transient_lines; +pub mod shared_transient_line_coordinator; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/register_transient_lines.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/register_transient_lines.rs new file mode 100644 index 0000000000..ae344642d9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/register_transient_lines.rs @@ -0,0 +1,12 @@ +//! Register the transient lines functions and classes. + +use crate::build_progress::_helpers::transient_lines; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the transient lines functions and classes. +#[inline] +pub fn register_transient_lines(module: &Bound<'_, PyModule>) -> PyResult<()> { + transient_lines::register(module) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/shared_transient_line_coordinator.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/shared_transient_line_coordinator.rs new file mode 100644 index 0000000000..d9a7a3f454 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_lines/shared_transient_line_coordinator.rs @@ -0,0 +1,15 @@ +//! Return the coordinator shared by every terminal writer in this process. + +use crate::build_progress::_helpers::transient_lines; +use crate::build_progress::models::TransientLineCoordinator; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Return the coordinator shared by every terminal writer in this process. +#[inline] +pub fn shared_transient_line_coordinator( + py: Python<'_>, +) -> PyResult> { + transient_lines::shared_transient_line_coordinator(py) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs new file mode 100644 index 0000000000..ff290ab9f7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs @@ -0,0 +1,16 @@ +//! Run `body`, drawing a stderr spinner named `message` only once it has run for `delay`. + +use crate::build_progress::_helpers::transient_status; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::time::Duration; + +/// Run `body`, drawing a stderr spinner named `message` only once it has run for `delay`. +#[inline] +pub fn delayed_status<'py, T>( + py: Python<'py>, + (message, delay): (&str, Duration), + body: impl FnOnce() -> PyResult, +) -> PyResult { + transient_status::delayed_status(py, (message, delay), body) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs new file mode 100644 index 0000000000..5a049a5755 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs @@ -0,0 +1,15 @@ +//! Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. + +use crate::build_progress::_helpers::transient_status; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. +#[inline] +pub fn maybe_status<'py, T>( + py: Python<'py>, + (message, enabled): (&str, bool), + body: impl FnOnce() -> PyResult, +) -> PyResult { + transient_status::maybe_status(py, (message, enabled), body) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/mod.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/mod.rs new file mode 100644 index 0000000000..9ffa6dab98 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/mod.rs @@ -0,0 +1,5 @@ +pub mod delayed_status; +pub mod maybe_status; +pub mod progress_spinners_disabled; +pub mod register_transient_status; +pub mod status_reporter; diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/progress_spinners_disabled.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/progress_spinners_disabled.rs new file mode 100644 index 0000000000..a409a9672d --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/progress_spinners_disabled.rs @@ -0,0 +1,9 @@ +//! Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. + +use crate::build_progress::_helpers::transient_status; + +/// Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. +#[inline] +pub fn progress_spinners_disabled() -> bool { + transient_status::progress_spinners_disabled() +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/register_transient_status.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/register_transient_status.rs new file mode 100644 index 0000000000..2a350c2f77 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/register_transient_status.rs @@ -0,0 +1,12 @@ +//! Register the transient status functions and classes. + +use crate::build_progress::_helpers::transient_status; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the transient status functions and classes. +#[inline] +pub fn register_transient_status(module: &Bound<'_, PyModule>) -> PyResult<()> { + transient_status::register(module) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/main/transient_status/status_reporter.rs b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/status_reporter.rs new file mode 100644 index 0000000000..7007f17fd7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/main/transient_status/status_reporter.rs @@ -0,0 +1,15 @@ +//! A native status reporter writing to `stream`. + +use crate::build_progress::_helpers::transient_status; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A native status reporter writing to `stream`. +#[inline] +pub fn status_reporter<'py>( + stream: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + transient_status::status_reporter(stream, use_color) +} diff --git a/crates/sqlbuild-python-base/src/build_progress/mod.rs b/crates/sqlbuild-python-base/src/build_progress/mod.rs new file mode 100644 index 0000000000..ab860314b4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/mod.rs @@ -0,0 +1,6 @@ +//! Progress reporting on the terminal: the lifecycle projection, the transient line coordinator and status spinners. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/build_progress/models.rs b/crates/sqlbuild-python-base/src/build_progress/models.rs new file mode 100644 index 0000000000..dd2c38c3b9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/build_progress/models.rs @@ -0,0 +1,653 @@ +//! Public records of build progress. + +use crate::boundary::main::imported::imported; +use crate::boundary::main::within::within; +use crate::build_progress::_helpers::progress::context_variable; +use crate::build_progress::_helpers::progress::persist; +use crate::build_progress::_helpers::progress::progress_event; +use crate::build_progress::_helpers::progress::shared_coordinator; +use crate::build_progress::_helpers::progress::terminal_request; +use crate::build_progress::_helpers::progress::terminal_stream; +use crate::build_progress::_helpers::terminal::claim; +use crate::build_progress::_helpers::terminal::flush; +use crate::build_progress::_helpers::terminal::join_spinner; +use crate::build_progress::_helpers::terminal::release; +use crate::build_progress::_helpers::terminal::shared_lines; +use crate::build_progress::_helpers::terminal::start_spinner; +use crate::build_progress::_helpers::terminal::terminal_columns; +use crate::build_progress::_helpers::terminal::tty_flag; +use crate::build_progress::_helpers::terminal::write; +use crate::build_progress::_helpers::transient_lines::ActiveLine; +use crate::build_progress::_helpers::transient_status::PREFLIGHT_COMPLETION_PREFIX; +use crate::build_progress::_helpers::transient_status::StatusState; +use crate::build_progress::_helpers::transient_status::progress_spinners_disabled; +use crate::build_progress::constants::CLEAR_LINE; +use crate::build_progress::constants::HIDE_CURSOR; +use crate::build_progress::constants::SHOW_CURSOR; +use crate::build_progress::constants::SPINNER_FRAMES; +use crate::build_progress::constants::SPINNER_TICK_SECONDS; +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_items::mapping_items; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::claim_progress_terminal::claim_progress_terminal; +use sqlbuild_core::progress::main::close_progress_projection::close_progress_projection; +use sqlbuild_core::progress::main::consume_progress_event::consume_progress_event; +use sqlbuild_core::progress::main::format_status_spinner_line::format_status_spinner_line; +use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal_duration; +use sqlbuild_core::progress::models::ProgressEvent; +use sqlbuild_core::progress::models::ProgressProjection; +use sqlbuild_core::progress::models::ProgressSettings; +use sqlbuild_core::terminal_events::models::TerminalRequest; +use std::sync::Mutex; +use std::sync::MutexGuard; +use std::sync::PoisonError; + +/// Project canonical starts and correlate enriched terminal presentation. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeProgressProjector { + pub(crate) state: Mutex, + pub(crate) stream: Py, + pub(crate) coordinator: Py, + /// The lines held back instead of written, while something holds them. + pub(crate) held: Mutex>>, +} + +impl NativeProgressProjector { + pub(crate) fn held(&self) -> MutexGuard<'_, Option>> { + self.held.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Hold back every line from now on instead of writing it. + pub fn hold_lines(&self) { + *self.held() = Some(Vec::new()); + } + + /// Replace the lines held back so far with `lines`, returning them; nothing while not holding. + pub fn exchange_held_lines(&self, lines: Vec) -> Vec { + match self.held().as_mut() { + Some(held) => std::mem::replace(held, lines), + None => Vec::new(), + } + } + + /// Write lines as they come again, returning those held back since `hold_lines`. + pub fn release_lines(&self) -> Vec { + self.held().take().unwrap_or_default() + } + + /// Write lines held back earlier, as they would have been written. + pub fn write_held(&self, py: Python<'_>, lines: &[String]) -> PyResult<()> { + let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); + for line in lines { + persist(coordinator, self.stream.bind(py), line)?; + } + Ok(()) + } + + pub(crate) fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("native progress projector lock is poisoned")) + } + + /// Run `update` and write its lines while holding the shared terminal line lock. + pub(crate) fn write_with( + &self, + py: Python<'_>, + update: impl FnOnce(ProgressProjection) -> (ProgressProjection, Vec, T), + ) -> PyResult { + let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); + let lock: Bound<'_, PyAny> = native_field(coordinator, "lock")?; + lock.call_method0("acquire")?; + let result: PyResult = (|| { + let (lines, value) = { + let mut state: MutexGuard<'_, ProgressProjection> = self.locked()?; + let (projection, lines, value) = update(std::mem::take(&mut *state)); + *state = projection; + (lines, value) + }; + for line in lines { + let text: String = format!("{line}\n"); + if let Some(held) = self.held().as_mut() { + held.push(text); + continue; + } + persist(coordinator, self.stream.bind(py), &text)?; + } + Ok(value) + })(); + lock.call_method0("release")?; + result + } +} + +#[pymethods] +impl NativeProgressProjector { + #[new] + #[pyo3(signature = (*, stream, use_color, debug = false))] + pub fn new( + py: Python<'_>, + stream: Bound<'_, PyAny>, + use_color: bool, + debug: bool, + ) -> PyResult { + let coordinator: Bound<'_, PyAny> = shared_coordinator(py)?; + Ok(Self { + state: Mutex::new(ProgressProjection::new(ProgressSettings { + use_color, + debug, + is_tty: terminal_stream(&stream)?, + })), + stream: stream.unbind(), + coordinator: coordinator.unbind(), + held: Mutex::new(None), + }) + } + + /// Install this projector for presentation adapters in the invocation context. + pub(crate) fn install<'py>(slf: &Bound<'py, Self>) -> PyResult> { + context_variable(slf.py())?.call_method1("set", (slf,)) + } + + /// Restore the previous invocation projector. + pub(crate) fn restore(&self, py: Python<'_>, token: &Bound<'_, PyAny>) -> PyResult<()> { + context_variable(py)?.call_method1("reset", (token,))?; + Ok(()) + } + + /// Add safe display order metadata without changing canonical facts. + #[pyo3(signature = (*, ordinals, total))] + pub(crate) fn configure_resources( + &self, + ordinals: &Bound<'_, PyAny>, + total: i64, + ) -> PyResult<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + for item in mapping_items(ordinals)?.try_iter()? { + entries.push(item?.extract()?); + } + self.locked()?.configure_resources(entries, total); + Ok(()) + } + + /// Declare that an existing result callback will enrich this resource terminal. + #[pyo3(signature = (*, resource_name))] + pub(crate) fn expect_resource_enrichment(&self, resource_name: &str) -> PyResult<()> { + self.locked()?.expect_resource_enrichment(resource_name); + Ok(()) + } + + /// Record how one audit attempt ended, so its terminal row shows the outcome. + #[pyo3(signature = (*, resource_attempt_id, outcome))] + pub(crate) fn record_audit_outcome( + &self, + resource_attempt_id: &str, + outcome: &str, + ) -> PyResult<()> { + self.locked()? + .record_audit_outcome(resource_attempt_id, outcome); + Ok(()) + } + + /// Return whether a visible canonical operation currently owns progress. + #[pyo3(name = "is_operation_active", signature = (*, operation_name))] + pub fn operation_active(&self, operation_name: &str) -> PyResult { + Ok(self.locked()?.is_operation_active(operation_name)) + } + + /// Consume one known lifecycle fact idempotently. + pub(crate) fn consume(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event: ProgressEvent = progress_event(event)?; + self.write_with(py, |projection| { + let (projection, lines) = consume_progress_event(projection, &event); + (projection, lines, ()) + }) + } + + /// Claim the targeted or latest terminal for one final resource result. + #[pyo3(signature = (**options))] + pub(crate) fn consume_resource_terminal( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + let request: TerminalRequest = terminal_request(py, "consume_resource_terminal", options)?; + self.write_with(py, |projection| { + claim_progress_terminal(projection, &request) + }) + } + + /// Return a terminal duration without claiming its presentation. + #[pyo3(signature = (**options))] + pub(crate) fn resource_terminal_duration( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + let request: TerminalRequest = terminal_request(py, "resource_terminal_duration", options)?; + Ok(progress_terminal_duration(&*self.locked()?, &request)) + } + + /// Render resource terminals no result callback claimed. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + self.write_with(py, |projection| { + let (projection, lines) = close_progress_projection(projection); + (projection, lines, ()) + }) + } +} + +/// Keep persistent terminal lines from landing on a live spinner or status line. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct TransientLineCoordinator { + /// The re-entrant lock every writer holds while it draws. + #[pyo3(get)] + pub lock: Py, + pub(crate) active: Mutex>, +} + +impl TransientLineCoordinator { + pub(crate) fn active(&self) -> MutexGuard<'_, Option> { + self.active.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Run `body` while holding the coordinator lock. + pub(crate) fn locked( + &self, + py: Python<'_>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + let lock: &Bound<'_, PyAny> = self.lock.bind(py); + lock.call_method0("acquire")?; + let result: PyResult = body(); + lock.call_method0("release")?; + result + } + + /// The owner whose line a write to `stream` would land on. + pub(crate) fn interrupted_owner<'py>( + &self, + stream: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py: Python<'py> = stream.py(); + let Some((owner_stream, owner)) = self + .active() + .as_ref() + .map(|(owner_stream, owner)| (owner_stream.bind(py).clone(), owner.bind(py).clone())) + else { + return Ok(None); + }; + if owner_stream.is(stream) || (tty_flag(&owner_stream)? && tty_flag(stream)?) { + return Ok(Some(owner)); + } + Ok(None) + } +} + +#[pymethods] +impl TransientLineCoordinator { + #[new] + pub(crate) fn new(py: Python<'_>) -> PyResult { + Ok(Self { + lock: imported(py, "threading", "RLock")?.call0()?.unbind(), + active: Mutex::new(None), + }) + } + + /// Record the owner of the transient line currently drawn on a terminal stream. + #[pyo3(signature = (*, stream, owner))] + pub(crate) fn claim( + &self, + stream: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.locked(stream.py(), || { + *self.active() = Some((stream.clone().unbind(), owner.clone().unbind())); + Ok(()) + }) + } + + /// Forget a transient owner once its line is no longer drawn. + #[pyo3(signature = (*, owner))] + pub(crate) fn release(&self, owner: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(owner.py(), || { + let mut active = self.active(); + if active + .as_ref() + .is_some_and(|(_, current)| current.is(owner)) + { + *active = None; + } + Ok(()) + }) + } + + /// Erase and forget any live transient line, so nothing redraws it afterwards. + pub(crate) fn clear_active(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let taken: Option = self.active().take(); + if let Some((_, owner)) = taken { + owner.bind(py).call_method0("clear_transient_line")?; + } + Ok(()) + }) + } + + /// Write lasting text, clearing and then redrawing any live transient line. + #[pyo3(signature = (*, stream, text))] + pub(crate) fn write_persistent( + &self, + stream: &Bound<'_, PyAny>, + text: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.locked(stream.py(), || { + let owner: Option> = self.interrupted_owner(stream)?; + if let Some(owner) = &owner { + owner.call_method0("clear_transient_line")?; + } + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + if let Some(owner) = &owner { + owner.call_method0("redraw_transient_line")?; + } + Ok(()) + }) + } +} + +/// Render one transient status line with optional spinner updates. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct TransientStatusReporter { + pub(crate) stream: Py, + pub(crate) lines: Py, + pub(crate) lock: Py, + pub(crate) style: CliStyle, + pub(crate) enabled: bool, + pub(crate) state: Mutex, +} + +impl TransientStatusReporter { + pub(crate) fn state(&self) -> MutexGuard<'_, StatusState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) fn locked( + &self, + py: Python<'_>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + pub fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + persist(self.lines.bind(py), self.stream.bind(py), text) + } + + pub(crate) fn write_message( + &self, + py: Python<'_>, + message: &str, + dim_output: bool, + ) -> PyResult<()> { + let formatted: String = if dim_output { + self.style.muted(message) + } else { + message.to_owned() + }; + self.write_persistent(py, &format!("{formatted}\n")) + } + + pub(crate) fn draw(&self, py: Python<'_>) -> PyResult<()> { + let (message, frame) = { + let mut state = self.state(); + let Some(message) = state.message.clone().filter(|_| state.drawn) else { + return Ok(()); + }; + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + (message, frame) + }; + let line: String = format_status_spinner_line( + SPINNER_FRAMES[frame], + &message, + terminal_columns(py)?, + self.style, + ); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{line}"))?; + flush(stream) + } + + pub(crate) fn enter_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let register_hook: bool = + !std::mem::replace(&mut reporter.state().exit_hook_registered, true); + if register_hook { + imported(py, "atexit", "register")?.call1((native_field(slf, "close")?,))?; + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, HIDE_CURSOR)?; + reporter.state().drawn = true; + reporter.draw(py)?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.threads.push(thread.unbind()); + Ok(()) + } + + pub(crate) fn close_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + release(reporter.lines.bind(py), slf.as_any())?; + reporter.exit_status(py)?; + let mut state = reporter.state(); + state.message = None; + state.active = false; + Ok(()) + }) + } + + pub(crate) fn exit_status(&self, py: Python<'_>) -> PyResult<()> { + let (drawn, event) = { + let mut state = self.state(); + (std::mem::take(&mut state.drawn), state.stop_event.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if drawn { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; + flush(stream)?; + } + Ok(()) + } +} + +#[pymethods] +impl TransientStatusReporter { + #[new] + #[pyo3(signature = (*, stream, use_color = false, enabled = true))] + pub fn new( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + use_color: bool, + enabled: bool, + ) -> PyResult { + let lines: Bound<'_, PyAny> = shared_lines(py)?; + Ok(Self { + enabled: enabled && !progress_spinners_disabled() && tty_flag(stream)?, + stream: stream.clone().unbind(), + lock: native_field(&lines, "lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { use_color }, + state: Mutex::new(StatusState::default()), + }) + } + + pub fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if reporter.enabled { + return reporter.locked(py, || { + Self::close_status(slf)?; + reporter.state().message = Some(message); + Self::enter_status(slf)?; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.state().active = true; + Ok(()) + }); + } + reporter.state().active = true; + reporter.write_message(py, &message, true) + } + + pub fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { + if self.enabled { + let updated: bool = self.locked(py, || { + let drawn: bool = { + let mut state = self.state(); + if state.message.is_none() { + return Ok(false); + } + state.message = Some(message.clone()); + state.drawn + }; + if drawn { + self.draw(py)?; + } + Ok(true) + })?; + if updated { + return Ok(()); + } + } + self.write_message(py, &message, true) + } + + #[pyo3(signature = (*, message, blank_line_after = false))] + pub fn complete(slf: &Bound<'_, Self>, message: &str, blank_line_after: bool) -> PyResult<()> { + let py: Python<'_> = slf.py(); + Self::close(slf)?; + slf.get().write_message(py, message, true)?; + if blank_line_after { + slf.get().write_blank_line(py)?; + } + Ok(()) + } + + /// Complete with a pre-styled message written verbatim. + #[pyo3(signature = (*, message, blank_line_after = false))] + pub fn complete_styled( + slf: &Bound<'_, Self>, + message: &str, + blank_line_after: bool, + ) -> PyResult<()> { + Self::close(slf)?; + let blank: &str = if blank_line_after { "\n" } else { "" }; + slf.get() + .write_persistent(slf.py(), &format!("{message}\n{blank}")) + } + + pub fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { + Self::close(slf)?; + slf.get().write_message(slf.py(), message, false) + } + + pub(crate) fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + Self::close_status(slf)?; + let threads: Vec> = std::mem::take(&mut slf.get().state().threads); + for thread in threads { + join_spinner(thread.bind(slf.py()))?; + } + Ok(()) + } + + /// Erase the live status so a persistent line can be written in its place; the spinner idles. + pub(crate) fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + if std::mem::take(&mut self.state().drawn) { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; + flush(stream)?; + } + Ok(()) + } + + /// Draw the live status again below persistent output, reusing its running spinner. + pub(crate) fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + let reporter: &Self = slf.get(); + let (redraw, spinning) = { + let state = reporter.state(); + ( + state.message.is_some() && !state.drawn, + state.stop_event.is_some(), + ) + }; + if !redraw { + return Ok(()); + } + if !spinning { + return Self::enter_status(slf); + } + write(reporter.stream.bind(slf.py()), HIDE_CURSOR)?; + reporter.state().drawn = true; + reporter.draw(slf.py()) + } + + /// Render one preflight progress update. + pub(crate) fn report_preflight_progress( + slf: &Bound<'_, Self>, + message: String, + ) -> PyResult<()> { + if message.starts_with(PREFLIGHT_COMPLETION_PREFIX) { + return Self::complete(slf, &message, true); + } + if !slf.get().state().active { + return Self::start(slf, message); + } + slf.get().update(slf.py(), message) + } + + pub(crate) fn write_blank_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_persistent(py, "\n") + } + + pub(crate) fn _spin_until_stopped( + &self, + py: Python<'_>, + stop_event: &Bound<'_, PyAny>, + ) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if stop_event.call_method0("is_set")?.is_truthy()? { + return Ok(()); + } + self.draw(py) + })?; + } + Ok(()) + } +} diff --git a/crates/sqlbuild-python-base/src/cache/_helpers/mod.rs b/crates/sqlbuild-python-base/src/cache/_helpers/mod.rs new file mode 100644 index 0000000000..9c31e52498 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod native_parse_store; +pub(crate) mod native_store; +pub(crate) mod sql_test_scan_store; diff --git a/crates/sqlbuild-python-base/src/cache/_helpers/native_parse_store.rs b/crates/sqlbuild-python-base/src/cache/_helpers/native_parse_store.rs new file mode 100644 index 0000000000..eb2803ad2f --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/_helpers/native_parse_store.rs @@ -0,0 +1,22 @@ +//! Statements a dialect's native parser read whole, kept so a warm compile skips parsing them again. + +use sqlbuild_analysis::assembly::project::models::SyntaxCheck; +use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::types::ContentDigest; + +/// The native store kind holding read-whole verdicts. +pub(crate) const NATIVE_PARSE_STORE_KIND: &str = "native-parses"; +/// The store file below the compile cache folder. +pub(crate) const NATIVE_PARSE_STORE_FILE_NAME: &str = "native-parses.bin"; +/// Bumped whenever what a key covers changes; older stores open empty. +pub(crate) const NATIVE_PARSE_STORE_VERSION: &str = "native-parse-store-v1"; + +/// The key `dialect`'s verdict on `check` is stored under. +pub(crate) fn parse_key(dialect: &str, check: &SyntaxCheck) -> ContentDigest { + let mut parts: Vec<&str> = vec![dialect, &check.sql]; + for (name, value) in &check.placeholders { + parts.push(name); + parts.push(value); + } + content_digest(&parts) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs rename to crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs index 5922e33080..cbf4d147d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs +++ b/crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs @@ -18,10 +18,10 @@ use sqlbuild_host::entries::models::FacadeHost; use std::collections::HashSet; use std::path::{Path, PathBuf}; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; -use sqlbuild_python_base::types::CompilerDetach; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::registry::native_attr::native_attr; +use crate::records::types::NativeAttr; +use crate::types::CompilerDetach; /// Hexadecimal digest of length-prefixed UTF-8 `parts`, the digest every native cache key uses. #[pyfunction] diff --git a/crates/sqlbuild-python-base/src/cache/_helpers/sql_test_scan_store.rs b/crates/sqlbuild-python-base/src/cache/_helpers/sql_test_scan_store.rs new file mode 100644 index 0000000000..0b649f8320 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/_helpers/sql_test_scan_store.rs @@ -0,0 +1,39 @@ +//! Whole per-file SQL-test scan results kept as one kind of the shared native store. + +use crate::cache::models::SqlTestScanCache; +use crate::cache::models::SqlTestScanStore; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; + +use crate::boundary::main::imported::imported; + +const SCAN_CACHE_LOGGER: &str = "sqlbuild.compile"; + +/// The native store kind holding SQL-test scan results. +pub(crate) const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; +/// The store file below the compile cache folder. +pub(crate) const SQL_TEST_SCAN_STORE_FILE_NAME: &str = "sql-test-scans.bin"; +/// Bumped whenever stored scan values change shape; older stores open empty. +pub(crate) const SQL_TEST_SCAN_STORE_VERSION: &str = "sql-test-scan-store-v1"; + +pub(crate) fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("SQL test scan store is poisoned") +} + +/// `message` with `error` at debug level on the compile logger, as the compile caches report. +pub(crate) fn debug(py: Python<'_>, message: &str, error: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((SCAN_CACHE_LOGGER,))? + .call_method1("debug", (message, error))?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add( + "SQL_TEST_SCAN_STORE_FILE_NAME", + SQL_TEST_SCAN_STORE_FILE_NAME, + )?; + Ok(()) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/content_digest.rs b/crates/sqlbuild-python-base/src/cache/main/content_digest.rs new file mode 100644 index 0000000000..5dd461ec68 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/content_digest.rs @@ -0,0 +1,9 @@ +//! Hexadecimal digest of length-prefixed UTF-8 `parts`, the digest every native cache key uses. + +use crate::cache::_helpers::native_store; + +/// Hexadecimal digest of length-prefixed UTF-8 `parts`, the digest every native cache key uses. +#[inline] +pub fn content_digest(parts: Vec) -> String { + native_store::content_digest(parts) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/current_code_identity.rs b/crates/sqlbuild-python-base/src/cache/main/current_code_identity.rs new file mode 100644 index 0000000000..7897cef30a --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/current_code_identity.rs @@ -0,0 +1,11 @@ +//! `compiled_code_identity`, looked up on the extension module so tests can replace it. + +use crate::cache::_helpers::native_store; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `compiled_code_identity`, looked up on the extension module so tests can replace it. +#[inline] +pub fn current_code_identity(py: Python<'_>) -> PyResult { + native_store::current_code_identity(py) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/digest_files.rs b/crates/sqlbuild-python-base/src/cache/main/digest_files.rs new file mode 100644 index 0000000000..7f9029b8be --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/digest_files.rs @@ -0,0 +1,12 @@ +//! Hexadecimal content digest of each file in `paths`, or `None` where it cannot be read. + +use crate::cache::_helpers::native_store; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::PathBuf; + +/// Hexadecimal content digest of each file in `paths`, or `None` where it cannot be read. +#[inline] +pub fn digest_files(py: Python<'_>, paths: Vec) -> PyResult>> { + native_store::digest_files(py, paths) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/mod.rs b/crates/sqlbuild-python-base/src/cache/main/mod.rs new file mode 100644 index 0000000000..7ab4e91ae8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/mod.rs @@ -0,0 +1,8 @@ +pub mod content_digest; +pub mod current_code_identity; +pub mod digest_files; +pub mod open_store; +pub mod parse_key; +pub mod register_native_store; +pub mod register_sql_test_scan_store; +pub mod save_store; diff --git a/crates/sqlbuild-python-base/src/cache/main/open_store.rs b/crates/sqlbuild-python-base/src/cache/main/open_store.rs new file mode 100644 index 0000000000..82b792152b --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/open_store.rs @@ -0,0 +1,18 @@ +//! Open one cache kind's store with the GIL released; a failed read is an `OSError`. + +use crate::cache::_helpers::native_store; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_cache::store::models::NativeStore; +use std::path::Path; + +/// Open one cache kind's store with the GIL released; a failed read is an `OSError`. +#[inline] +pub fn open_store( + py: Python<'_>, + path: &Path, + kind: &str, + environment: &str, +) -> PyResult { + native_store::open_store(py, path, kind, environment) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/parse_key.rs b/crates/sqlbuild-python-base/src/cache/main/parse_key.rs new file mode 100644 index 0000000000..a4b390b8f9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/parse_key.rs @@ -0,0 +1,11 @@ +//! The key `dialect`'s verdict on `check` is stored under. + +use crate::cache::_helpers::native_parse_store; +use sqlbuild_analysis::assembly::project::models::SyntaxCheck; +use sqlbuild_cache::digest::types::ContentDigest; + +/// The key `dialect`'s verdict on `check` is stored under. +#[inline] +pub fn parse_key(dialect: &str, check: &SyntaxCheck) -> ContentDigest { + native_parse_store::parse_key(dialect, check) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/register_native_store.rs b/crates/sqlbuild-python-base/src/cache/main/register_native_store.rs new file mode 100644 index 0000000000..da771211dc --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/register_native_store.rs @@ -0,0 +1,12 @@ +//! Register the native store functions and classes. + +use crate::cache::_helpers::native_store; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the native store functions and classes. +#[inline] +pub fn register_native_store(module: &Bound<'_, PyModule>) -> PyResult<()> { + native_store::register(module) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/register_sql_test_scan_store.rs b/crates/sqlbuild-python-base/src/cache/main/register_sql_test_scan_store.rs new file mode 100644 index 0000000000..6c341fd9e7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/register_sql_test_scan_store.rs @@ -0,0 +1,12 @@ +//! Register the sql test scan store functions and classes. + +use crate::cache::_helpers::sql_test_scan_store; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sql test scan store functions and classes. +#[inline] +pub fn register_sql_test_scan_store(module: &Bound<'_, PyModule>) -> PyResult<()> { + sql_test_scan_store::register(module) +} diff --git a/crates/sqlbuild-python-base/src/cache/main/save_store.rs b/crates/sqlbuild-python-base/src/cache/main/save_store.rs new file mode 100644 index 0000000000..90673ecd4c --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/main/save_store.rs @@ -0,0 +1,18 @@ +//! Atomically save a store holding unsaved values with the GIL released; returns entries written. + +use crate::cache::_helpers::native_store; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_cache::store::models::NativeStore; +use std::path::Path; + +/// Atomically save a store holding unsaved values with the GIL released; returns entries written. +#[inline] +pub fn save_store( + py: Python<'_>, + store: &NativeStore, + path: &Path, + metadata: &[u8], +) -> PyResult> { + native_store::save_store(py, store, path, metadata) +} diff --git a/crates/sqlbuild-python-base/src/cache/mod.rs b/crates/sqlbuild-python-base/src/cache/mod.rs new file mode 100644 index 0000000000..f5214f24c8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cache/mod.rs @@ -0,0 +1,5 @@ +//! The shared native store and the parse and SQL-test scan results kept in it. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python-base/src/cache/models.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs rename to crates/sqlbuild-python-base/src/cache/models.rs index 18a1fc6afc..5f606e46ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs +++ b/crates/sqlbuild-python-base/src/cache/models.rs @@ -1,49 +1,89 @@ -//! Whole per-file SQL-test scan results kept as one kind of the shared native store. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyclass, pymethods}; +//! Public records of cache. + +use crate::cache::_helpers::native_parse_store::NATIVE_PARSE_STORE_FILE_NAME; +use crate::cache::_helpers::native_parse_store::NATIVE_PARSE_STORE_KIND; +use crate::cache::_helpers::native_parse_store::NATIVE_PARSE_STORE_VERSION; +use crate::cache::_helpers::native_store::current_code_identity; +use crate::cache::_helpers::native_store::open_store; +use crate::cache::_helpers::native_store::save_store; +use crate::cache::_helpers::sql_test_scan_store::SQL_TEST_SCAN_STORE_FILE_NAME; +use crate::cache::_helpers::sql_test_scan_store::SQL_TEST_SCAN_STORE_KIND; +use crate::cache::_helpers::sql_test_scan_store::SQL_TEST_SCAN_STORE_VERSION; +use crate::cache::_helpers::sql_test_scan_store::debug; +use crate::cache::_helpers::sql_test_scan_store::poisoned; +use pyo3::prelude::Py; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; use sqlbuild_cache::digest::main::content_digest::content_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; use sqlbuild_host::compile_context::main::record_compile_metric::record_compile_metric; +use std::path::Path; use std::path::PathBuf; use std::sync::Mutex; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; -use crate::bindings::_helpers::cache::native_store::current_code_identity; -use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; -use sqlbuild_python_base::boundary::main::imported::imported; +/// The statements this build's native parsers read whole, by dialect, SQL and placeholders. +#[derive(Debug)] +pub struct NativeParseStore { + pub(crate) store: NativeStore, + pub(crate) path: PathBuf, +} -const SCAN_CACHE_LOGGER: &str = "sqlbuild.compile"; +impl NativeParseStore { + /// The compile cache folder's store; none when it cannot open, which is logged for debugging. + pub fn open_in(py: Python<'_>, cache_dir: &Path) -> PyResult> { + let path: PathBuf = cache_dir.join(NATIVE_PARSE_STORE_FILE_NAME); + let identity: String = current_code_identity(py)?; + let environment: String = + hex_digest(&content_digest(&[NATIVE_PARSE_STORE_VERSION, &identity])); + match open_store(py, &path, NATIVE_PARSE_STORE_KIND, &environment) { + Ok(store) => Ok(Some(Self { store, path })), + Err(error) => { + debug(py, "native parse store not loaded: %s", &error.to_string())?; + Ok(None) + } + } + } -/// The native store kind holding SQL-test scan results. -const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; -/// The store file below the compile cache folder. -const SQL_TEST_SCAN_STORE_FILE_NAME: &str = "sql-test-scans.bin"; -/// Bumped whenever stored scan values change shape; older stores open empty. -const SQL_TEST_SCAN_STORE_VERSION: &str = "sql-test-scan-store-v1"; + /// Whether the parser read the statement under `key` whole in an earlier compile. + pub fn read_whole(&mut self, key: &ContentDigest) -> bool { + self.store.get(key).is_some() + } + + /// Record that the parser read the statement under `key` whole. + pub fn record(&mut self, key: ContentDigest) { + self.store.put(key, Vec::new()); + } + + /// Atomically save verdicts this compile recorded; a failed save is logged for debugging. + pub fn save(&self, py: Python<'_>) -> PyResult<()> { + if let Err(error) = save_store(py, &self.store, &self.path, &[]) { + debug(py, "native parse store not saved: %s", &error.to_string())?; + } + Ok(()) + } +} /// One compile's view of the stored scan results, keyed by the digest of the caller's key parts. #[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] -pub(crate) struct SqlTestScanStore { - store: Mutex, - path: PathBuf, - hits: AtomicUsize, - misses: AtomicUsize, -} - -fn poisoned(_: T) -> pyo3::PyErr { - PyValueError::new_err("SQL test scan store is poisoned") +pub struct SqlTestScanStore { + pub(crate) store: Mutex, + pub(crate) path: PathBuf, + pub(crate) hits: AtomicUsize, + pub(crate) misses: AtomicUsize, } #[pymethods] impl SqlTestScanStore { /// Open the compile cache folder's store for `code_identity`; the error when it cannot open. #[staticmethod] - fn open_in( + pub(crate) fn open_in( py: Python<'_>, cache_dir: PathBuf, code_identity: &str, @@ -68,7 +108,7 @@ impl SqlTestScanStore { } /// The hits and misses counted by this compile. - fn counts(&self) -> (usize, usize) { + pub(crate) fn counts(&self) -> (usize, usize) { ( self.hits.load(Ordering::Relaxed), self.misses.load(Ordering::Relaxed), @@ -76,7 +116,7 @@ impl SqlTestScanStore { } /// Atomically save values this compile stored; entries written, or the error that stopped it. - fn save(&self, py: Python<'_>) -> PyResult<(Option, Option)> { + pub(crate) fn save(&self, py: Python<'_>) -> PyResult<(Option, Option)> { let store = self.store.lock().map_err(poisoned)?; Ok(match save_store(py, &store, &self.path, &[]) { Ok(written) => (written, None), @@ -87,7 +127,7 @@ impl SqlTestScanStore { impl SqlTestScanStore { /// The stored value under `key_parts`, counted as a hit when `decode` accepts it. - pub(crate) fn decoded( + pub fn decoded( &self, key_parts: Vec, decode: impl FnOnce(&[u8]) -> Option, @@ -109,7 +149,7 @@ impl SqlTestScanStore { } /// Store one complete result under `key_parts`. - pub(crate) fn stored(&self, key_parts: Vec, value: Vec) -> PyResult<()> { + pub fn stored(&self, key_parts: Vec, value: Vec) -> PyResult<()> { let key: ContentDigest = content_digest(&key_parts); self.store.lock().map_err(poisoned)?.put(key, value); Ok(()) @@ -118,22 +158,23 @@ impl SqlTestScanStore { /// The compile cache folder's scan store, opened on first use and saved once. #[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct SqlTestScanCache { - state: Mutex<(Option, Option>)>, +#[derive(Debug)] +pub struct SqlTestScanCache { + pub(crate) state: Mutex<(Option, Option>)>, } #[pymethods] impl SqlTestScanCache { #[new] #[pyo3(signature = (*, cache_dir))] - pub(crate) fn new(cache_dir: Option) -> Self { + pub fn new(cache_dir: Option) -> Self { Self { state: Mutex::new((cache_dir, None)), } } /// The native store this compile reads and writes, or None when it has none. - fn opened(&self, py: Python<'_>) -> PyResult>> { + pub(crate) fn opened(&self, py: Python<'_>) -> PyResult>> { let mut state = self.state.lock().map_err(poisoned)?; let (cache_dir, store) = &mut *state; let Some(directory) = cache_dir.as_ref().filter(|_| store.is_none()) else { @@ -150,7 +191,7 @@ impl SqlTestScanCache { } /// Persist results stored by this compile and report hit and miss counts. - fn save(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn save(&self, py: Python<'_>) -> PyResult<()> { let store: Option> = self .state .lock() @@ -176,21 +217,3 @@ impl SqlTestScanCache { Ok(()) } } - -/// `message` with `error` at debug level on the compile logger, as the compile caches report. -pub(crate) fn debug(py: Python<'_>, message: &str, error: &str) -> PyResult<()> { - imported(py, "logging", "getLogger")? - .call1((SCAN_CACHE_LOGGER,))? - .call_method1("debug", (message, error))?; - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add( - "SQL_TEST_SCAN_STORE_FILE_NAME", - SQL_TEST_SCAN_STORE_FILE_NAME, - )?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs b/crates/sqlbuild-python-base/src/cli/_helpers/launch_state.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs rename to crates/sqlbuild-python-base/src/cli/_helpers/launch_state.rs diff --git a/crates/sqlbuild-python-base/src/cli/_helpers/mod.rs b/crates/sqlbuild-python-base/src/cli/_helpers/mod.rs new file mode 100644 index 0000000000..05ff5f3038 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod launch_state; diff --git a/crates/sqlbuild-python-base/src/cli/main/launch_record_path.rs b/crates/sqlbuild-python-base/src/cli/main/launch_record_path.rs new file mode 100644 index 0000000000..21752f14d0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/main/launch_record_path.rs @@ -0,0 +1,13 @@ +//! The offered launch record path, unless an input outside its key was read. + +use crate::cli::_helpers::launch_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The offered launch record path, unless an input outside its key was read. +#[inline] +pub fn launch_record_path(py: Python<'_>) -> PyResult> { + launch_state::launch_record_path(py) +} diff --git a/crates/sqlbuild-python-base/src/cli/main/mod.rs b/crates/sqlbuild-python-base/src/cli/main/mod.rs new file mode 100644 index 0000000000..1125121246 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/main/mod.rs @@ -0,0 +1,4 @@ +pub mod launch_record_path; +pub mod offer_launch_record; +pub mod register_launch_state; +pub mod withhold_launch_record; diff --git a/crates/sqlbuild-python-base/src/cli/main/offer_launch_record.rs b/crates/sqlbuild-python-base/src/cli/main/offer_launch_record.rs new file mode 100644 index 0000000000..ca360968ca --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/main/offer_launch_record.rs @@ -0,0 +1,13 @@ +//! Remember where `sqb` looks for this invocation's launch record, or that it offered none. + +use crate::cli::_helpers::launch_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Remember where `sqb` looks for this invocation's launch record, or that it offered none. +#[inline] +pub fn offer_launch_record(py: Python<'_>, path: Option>) -> PyResult<()> { + launch_state::offer_launch_record(py, path) +} diff --git a/crates/sqlbuild-python-base/src/cli/main/register_launch_state.rs b/crates/sqlbuild-python-base/src/cli/main/register_launch_state.rs new file mode 100644 index 0000000000..4097b6dc67 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/main/register_launch_state.rs @@ -0,0 +1,12 @@ +//! Register the launch state functions and classes. + +use crate::cli::_helpers::launch_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the launch state functions and classes. +#[inline] +pub fn register_launch_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + launch_state::register(module) +} diff --git a/crates/sqlbuild-python-base/src/cli/main/withhold_launch_record.rs b/crates/sqlbuild-python-base/src/cli/main/withhold_launch_record.rs new file mode 100644 index 0000000000..4a1d4f7798 --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/main/withhold_launch_record.rs @@ -0,0 +1,11 @@ +//! Withhold the launch record: `sqb` cannot replay what this invocation also depends on. + +use crate::cli::_helpers::launch_state; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Withhold the launch record: `sqb` cannot replay what this invocation also depends on. +#[inline] +pub fn withhold_launch_record(py: Python<'_>) -> PyResult<()> { + launch_state::withhold_launch_record(py) +} diff --git a/crates/sqlbuild-python-base/src/cli/mod.rs b/crates/sqlbuild-python-base/src/cli/mod.rs new file mode 100644 index 0000000000..2e26621eda --- /dev/null +++ b/crates/sqlbuild-python-base/src/cli/mod.rs @@ -0,0 +1,4 @@ +//! The launch record the native `sqb` executable offered this invocation. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs b/crates/sqlbuild-python-base/src/command_runtime/_helpers/adapters.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs rename to crates/sqlbuild-python-base/src/command_runtime/_helpers/adapters.rs index 83745810bc..798a4e6da7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs +++ b/crates/sqlbuild-python-base/src/command_runtime/_helpers/adapters.rs @@ -7,11 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::ErrorId; +use crate::boundary::main::call_with::call_with; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::construct::error_class::error_class; +use crate::records::main::fields::field::field as native_field; +use crate::records::types::ErrorId; const ADAPTER_CONFIGURATION_CODE: &str = "C601"; diff --git a/crates/sqlbuild-python-base/src/command_runtime/_helpers/mod.rs b/crates/sqlbuild-python-base/src/command_runtime/_helpers/mod.rs new file mode 100644 index 0000000000..0ec4e62d43 --- /dev/null +++ b/crates/sqlbuild-python-base/src/command_runtime/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod adapters; diff --git a/crates/sqlbuild-python-base/src/command_runtime/main/mod.rs b/crates/sqlbuild-python-base/src/command_runtime/main/mod.rs new file mode 100644 index 0000000000..ba9f491ad3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/command_runtime/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_adapters; +pub mod resolve_cli_adapter; diff --git a/crates/sqlbuild-python-base/src/command_runtime/main/register_adapters.rs b/crates/sqlbuild-python-base/src/command_runtime/main/register_adapters.rs new file mode 100644 index 0000000000..3dd87e6f98 --- /dev/null +++ b/crates/sqlbuild-python-base/src/command_runtime/main/register_adapters.rs @@ -0,0 +1,12 @@ +//! Register the adapters functions and classes. + +use crate::command_runtime::_helpers::adapters; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the adapters functions and classes. +#[inline] +pub fn register_adapters(module: &Bound<'_, PyModule>) -> PyResult<()> { + adapters::register(module) +} diff --git a/crates/sqlbuild-python-base/src/command_runtime/main/resolve_cli_adapter.rs b/crates/sqlbuild-python-base/src/command_runtime/main/resolve_cli_adapter.rs new file mode 100644 index 0000000000..c9bd4ce08f --- /dev/null +++ b/crates/sqlbuild-python-base/src/command_runtime/main/resolve_cli_adapter.rs @@ -0,0 +1,15 @@ +//! The registered adapter for `adapter_name`, resolved through the adapter registry. + +use crate::command_runtime::_helpers::adapters; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The registered adapter for `adapter_name`, resolved through the adapter registry. +#[inline] +pub fn resolve_cli_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + adapters::resolve_cli_adapter(adapter_name, project_dir) +} diff --git a/crates/sqlbuild-python-base/src/command_runtime/mod.rs b/crates/sqlbuild-python-base/src/command_runtime/mod.rs new file mode 100644 index 0000000000..d142c2b2a6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/command_runtime/mod.rs @@ -0,0 +1,4 @@ +//! Adapter resolution for CLI commands. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/coverage.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/coverage.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/coverage.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/coverage.rs index 1c14d9a98d..4d92db5c5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/coverage.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/coverage.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap, HashSet}; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; +use crate::compile_context::models::ModelCoverage; /// The reason a model that opts out of SQL analysis is partial. pub(crate) const ANALYSIS_DISABLED: &str = "SQL analysis disabled"; diff --git a/crates/sqlbuild-python-base/src/compile_context/_helpers/delta.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/delta.rs new file mode 100644 index 0000000000..868339fa09 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/delta.rs @@ -0,0 +1,250 @@ +//! Process-wide slots for what a full compile captures and what a delta compile injects. + +use crate::compile_context::constants::ERRORS_ON_DELTA_PATH; +use crate::compile_context::models::DeltaInjection; +use crate::compile_context::models::ImageCapture; +use crate::compile_context::models::RecordMerge; +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::analysis_session::models::{ + DynamicFamily, RelationTables, SessionRequest, +}; +use sqlbuild_cache::project_reuse::types::ImageSections; + +static CAPTURE: Mutex> = Mutex::new(None); +static INJECTION: Mutex>> = Mutex::new(None); + +/// Why the running delta compile cannot continue, once something ruled it out. +static UNSUPPORTED: Mutex> = Mutex::new(None); +static PENDING_IMAGE: Mutex> = Mutex::new(None); +static RECORD_MERGE: Mutex> = Mutex::new(None); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Start capturing image facts for the compile about to run. +pub(crate) fn start_capture() { + *locked(&CAPTURE) = Some(ImageCapture::default()); +} + +/// Stop capturing without keeping anything, for a project that turns delta compiles off. +pub(crate) fn stop_capture() { + *locked(&CAPTURE) = None; +} + +/// Stop capturing and take what was captured. +pub(crate) fn take_capture() -> Option { + locked(&CAPTURE).take() +} + +/// Whether the running compile captures image facts. +pub(crate) fn capturing() -> bool { + locked(&CAPTURE).is_some() +} + +/// Record into the capture, when one is running. +pub(crate) fn capture(record: impl FnOnce(&mut ImageCapture)) { + if let Some(capture) = locked(&CAPTURE).as_mut() { + record(capture); + } +} + +/// Record sources a resource other than a model reads, when a capture is running. +pub(crate) fn record_source_reads<'a>(names: impl IntoIterator) { + capture(|capture| { + capture + .pinned_sources + .extend(names.into_iter().map(str::to_owned)); + }); +} + +/// Record sources an SQL test file reads, when a capture is running. +pub(crate) fn record_test_source_reads<'a>( + relative_path: &str, + names: impl IntoIterator, +) { + capture(|capture| { + capture + .test_sources + .entry(relative_path.replace('\\', "/")) + .or_default() + .extend(names.into_iter().map(str::to_owned)); + }); +} + +/// Mark the running capture as unusable as a delta base. +pub(crate) fn capture_unsupported(reason: &str) { + capture(|capture| { + if capture.unsupported.is_none() { + capture.unsupported = Some(reason.to_owned()); + } + }); +} + +/// Run `body` with `injection` visible to the rerun stages; returns it unless still held. +pub(crate) fn with_injection( + injection: DeltaInjection, + body: impl FnOnce() -> T, +) -> (T, Option) { + *locked(&INJECTION) = Some(Arc::new(injection)); + let result: T = body(); + let injection: Option> = locked(&INJECTION).take(); + ( + result, + injection.and_then(|injection| { + let Ok(injection) = Arc::try_unwrap(injection) else { + return None; + }; + Some(injection) + }), + ) +} + +/// The running delta compile's injection, if any. +pub(crate) fn injection() -> Option> { + locked(&INJECTION).clone() +} + +/// Rule out the running delta compile: returns the error that unwinds to its fallback. +pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { + let mut slot = locked(&UNSUPPORTED); + if slot.is_none() { + *slot = Some(reason.to_owned()); + } + pyo3::exceptions::PyRuntimeError::new_err(format!("delta compile unsupported: {reason}")) +} + +/// In a delta compile that leaves errors to full compiles, stop once `failed` reports one. +pub(crate) fn stop_on_errors(failed: bool) -> pyo3::PyResult<()> { + let errors_on_delta_path: bool = ERRORS_ON_DELTA_PATH; + if failed && !errors_on_delta_path && injection().is_some() { + return Err(delta_unsupported("the edit reports errors")); + } + Ok(()) +} + +/// Take the reason the delta compile was ruled out, if any. +pub(crate) fn take_delta_unsupported() -> Option { + locked(&UNSUPPORTED).take() +} + +/// The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. +pub(crate) fn injected_fact_digest(key: &[String]) -> Option { + let injection: Arc = injection()?; + if key + .iter() + .skip(1) + .any(|part| injection.rerun_names.contains(part)) + { + return None; + } + injection.fact_digests.get(key).cloned() +} + +/// Give a delta's analysis session the whole project's relation facts and analysis order. +pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { + let Some(injection) = injection() else { + return Ok(request); + }; + let relations: &RelationTables = &injection.relations; + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request.retain_relations = true; + let analysed: BTreeSet<&str> = request + .models + .iter() + .map(|model| model.name.as_str()) + .collect(); + let rerun: BTreeSet<&str> = analysed + .iter() + .copied() + .chain(injection.rerun_names.iter().map(String::as_str)) + .collect(); + request.dynamic_families_by_table = injected_families( + &request.dynamic_families_by_table, + &relations.families, + &rerun, + )?; + for (table, injected) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: BTreeSet = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + injected + .iter() + .filter(|(name, _)| !analysed.contains(name.as_str()) && !present.contains(name)) + .cloned(), + ); + } + Ok(request) +} + +/// The stored compile's dynamic column families, when no rerun model changed its own. +fn injected_families( + current: &[(String, Vec)], + stored: &[(String, Vec)], + rerun: &BTreeSet<&str>, +) -> pyo3::PyResult)>> { + let families_of = |table: &[(String, Vec)], name: &str| { + table + .iter() + .find(|(table, _)| table == name) + .map(|(_, families)| families.clone()) + }; + let changed: bool = current + .iter() + .any(|(name, _)| !rerun.contains(name.as_str())) + || rerun + .iter() + .any(|name| families_of(current, name) != families_of(stored, name)); + if changed { + return Err(delta_unsupported( + "a model's dynamic column families changed", + )); + } + Ok(stored.to_vec()) +} + +/// Hand the next reuse record the image to keep beside it. +pub(crate) fn set_pending_image(image: Option) { + *locked(&PENDING_IMAGE) = image; +} + +/// The image the next reuse record keeps, taken. +pub(crate) fn take_pending_image() -> Option { + locked(&PENDING_IMAGE).take() +} + +/// Hand the next reuse record what it keeps of the patched compile. +pub(crate) fn set_record_merge(merge: Option) { + *locked(&RECORD_MERGE) = merge; +} + +/// What the next reuse record keeps of the patched compile, taken. +pub(crate) fn take_record_merge() -> Option { + locked(&RECORD_MERGE).take() +} + +/// Forget every capture, injection and pending record fact, after a compile ends either way. +pub(crate) fn reset() { + let _ = take_capture(); + *locked(&INJECTION) = None; + let _ = take_delta_unsupported(); + set_pending_image(None); + set_record_merge(None); +} + +/// Run a capture hook; a failure only rules the compile out as a delta base. +pub(crate) fn guarded(hook: impl FnOnce() -> PyResult<()>) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + if let Err(error) = hook() { + capture_unsupported(&format!("an image fact could not be recorded: {error}")); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs similarity index 57% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs index 856e9f686e..c86be98316 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs @@ -1,87 +1,29 @@ //! Collect compile diagnostics so one compile reports every violation, not only the first. +use crate::compile_context::models::CollectedCompileDiagnostics; +use pyo3::types::PyCFunction; use std::sync::Mutex; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::compile_context::types::CompileDiagnosticSink; -use sqlbuild_python_base::boundary::main::context_variable::context_variable; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::ErrorId; +use crate::boundary::main::context_variable::context_variable; +use crate::records::main::construct::error_class::error_class; +use crate::records::main::fields::field::field as native_field; +use crate::records::types::ErrorId; const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the collection diagnostics are reported into, if any. -fn active_collector(py: Python<'_>) -> PyResult> { +pub(crate) fn active_collector(py: Python<'_>) -> PyResult> { context_variable(py, &ACTIVE_COLLECTOR, CONTEXT_VARIABLE_NAME) } -/// Deduplicated compile diagnostics gathered while the collection is entered. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CollectedCompileDiagnostics { - by_key: Py, - token: Mutex>>, -} - -#[pymethods] -impl CollectedCompileDiagnostics { - fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let token: Bound<'_, PyAny> = - active_collector(slf.py())?.call_method1("set", (slf.clone(),))?; - *slf.get() - .token - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token.unbind()); - Ok(slf.clone()) - } - - #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { - let token: Option> = self - .token - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); - if let Some(token) = token { - active_collector(py)?.call_method1("reset", (token,))?; - } - Ok(false) - } - - /// Record one diagnostic; a repeated key keeps the first diagnostic. - #[pyo3(signature = (*, key, diagnostic))] - fn add( - &self, - py: Python<'_>, - key: &Bound<'_, PyAny>, - diagnostic: &Bound<'_, PyAny>, - ) -> PyResult<()> { - self.by_key - .bind(py) - .call_method1("setdefault", (key, diagnostic))?; - Ok(()) - } - - /// The recorded diagnostics in deterministic key order. - #[getter] - pub(crate) fn diagnostics<'py>(&self, py: Python<'py>) -> PyResult> { - let by_key: &Bound<'py, PyDict> = self.by_key.bind(py); - let keys: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((by_key,))?; - let diagnostics: Vec> = keys - .try_iter()? - .map(|key| by_key.as_any().get_item(key?)) - .collect::>()?; - PyTuple::new(py, diagnostics) - } -} - /// A new collection; entering it collects diagnostics instead of raising the first one. #[pyfunction] pub(crate) fn collect_compile_diagnostics( @@ -151,3 +93,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(collect_compile_diagnostics, module)?)?; module.add_function(wrap_pyfunction!(report_compile_diagnostic, module)?) } + +/// `report_compile_diagnostic` as a Python function, for the callbacks user macros receive. +pub(crate) fn report_compile_diagnostic_function( + py: Python<'_>, +) -> PyResult> { + wrap_pyfunction!(report_compile_diagnostic, py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/getters.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/getters.rs similarity index 64% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/getters.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/getters.rs index edc10ea03e..46c11ec094 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/getters.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/getters.rs @@ -3,24 +3,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_context::diagnostic_collector::CollectedCompileDiagnostics; -use crate::bindings::_helpers::compile_context::input_reads::CompileInputReads; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::compile_context::_helpers::input_reads::CompileInputReads; +use crate::compile_context::models::CollectedCompileDiagnostics; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_collected_compile_diagnostics, - }, - GetterEntry { - class: class_of::, - read: read_compile_input_reads, - }, -]; - -fn read_collected_compile_diagnostics<'py>( +pub(crate) fn read_collected_compile_diagnostics<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -39,7 +25,7 @@ fn read_collected_compile_diagnostics<'py>( }) } -fn read_compile_input_reads<'py>( +pub(crate) fn read_compile_input_reads<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-base/src/compile_context/_helpers/image_codec.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/image_codec.rs new file mode 100644 index 0000000000..3dd7dc782e --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/image_codec.rs @@ -0,0 +1,22 @@ +//! Image sections as length-prefixed records over a table of each distinct string. + +use sqlbuild_cache::store::errors::StoreDecodeError; + +/// The most items a list reserves room for before reading them, against corrupt counts. +pub(crate) const MAX_RESERVED_ITEMS: usize = 1 << 16; + +/// A stored value, or `None` when the section is damaged. +pub(crate) fn read(value: Result) -> Option { + let Ok(value) = value else { + return None; + }; + Some(value) +} + +/// A stored count or index, or `None` when it does not fit this platform. +pub(crate) fn length(value: u64) -> Option { + let Ok(length) = usize::try_from(value) else { + return None; + }; + Some(length) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_facts.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/image_facts.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_facts.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/image_facts.rs index 3951b1d310..df40d84f59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_facts.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/image_facts.rs @@ -5,8 +5,9 @@ use pyo3::types::PyStringMethods; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use crate::bindings::_helpers::compile_context::delta::{capture, capture_unsupported}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::compile_context::_helpers::delta::capture; +use crate::compile_context::_helpers::delta::capture_unsupported; +use crate::records::main::fields::field::field as native_field; /// Record the digest of one model's scope usages, which decide declaration placement. pub(crate) fn record_model_usages( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs index ffe33a63f9..e06aca3b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs @@ -1,5 +1,6 @@ //! Environment, settings and run-identity reads templates make during one compile. +use crate::compile_context::models::CompileInputRecording; use std::collections::BTreeSet; use std::sync::{Mutex, MutexGuard, PoisonError}; @@ -12,12 +13,12 @@ const RUN_ID_KEY: &str = "run.id"; /// The recorders of every compile currently recording, delivered to from every thread. #[pyclass(frozen, module = "sqlbuild._native")] -struct CompileInputReadRegistry { +pub(crate) struct CompileInputReadRegistry { recorders: Mutex>>, } impl CompileInputReadRegistry { - fn locked(&self) -> MutexGuard<'_, Vec>> { + pub(crate) fn locked(&self) -> MutexGuard<'_, Vec>> { self.recorders .lock() .unwrap_or_else(PoisonError::into_inner) @@ -33,7 +34,7 @@ impl CompileInputReadRegistry { static REGISTRY: PyOnceLock> = PyOnceLock::new(); -fn registry(py: Python<'_>) -> PyResult<&Py> { +pub(crate) fn registry(py: Python<'_>) -> PyResult<&Py> { REGISTRY.get_or_try_init(py, || { Py::new( py, @@ -76,39 +77,6 @@ pub(crate) fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> P Ok(()) } -/// Records environment variables and run-identity reads while entered. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct CompileInputRecording { - recorder: Mutex>>, -} - -#[pymethods] -impl CompileInputRecording { - fn __enter__<'py>(&self, py: Python<'py>) -> PyResult> { - let recorder: Bound<'py, CompileInputReads> = Bound::new(py, CompileInputReads::new())?; - registry(py)?.get().locked().push(recorder.clone().unbind()); - *self.recorder.lock().unwrap_or_else(PoisonError::into_inner) = - Some(recorder.clone().unbind()); - Ok(recorder) - } - - #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { - if let Some(recorder) = self - .recorder - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take() - { - registry(py)? - .get() - .locked() - .retain(|item| !item.is(&recorder)); - } - Ok(false) - } -} - /// Record template input reads from every thread until the returned block exits. #[pyfunction] pub(crate) fn record_compile_input_reads() -> CompileInputRecording { @@ -118,7 +86,7 @@ pub(crate) fn record_compile_input_reads() -> CompileInputRecording { } #[derive(Default)] -struct Reads { +pub(crate) struct Reads { environment_names: BTreeSet, read_run_id: bool, settings_classes: Vec>, @@ -127,11 +95,11 @@ struct Reads { /// Environment, settings, and run-identity reads observed during one compile. #[pyclass(frozen, module = "sqlbuild._native")] pub(crate) struct CompileInputReads { - reads: Mutex, + pub(crate) reads: Mutex, } impl CompileInputReads { - fn locked(&self) -> MutexGuard<'_, Reads> { + pub(crate) fn locked(&self) -> MutexGuard<'_, Reads> { self.reads.lock().unwrap_or_else(PoisonError::into_inner) } } @@ -139,19 +107,19 @@ impl CompileInputReads { #[pymethods] impl CompileInputReads { #[new] - fn new() -> Self { + pub(crate) fn new() -> Self { Self { reads: Mutex::new(Reads::default()), } } /// Record one environment variable a template looked up, present or not. - fn add_environment_name(&self, name: String) { + pub(crate) fn add_environment_name(&self, name: String) { self.locked().environment_names.insert(name); } /// Record one settings class whose values come from the environment or files. - fn add_settings_class(&self, settings_class: &Bound<'_, PyAny>) { + pub(crate) fn add_settings_class(&self, settings_class: &Bound<'_, PyAny>) { let mut reads: MutexGuard<'_, Reads> = self.locked(); if !reads .settings_classes @@ -163,7 +131,7 @@ impl CompileInputReads { } /// Record that a template read the per-invocation run identity. - fn mark_run_id_read(&self) { + pub(crate) fn mark_run_id_read(&self) { self.locked().read_run_id = true; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/mod.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/mod.rs similarity index 64% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/mod.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/mod.rs index 41e63c7466..5d7ebdc6c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/mod.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/mod.rs @@ -1,6 +1,3 @@ -//! Per-compile state below the compile driver: collectors and the delta-compile slots. - -pub(crate) mod compile_timings; pub(crate) mod coverage; pub(crate) mod delta; pub(crate) mod diagnostic_collector; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/yaml_cache.rs b/crates/sqlbuild-python-base/src/compile_context/_helpers/yaml_cache.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_context/yaml_cache.rs rename to crates/sqlbuild-python-base/src/compile_context/_helpers/yaml_cache.rs index ac531495a0..acdbed1fce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/yaml_cache.rs +++ b/crates/sqlbuild-python-base/src/compile_context/_helpers/yaml_cache.rs @@ -1,5 +1,6 @@ //! Loaded YAML documents a delta compile keeps by content digest instead of parsing again. +use crate::compile_context::models::DocumentStore; use std::path::PathBuf; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; @@ -9,8 +10,9 @@ use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValu use sqlbuild_core::text::main::decode_text::decode_text; use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; -use crate::bindings::_helpers::compile_context::delta::injection; -use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; +use crate::compile_context::_helpers::delta::injection; +use crate::compile_context::models::Decoder; +use crate::compile_context::models::Encoder; /// The source YAML key that lists sources, and the key naming each source. const SOURCES_KEY: &str = "sources"; @@ -198,62 +200,12 @@ pub(crate) fn decoded_document(stored: &[u8]) -> Option { Some(document) } -/// The store a delta compile reads and fills, opened beside its reuse slots. -pub(crate) struct DocumentStore { - pub(crate) store: NativeStore, - pub(crate) path: PathBuf, -} - -impl DocumentStore { - /// Each file's document, decoded from the store or loaded by `load` and stored. - pub(crate) fn load_with( - &mut self, - paths: &[String], - absolute: impl Fn(&str) -> PathBuf, - load: impl FnOnce(&[String]) -> Result, E>, - ) -> Result, E> { - let mut outcomes: Vec> = Vec::with_capacity(paths.len()); - let mut missed: Vec = Vec::new(); - for path in paths { - let stored: Option = stored_outcome(&mut self.store, &absolute(path)); - if stored.is_none() { - missed.push(path.clone()); - } - outcomes.push(stored); - } - let mut loaded = if missed.is_empty() { - Vec::new() - } else { - load(&missed)? - } - .into_iter(); - let mut complete: Vec = Vec::with_capacity(paths.len()); - for outcome in outcomes { - let outcome: YamlFileOutcome = match outcome { - Some(outcome) => outcome, - None => { - let Some(outcome) = loaded.next() else { - break; - }; - if let YamlFileOutcome::Loaded { contents, value } = &outcome { - self.store - .put(document_key(contents), encoded_document(value)); - } - outcome - } - }; - complete.push(outcome); - } - Ok(complete) - } -} - /// In a delta compile, the document store beside its reuse slots, when it opens. pub(crate) fn open_documents(py: pyo3::Python<'_>) -> Option { - let path: PathBuf = crate::bindings::_helpers::compile_context::delta::injection()? + let path: PathBuf = crate::compile_context::_helpers::delta::injection()? .yaml_store .clone()?; - match crate::bindings::_helpers::cache::native_store::open_store( + match crate::cache::main::open_store::open_store( py, &path, YAML_STORE_KIND, @@ -265,7 +217,10 @@ pub(crate) fn open_documents(py: pyo3::Python<'_>) -> Option { } /// The stored document of the file at `path` when its current text is stored. -fn stored_outcome(store: &mut NativeStore, path: &std::path::Path) -> Option { +pub(crate) fn stored_outcome( + store: &mut NativeStore, + path: &std::path::Path, +) -> Option { let bytes: Vec = match std::fs::read(path) { Ok(bytes) => bytes, Err(_unreadable) => return None, @@ -280,25 +235,20 @@ fn stored_outcome(store: &mut NativeStore, path: &std::path::Path) -> Option, documents: &DocumentStore) { - let _ = crate::bindings::_helpers::cache::native_store::save_store( - py, - &documents.store, - &documents.path, - &[], - ); + let _ = crate::cache::main::save_store::save_store(py, &documents.store, &documents.path, &[]); } /// In a delta reading only some sources, drop every other loaded source YAML entry. pub(crate) fn retain_read_sources( - outcomes: &mut [sqlbuild_discovery::yaml_files::models::YamlFileOutcome], -) { + mut outcomes: Vec, +) -> Vec { use sqlbuild_config::models::ConfigValue; use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; let Some(injection) = injection() else { - return; + return outcomes; }; let Some(read) = injection.source_names.as_ref() else { - return; + return outcomes; }; fn text(value: &ConfigValue) -> Option<&str> { match value { @@ -306,7 +256,7 @@ pub(crate) fn retain_read_sources( _ => None, } } - for outcome in outcomes { + for outcome in &mut outcomes { let YamlFileOutcome::Loaded { value: ConfigValue::Map(entries), .. @@ -332,4 +282,5 @@ pub(crate) fn retain_read_sources( }); } } + outcomes } diff --git a/crates/sqlbuild-python-base/src/compile_context/constants.rs b/crates/sqlbuild-python-base/src/compile_context/constants.rs new file mode 100644 index 0000000000..668cf6becc --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/constants.rs @@ -0,0 +1,23 @@ +//! Public constants of compile context. + +use crate::compile_context::_helpers::getters::read_collected_compile_diagnostics; +use crate::compile_context::_helpers::getters::read_compile_input_reads; +use crate::compile_context::_helpers::input_reads::CompileInputReads; +use crate::compile_context::models::CollectedCompileDiagnostics; +use crate::records::main::registry::class_of::class_of; +use crate::records::models::GetterEntry; + +/// Whether edits that report or raise errors compile as deltas; until then they fall back. +pub const ERRORS_ON_DELTA_PATH: bool = true; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_collected_compile_diagnostics, + }, + GetterEntry { + class: class_of::, + read: read_compile_input_reads, + }, +]; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/coverage/coverage_reasons.rs b/crates/sqlbuild-python-base/src/compile_context/main/coverage/coverage_reasons.rs new file mode 100644 index 0000000000..53d66e5d1c --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/coverage/coverage_reasons.rs @@ -0,0 +1,13 @@ +//! Each model's partial-check reasons, in `order`, which pairs each model with its upstream. + +use crate::compile_context::_helpers::coverage; +use crate::compile_context::models::ModelCoverage; + +/// Each model's partial-check reasons, in `order`, which pairs each model with its upstream. +#[inline] +pub fn coverage_reasons( + order: &[(String, Vec)], + facts: &[ModelCoverage], +) -> Vec<(String, Vec)> { + coverage::coverage_reasons(order, facts) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_coverage.rs b/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_coverage.rs new file mode 100644 index 0000000000..b436953789 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_coverage.rs @@ -0,0 +1,10 @@ +//! The whole project's notable facts: `stored` with the fresh models' replaced by `fresh`. + +use crate::compile_context::_helpers::coverage; +use crate::compile_context::models::ModelCoverage; + +/// The whole project's notable facts: `stored` with the fresh models' replaced by `fresh`. +#[inline] +pub fn merged_coverage(stored: &[ModelCoverage], fresh: &[ModelCoverage]) -> Vec { + coverage::merged_coverage(stored, fresh) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_order.rs b/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_order.rs new file mode 100644 index 0000000000..dd5d760eb4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/coverage/merged_order.rs @@ -0,0 +1,12 @@ +//! Every project model with its upstream names, the fresh models' replacing the stored. + +use crate::compile_context::_helpers::coverage; + +/// Every project model with its upstream names, the fresh models' replacing the stored. +#[inline] +pub fn merged_order( + stored: &[(String, Vec)], + fresh: &[(String, Vec)], +) -> Vec<(String, Vec)> { + coverage::merged_order(stored, fresh) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/coverage/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/coverage/mod.rs new file mode 100644 index 0000000000..19b69ce611 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/coverage/mod.rs @@ -0,0 +1,3 @@ +pub mod coverage_reasons; +pub mod merged_coverage; +pub mod merged_order; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/capture.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/capture.rs new file mode 100644 index 0000000000..c9c7fa1ac5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/capture.rs @@ -0,0 +1,10 @@ +//! Record into the capture, when one is running. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::ImageCapture; + +/// Record into the capture, when one is running. +#[inline] +pub fn capture(record: impl FnOnce(&mut ImageCapture)) { + delta::capture(record) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/capturing.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/capturing.rs new file mode 100644 index 0000000000..05dd359708 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/capturing.rs @@ -0,0 +1,9 @@ +//! Whether the running compile captures image facts. + +use crate::compile_context::_helpers::delta; + +/// Whether the running compile captures image facts. +#[inline] +pub fn capturing() -> bool { + delta::capturing() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/delta_unsupported.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/delta_unsupported.rs new file mode 100644 index 0000000000..28c8487aec --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/delta_unsupported.rs @@ -0,0 +1,9 @@ +//! Rule out the running delta compile: returns the error that unwinds to its fallback. + +use crate::compile_context::_helpers::delta; + +/// Rule out the running delta compile: returns the error that unwinds to its fallback. +#[inline] +pub fn delta_unsupported(reason: &str) -> pyo3::PyErr { + delta::delta_unsupported(reason) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/guarded.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/guarded.rs new file mode 100644 index 0000000000..5ed0c74edb --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/guarded.rs @@ -0,0 +1,10 @@ +//! Run a capture hook; a failure only rules the compile out as a delta base. + +use crate::compile_context::_helpers::delta; +use pyo3::prelude::PyResult; + +/// Run a capture hook; a failure only rules the compile out as a delta base. +#[inline] +pub fn guarded(hook: impl FnOnce() -> PyResult<()>) -> PyResult<()> { + delta::guarded(hook) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/inject_relations.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/inject_relations.rs new file mode 100644 index 0000000000..88be346388 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/inject_relations.rs @@ -0,0 +1,10 @@ +//! Give a delta's analysis session the whole project's relation facts and analysis order. + +use crate::compile_context::_helpers::delta; +use sqlbuild_analysis::assembly::analysis_session::models::SessionRequest; + +/// Give a delta's analysis session the whole project's relation facts and analysis order. +#[inline] +pub fn inject_relations(request: SessionRequest) -> pyo3::PyResult { + delta::inject_relations(request) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/injected_fact_digest.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/injected_fact_digest.rs new file mode 100644 index 0000000000..30db6566ff --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/injected_fact_digest.rs @@ -0,0 +1,9 @@ +//! The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. + +use crate::compile_context::_helpers::delta; + +/// The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. +#[inline] +pub fn injected_fact_digest(key: &[String]) -> Option { + delta::injected_fact_digest(key) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/injection.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/injection.rs new file mode 100644 index 0000000000..df445928e4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/injection.rs @@ -0,0 +1,11 @@ +//! The running delta compile's injection, if any. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::DeltaInjection; +use std::sync::Arc; + +/// The running delta compile's injection, if any. +#[inline] +pub fn injection() -> Option> { + delta::injection() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/mod.rs new file mode 100644 index 0000000000..0840751148 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/mod.rs @@ -0,0 +1,20 @@ +pub mod capture; +pub mod capturing; +pub mod delta_unsupported; +pub mod guarded; +pub mod inject_relations; +pub mod injected_fact_digest; +pub mod injection; +pub mod record_source_reads; +pub mod record_test_source_reads; +pub mod reset; +pub mod set_pending_image; +pub mod set_record_merge; +pub mod start_capture; +pub mod stop_capture; +pub mod stop_on_errors; +pub mod take_capture; +pub mod take_delta_unsupported; +pub mod take_pending_image; +pub mod take_record_merge; +pub mod with_injection; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/record_source_reads.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/record_source_reads.rs new file mode 100644 index 0000000000..bbd857aa4c --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/record_source_reads.rs @@ -0,0 +1,9 @@ +//! Record sources a resource other than a model reads, when a capture is running. + +use crate::compile_context::_helpers::delta; + +/// Record sources a resource other than a model reads, when a capture is running. +#[inline] +pub fn record_source_reads<'a>(names: impl IntoIterator) { + delta::record_source_reads(names) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/record_test_source_reads.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/record_test_source_reads.rs new file mode 100644 index 0000000000..8ce724093b --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/record_test_source_reads.rs @@ -0,0 +1,9 @@ +//! Record sources an SQL test file reads, when a capture is running. + +use crate::compile_context::_helpers::delta; + +/// Record sources an SQL test file reads, when a capture is running. +#[inline] +pub fn record_test_source_reads<'a>(relative_path: &str, names: impl IntoIterator) { + delta::record_test_source_reads(relative_path, names) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/reset.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/reset.rs new file mode 100644 index 0000000000..b396fbcc66 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/reset.rs @@ -0,0 +1,9 @@ +//! Forget every capture, injection and pending record fact, after a compile ends either way. + +use crate::compile_context::_helpers::delta; + +/// Forget every capture, injection and pending record fact, after a compile ends either way. +#[inline] +pub fn reset() { + delta::reset() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/set_pending_image.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/set_pending_image.rs new file mode 100644 index 0000000000..a609cd0884 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/set_pending_image.rs @@ -0,0 +1,10 @@ +//! Hand the next reuse record the image to keep beside it. + +use crate::compile_context::_helpers::delta; +use sqlbuild_cache::project_reuse::types::ImageSections; + +/// Hand the next reuse record the image to keep beside it. +#[inline] +pub fn set_pending_image(image: Option) { + delta::set_pending_image(image) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/set_record_merge.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/set_record_merge.rs new file mode 100644 index 0000000000..089ac1d03b --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/set_record_merge.rs @@ -0,0 +1,10 @@ +//! Hand the next reuse record what it keeps of the patched compile. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::RecordMerge; + +/// Hand the next reuse record what it keeps of the patched compile. +#[inline] +pub fn set_record_merge(merge: Option) { + delta::set_record_merge(merge) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/start_capture.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/start_capture.rs new file mode 100644 index 0000000000..d7ce8e03a8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/start_capture.rs @@ -0,0 +1,9 @@ +//! Start capturing image facts for the compile about to run. + +use crate::compile_context::_helpers::delta; + +/// Start capturing image facts for the compile about to run. +#[inline] +pub fn start_capture() { + delta::start_capture() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_capture.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_capture.rs new file mode 100644 index 0000000000..cf3905e14a --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_capture.rs @@ -0,0 +1,9 @@ +//! Stop capturing without keeping anything, for a project that turns delta compiles off. + +use crate::compile_context::_helpers::delta; + +/// Stop capturing without keeping anything, for a project that turns delta compiles off. +#[inline] +pub fn stop_capture() { + delta::stop_capture() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_on_errors.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_on_errors.rs new file mode 100644 index 0000000000..a87f6deb66 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/stop_on_errors.rs @@ -0,0 +1,9 @@ +//! In a delta compile that leaves errors to full compiles, stop once `failed` reports one. + +use crate::compile_context::_helpers::delta; + +/// In a delta compile that leaves errors to full compiles, stop once `failed` reports one. +#[inline] +pub fn stop_on_errors(failed: bool) -> pyo3::PyResult<()> { + delta::stop_on_errors(failed) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/take_capture.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_capture.rs new file mode 100644 index 0000000000..e9f8978216 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_capture.rs @@ -0,0 +1,10 @@ +//! Stop capturing and take what was captured. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::ImageCapture; + +/// Stop capturing and take what was captured. +#[inline] +pub fn take_capture() -> Option { + delta::take_capture() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/take_delta_unsupported.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_delta_unsupported.rs new file mode 100644 index 0000000000..50e160adf4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_delta_unsupported.rs @@ -0,0 +1,9 @@ +//! Take the reason the delta compile was ruled out, if any. + +use crate::compile_context::_helpers::delta; + +/// Take the reason the delta compile was ruled out, if any. +#[inline] +pub fn take_delta_unsupported() -> Option { + delta::take_delta_unsupported() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/take_pending_image.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_pending_image.rs new file mode 100644 index 0000000000..ac405539ff --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_pending_image.rs @@ -0,0 +1,10 @@ +//! The image the next reuse record keeps, taken. + +use crate::compile_context::_helpers::delta; +use sqlbuild_cache::project_reuse::types::ImageSections; + +/// The image the next reuse record keeps, taken. +#[inline] +pub fn take_pending_image() -> Option { + delta::take_pending_image() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/take_record_merge.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_record_merge.rs new file mode 100644 index 0000000000..92705f98f3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/take_record_merge.rs @@ -0,0 +1,10 @@ +//! What the next reuse record keeps of the patched compile, taken. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::RecordMerge; + +/// What the next reuse record keeps of the patched compile, taken. +#[inline] +pub fn take_record_merge() -> Option { + delta::take_record_merge() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/delta/with_injection.rs b/crates/sqlbuild-python-base/src/compile_context/main/delta/with_injection.rs new file mode 100644 index 0000000000..64596766df --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/delta/with_injection.rs @@ -0,0 +1,13 @@ +//! Run `body` with `injection` visible to the rerun stages; returns it unless still held. + +use crate::compile_context::_helpers::delta; +use crate::compile_context::models::DeltaInjection; + +/// Run `body` with `injection` visible to the rerun stages; returns it unless still held. +#[inline] +pub fn with_injection( + injection: DeltaInjection, + body: impl FnOnce() -> T, +) -> (T, Option) { + delta::with_injection(injection, body) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/collect_compile_diagnostics.rs b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/collect_compile_diagnostics.rs new file mode 100644 index 0000000000..01b8d2240f --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/collect_compile_diagnostics.rs @@ -0,0 +1,15 @@ +//! A new collection; entering it collects diagnostics instead of raising the first one. + +use crate::compile_context::_helpers::diagnostic_collector; +use crate::compile_context::models::CollectedCompileDiagnostics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// A new collection; entering it collects diagnostics instead of raising the first one. +#[inline] +pub fn collect_compile_diagnostics( + py: Python<'_>, +) -> PyResult> { + diagnostic_collector::collect_compile_diagnostics(py) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/mod.rs new file mode 100644 index 0000000000..d943733df5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/mod.rs @@ -0,0 +1,4 @@ +pub mod collect_compile_diagnostics; +pub mod register_diagnostic_collector; +pub mod report_compile_diagnostic; +pub mod report_compile_diagnostic_function; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/register_diagnostic_collector.rs b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/register_diagnostic_collector.rs new file mode 100644 index 0000000000..fbeb44ecc4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/register_diagnostic_collector.rs @@ -0,0 +1,12 @@ +//! Register the diagnostic collector functions and classes. + +use crate::compile_context::_helpers::diagnostic_collector; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the diagnostic collector functions and classes. +#[inline] +pub fn register_diagnostic_collector(module: &Bound<'_, PyModule>) -> PyResult<()> { + diagnostic_collector::register(module) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic.rs b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic.rs new file mode 100644 index 0000000000..069c03fe5b --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic.rs @@ -0,0 +1,15 @@ +//! Record a diagnostic in the active collection, or raise it when none is active. + +use crate::compile_context::_helpers::diagnostic_collector; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Record a diagnostic in the active collection, or raise it when none is active. +#[inline] +pub fn report_compile_diagnostic( + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, +) -> PyResult<()> { + diagnostic_collector::report_compile_diagnostic(key, diagnostic) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic_function.rs b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic_function.rs new file mode 100644 index 0000000000..04064a6ea3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/diagnostic_collector/report_compile_diagnostic_function.rs @@ -0,0 +1,13 @@ +//! `report_compile_diagnostic` as a Python function, for the callbacks user macros receive. + +use crate::compile_context::_helpers::diagnostic_collector; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyCFunction; + +/// `report_compile_diagnostic` as a Python function, for the callbacks user macros receive. +#[inline] +pub fn report_compile_diagnostic_function(py: Python<'_>) -> PyResult> { + diagnostic_collector::report_compile_diagnostic_function(py) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/image_facts/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/image_facts/mod.rs new file mode 100644 index 0000000000..af7d5c54de --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/image_facts/mod.rs @@ -0,0 +1 @@ +pub mod record_model_usages; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/image_facts/record_model_usages.rs b/crates/sqlbuild-python-base/src/compile_context/main/image_facts/record_model_usages.rs new file mode 100644 index 0000000000..32c2bd71d7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/image_facts/record_model_usages.rs @@ -0,0 +1,15 @@ +//! Record the digest of one model's scope usages, which decide declaration placement. + +use crate::compile_context::_helpers::image_facts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Record the digest of one model's scope usages, which decide declaration placement. +#[inline] +pub fn record_model_usages( + model: &Bound<'_, PyAny>, + usages: (&[Bound<'_, PyAny>], &[Bound<'_, PyAny>]), +) -> PyResult<()> { + image_facts::record_model_usages(model, usages) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/context_read.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/context_read.rs new file mode 100644 index 0000000000..571c8135ec --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/context_read.rs @@ -0,0 +1,11 @@ +//! Note a CTX lookup; only the per-invocation run identity is volatile. + +use crate::compile_context::_helpers::input_reads; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Note a CTX lookup; only the per-invocation run identity is volatile. +#[inline] +pub fn context_read(py: Python<'_>, name: &str) -> PyResult<()> { + input_reads::context_read(py, name) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/environment_read.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/environment_read.rs new file mode 100644 index 0000000000..56100380a9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/environment_read.rs @@ -0,0 +1,11 @@ +//! Note that a template looked up one environment variable, present or not. + +use crate::compile_context::_helpers::input_reads; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Note that a template looked up one environment variable, present or not. +#[inline] +pub fn environment_read(py: Python<'_>, name: &str) -> PyResult<()> { + input_reads::environment_read(py, name) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/mod.rs new file mode 100644 index 0000000000..d355175495 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/mod.rs @@ -0,0 +1,5 @@ +pub mod context_read; +pub mod environment_read; +pub mod record_compile_input_reads; +pub mod record_settings_class_read; +pub mod register_input_reads; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_compile_input_reads.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_compile_input_reads.rs new file mode 100644 index 0000000000..062e94d7e6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_compile_input_reads.rs @@ -0,0 +1,10 @@ +//! Record template input reads from every thread until the returned block exits. + +use crate::compile_context::_helpers::input_reads; +use crate::compile_context::models::CompileInputRecording; + +/// Record template input reads from every thread until the returned block exits. +#[inline] +pub fn record_compile_input_reads() -> CompileInputRecording { + input_reads::record_compile_input_reads() +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_settings_class_read.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_settings_class_read.rs new file mode 100644 index 0000000000..bb389dfb0e --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/record_settings_class_read.rs @@ -0,0 +1,12 @@ +//! Note a provider settings class instantiated from the environment and files. + +use crate::compile_context::_helpers::input_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Note a provider settings class instantiated from the environment and files. +#[inline] +pub fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { + input_reads::record_settings_class_read(settings_class) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/input_reads/register_input_reads.rs b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/register_input_reads.rs new file mode 100644 index 0000000000..c16af31811 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/input_reads/register_input_reads.rs @@ -0,0 +1,12 @@ +//! Register the input reads functions and classes. + +use crate::compile_context::_helpers::input_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the input reads functions and classes. +#[inline] +pub fn register_input_reads(module: &Bound<'_, PyModule>) -> PyResult<()> { + input_reads::register(module) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/mod.rs new file mode 100644 index 0000000000..147e2dbcbc --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/mod.rs @@ -0,0 +1,6 @@ +pub mod coverage; +pub mod delta; +pub mod diagnostic_collector; +pub mod image_facts; +pub mod input_reads; +pub mod yaml_cache; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/mod.rs b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/mod.rs new file mode 100644 index 0000000000..0c0bcc60e9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/mod.rs @@ -0,0 +1,4 @@ +pub mod open_documents; +pub mod retain_read_sources; +pub mod save_documents; +pub mod yaml_store_path; diff --git a/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/open_documents.rs b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/open_documents.rs new file mode 100644 index 0000000000..0322f7b516 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/open_documents.rs @@ -0,0 +1,10 @@ +//! In a delta compile, the document store beside its reuse slots, when it opens. + +use crate::compile_context::_helpers::yaml_cache; +use crate::compile_context::models::DocumentStore; + +/// In a delta compile, the document store beside its reuse slots, when it opens. +#[inline] +pub fn open_documents(py: pyo3::Python<'_>) -> Option { + yaml_cache::open_documents(py) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/retain_read_sources.rs b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/retain_read_sources.rs new file mode 100644 index 0000000000..3b1bf54145 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/retain_read_sources.rs @@ -0,0 +1,11 @@ +//! In a delta reading only some sources, drop every other loaded source YAML entry. + +use crate::compile_context::_helpers::yaml_cache; + +/// In a delta reading only some sources, drop every other loaded source YAML entry. +#[inline] +pub fn retain_read_sources( + outcomes: Vec, +) -> Vec { + yaml_cache::retain_read_sources(outcomes) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/save_documents.rs b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/save_documents.rs new file mode 100644 index 0000000000..5f312549d8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/save_documents.rs @@ -0,0 +1,10 @@ +//! Save the documents this compile loaded; failing to only means parsing them again. + +use crate::compile_context::_helpers::yaml_cache; +use crate::compile_context::models::DocumentStore; + +/// Save the documents this compile loaded; failing to only means parsing them again. +#[inline] +pub fn save_documents(py: pyo3::Python<'_>, documents: &DocumentStore) { + yaml_cache::save_documents(py, documents) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/yaml_store_path.rs b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/yaml_store_path.rs new file mode 100644 index 0000000000..d25ac40dbe --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/main/yaml_cache/yaml_store_path.rs @@ -0,0 +1,10 @@ +//! Where a delta compile keeps loaded YAML documents. + +use crate::compile_context::_helpers::yaml_cache; +use std::path::PathBuf; + +/// Where a delta compile keeps loaded YAML documents. +#[inline] +pub fn yaml_store_path(cache_dir: &std::path::Path) -> PathBuf { + yaml_cache::yaml_store_path(cache_dir) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/mod.rs b/crates/sqlbuild-python-base/src/compile_context/mod.rs new file mode 100644 index 0000000000..44a90962a7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/mod.rs @@ -0,0 +1,10 @@ +//! Per-compile state below the compile driver: collectors, input reads and the delta-compile slots. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-python-base/src/compile_context/models.rs b/crates/sqlbuild-python-base/src/compile_context/models.rs new file mode 100644 index 0000000000..f62143f901 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/models.rs @@ -0,0 +1,549 @@ +//! Public records of compile context. + +use crate::boundary::main::imported::imported; +use crate::compile_context::_helpers::diagnostic_collector::active_collector; +use crate::compile_context::_helpers::image_codec::MAX_RESERVED_ITEMS; +use crate::compile_context::_helpers::image_codec::length; +use crate::compile_context::_helpers::image_codec::read; +use crate::compile_context::_helpers::input_reads::CompileInputReads; +use crate::compile_context::_helpers::input_reads::registry; +use crate::compile_context::_helpers::yaml_cache::document_key; +use crate::compile_context::_helpers::yaml_cache::encoded_document; +use crate::compile_context::_helpers::yaml_cache::stored_outcome; +use crate::compile_context::types::ModelShape; +use pyo3::PyErr; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; +use rustc_hash::FxHashMap; +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_cache::store::models::RecordReader; +use sqlbuild_cache::store::models::RecordWriter; +use sqlbuild_discovery::header_changes::types::HeaderKeys; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; +use sqlbuild_host::compile_context::types::ProcessClock; +use sqlbuild_rules::engine::main::retained_request::retained_request; +use sqlbuild_rules::models::EvaluateRequest; +use sqlbuild_rules::models::RetainedRows; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::path::PathBuf; +use std::sync::Mutex; +use std::sync::PoisonError; + +/// The process CPU clock of one interpreter, including its worker threads. +#[derive(Clone, Copy)] +pub struct PythonProcessClock<'py>(pub Python<'py>); + +impl ProcessClock for PythonProcessClock<'_> { + type Error = PyErr; + + fn process_time_ns(&self) -> PyResult { + imported(self.0, "time", "process_time_ns")? + .call0()? + .extract() + } +} + +/// What one model contributes to the partial-check reasons, apart from its dependencies. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ModelCoverage { + pub name: String, + /// The model opts out of SQL analysis. + pub disabled: bool, + /// The model has a binding error, which makes every model reading it partial. + pub blocked: bool, + /// Its own reasons, sorted; those from invalid dependencies are derived. + pub partial: Vec, +} + +impl ModelCoverage { + /// Whether the model adds anything to the reasons. + pub fn notable(&self) -> bool { + self.disabled || self.blocked || !self.partial.is_empty() + } +} + +/// A stored compile's built-in rules request, as decoded for a delta compile. +#[derive(Clone, Debug)] +pub struct BaseRules { + pub request: EvaluateRequest, + /// Each model's payload digest, which keys its cached findings, in model order. + pub digests: Vec, +} + +impl BaseRules { + /// The rows a compile retained, decoded. + pub fn decoded(retained: &RetainedRows) -> Option { + Some(Self { + request: retained_request(retained)?, + digests: retained.digests.clone(), + }) + } +} + +/// A stored rules request, decoding on its own thread or decoded already. +#[derive(Debug)] +pub enum PendingRules { + Decoding(std::thread::JoinHandle>), + Ready(Box), +} + +impl PendingRules { + /// The decoded request, or `None` when it could not be decoded. + pub fn resolved(self) -> Option { + match self { + Self::Decoding(handle) => { + let Ok(rules) = handle.join() else { + return None; + }; + rules + } + Self::Ready(rules) => Some(*rules), + } + } +} + +/// The facts a full compile leaves for its project image beyond its analysis record. +#[derive(Debug, Default)] +pub struct ImageCapture { + /// The sources functions and audits read, which a delta compile always keeps. + pub pinned_sources: BTreeSet, + /// The sources each SQL test file reads, which a delta compile keeps when it reruns it. + pub test_sources: BTreeMap>, + /// The relation facts the model analysis session ended with. + pub relations: Option, + /// The built-in rules request and its raw findings, before suppressions. + pub rules: Option, + /// Each custom-rule fact digest the custom rule cache compared, by fact key. + pub fact_digests: BTreeMap, String>, + /// Whether the JSON report fell back to the standard library's serializer. + pub stdlib_json: bool, + /// Each model's scope usage digest, by model file path. + pub usages: BTreeMap, + /// Each SQL test's compiled-artifact chain, by test key text. + pub test_chains: Vec, + /// Each project model's `(path, name, materialized)`, when custom rules ran. + pub model_subjects: Option)>>, + /// Each compiled model's `(name, relative path, upstream (kind, name) pairs)`. + pub models: Vec, + /// Each report diagnostic as its JSON item text, with what it names. + pub diagnostics: Vec, + /// The text report's model lines `(name, failed, column count)`, when it rendered one. + pub text_models: Option>, + /// The template environment names the compile read. + pub environment_names: Vec, + /// Each discovered model's header keys, by model file path. + pub header_keys: BTreeMap, + /// The audit factories each model file's header attaches, for files attaching any. + pub audit_references: BTreeMap>, + /// Each model file's SQL lint cache key, in project model order. + pub lint_model_keys: Option>, + /// Each other SQL file's lint cache key, in walk order. + pub lint_file_keys: Option)>>, + /// The closed shape each model published for semantic checks, in publication order. + pub model_shapes: Option>, + /// The notable semantic coverage facts of every model, when semantic checks reported them. + pub coverage: Option>, + /// Why this compile cannot be the base of a delta compile, when something ruled it out. + pub unsupported: Option, +} + +/// The built-in rules request a compile evaluated and its raw findings. +#[derive(Debug)] +pub struct CapturedRules { + pub retained: RetainedRows, +} + +/// One compiled model's identity and references. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct CapturedModel { + pub name: String, + pub relative_path: String, + pub upstream: Vec<(String, String)>, +} + +/// One report diagnostic: its JSON text and the resource and file it names. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct CapturedDiagnostic { + pub json: String, + pub resource_name: Option, + pub paths: Vec, + /// Whether the diagnostic is an error. + #[serde(default)] + pub error: bool, +} + +/// One SQL test's file and the models its compiled artifact is keyed by. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CapturedTest { + pub key: String, + pub relative_path: String, + pub chain: Vec, +} + +/// What a delta compile hands the stages it reruns for the changed models. +#[derive(Debug, Default)] +pub struct DeltaInjection { + /// Model files to discover, as project-relative POSIX paths. + pub model_paths: BTreeSet, + /// SQL test files to discover, as project-relative POSIX paths. + pub test_paths: BTreeSet, + /// The paths and names of every model the delta compile reruns. + pub rerun_names: BTreeSet, + /// The project directory the stored compile and the delta compile both read. + pub project_dir: std::path::PathBuf, + /// Every model and SQL test file of the whole project, as project-relative POSIX paths. + pub source_paths: BTreeSet, + /// Every model file of the whole project, which path defaults are matched against. + pub project_model_paths: Vec, + /// Every model name of the whole project, which references resolve against. + pub model_names: Vec, + /// The whole project's relation facts, injected for relations the delta does not analyse. + pub relations: RelationTables, + /// The whole project's built-in rules request, taken by the round that evaluates it. + pub rules: Mutex>, + /// The whole project's custom-rule fact digests. + pub fact_digests: BTreeMap, String>, + /// Each project model's `(path, name, materialized)`, the custom-rule subjects. + pub model_subjects: Option)>>, + /// The files that changed since the stored compile, as project-relative POSIX paths. + pub changed_paths: BTreeSet, + /// Where loaded YAML documents are kept for later delta compiles. + pub yaml_store: Option, + /// The only sources discovery keeps, those the rerun models read; `None` keeps every one. + pub source_names: Option>, + /// The whole project's SQL lint cache keys, as the image recorded them. + pub lint_model_keys: Option>, + pub lint_file_keys: Option)>>, + /// The closed shapes the whole project's models published. + pub model_shapes: Vec, + /// The whole project's text report model lines, with the rerun models' lines to replace. + pub text_models: Vec<(String, bool, i64)>, + /// The whole project's notable semantic coverage facts, as the image recorded them. + pub coverage: Option>, + /// Every project model in project order, with the models it reads. + pub coverage_order: Vec<(String, Vec)>, + /// The audit factories each model file's header attaches, for files attaching any. + pub audit_references: BTreeMap>, +} + +/// What a delta compile's reuse record keeps of the stored compile it patched. +#[derive(Debug, Default)] +pub struct RecordMerge { + /// Each stored artifact's stamp, verified by the reuse check, kept unless the delta wrote it. + pub kept: Vec, + /// The template environment names the stored compile read. + pub environment_names: Vec, + /// The module files the stored compile depended on, such as those behind stored macro calls. + pub module_paths: Vec, +} + +/// Deduplicated compile diagnostics gathered while the collection is entered. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct CollectedCompileDiagnostics { + pub(crate) by_key: Py, + pub(crate) token: Mutex>>, +} + +#[pymethods] +impl CollectedCompileDiagnostics { + pub(crate) fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let token: Bound<'_, PyAny> = + active_collector(slf.py())?.call_method1("set", (slf.clone(),))?; + *slf.get() + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token.unbind()); + Ok(slf.clone()) + } + + #[pyo3(signature = (*_exception))] + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _exception: &Bound<'_, PyTuple>, + ) -> PyResult { + let token: Option> = self + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(token) = token { + active_collector(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } + + /// Record one diagnostic; a repeated key keeps the first diagnostic. + #[pyo3(signature = (*, key, diagnostic))] + pub(crate) fn add( + &self, + py: Python<'_>, + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.by_key + .bind(py) + .call_method1("setdefault", (key, diagnostic))?; + Ok(()) + } + + /// The recorded diagnostics in deterministic key order. + #[getter] + pub fn diagnostics<'py>(&self, py: Python<'py>) -> PyResult> { + let by_key: &Bound<'py, PyDict> = self.by_key.bind(py); + let keys: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((by_key,))?; + let diagnostics: Vec> = keys + .try_iter()? + .map(|key| by_key.as_any().get_item(key?)) + .collect::>()?; + PyTuple::new(py, diagnostics) + } +} + +/// Writes one section: its values refer to a table of the distinct strings they hold. +#[derive(Debug, Default)] +pub struct Encoder<'a> { + pub(crate) ids: FxHashMap<&'a str, u64>, + pub(crate) strings: Vec<&'a str>, + pub(crate) body: RecordWriter, +} + +impl<'a> Encoder<'a> { + pub fn string(&mut self, value: &'a str) { + let next: u64 = self.strings.len() as u64; + let id: u64 = *self.ids.entry(value).or_insert(next); + if id == next { + self.strings.push(value); + } + self.body.put_u64(id); + } + + pub fn optional_string(&mut self, value: Option<&'a str>) { + self.flag(value.is_some()); + if let Some(value) = value { + self.string(value); + } + } + + pub fn count(&mut self, count: usize) { + self.body.put_u64(count as u64); + } + + pub fn flag(&mut self, value: bool) { + self.body.put_u64(u64::from(value)); + } + + pub fn bytes(&mut self, value: &[u8]) { + self.body.put_bytes(value); + } + + pub fn strings(&mut self, values: impl ExactSizeIterator) { + self.count(values.len()); + for value in values { + self.string(value); + } + } + + pub fn pairs(&mut self, pairs: &'a [(String, String)]) { + self.count(pairs.len()); + for (left, right) in pairs { + self.string(left); + self.string(right); + } + } + + /// The section: the string table, then the values. + pub fn finish(self) -> Vec { + let mut section: RecordWriter = RecordWriter::default(); + section.put_u64(self.strings.len() as u64); + for value in &self.strings { + section.put_str(value); + } + section.put_bytes(&self.body.into_bytes()); + section.into_bytes() + } +} + +/// Reads a section an [`Encoder`] wrote; any malformed value fails the whole read. +#[derive(Debug)] +pub struct Decoder<'a> { + pub(crate) strings: Vec, + pub(crate) body: RecordReader<'a>, +} + +impl<'a> Decoder<'a> { + pub fn new(section: &'a [u8]) -> Option { + let mut reader: RecordReader<'a> = RecordReader::new(section); + let count: usize = length(read(reader.u64())?)?; + let mut strings: Vec = Vec::with_capacity(count.min(section.len())); + for _ in 0..count { + strings.push(read(reader.string())?); + } + let body: &'a [u8] = read(reader.bytes())?; + read(reader.finish())?; + Some(Self { + strings, + body: RecordReader::new(body), + }) + } + + pub fn string(&mut self) -> Option { + let id: usize = length(read(self.body.u64())?)?; + self.strings.get(id).cloned() + } + + pub fn optional_string(&mut self) -> Option> { + Some(if self.flag()? { + Some(self.string()?) + } else { + None + }) + } + + pub fn count(&mut self) -> Option { + length(read(self.body.u64())?) + } + + pub fn flag(&mut self) -> Option { + match read(self.body.u64())? { + 0 => Some(false), + 1 => Some(true), + _ => None, + } + } + + pub fn bytes(&mut self) -> Option<&'a [u8]> { + read(self.body.bytes()) + } + + pub fn list(&mut self, mut item: impl FnMut(&mut Self) -> Option) -> Option> { + let count: usize = self.count()?; + let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); + for _ in 0..count { + items.push(item(self)?); + } + Some(items) + } + + pub fn strings(&mut self) -> Option> { + self.list(Self::string) + } + + pub fn pairs(&mut self) -> Option> { + self.list(|decoder| Some((decoder.string()?, decoder.string()?))) + } + + /// Whether every value was read. + pub fn finish(self) -> Option<()> { + read(self.body.finish()) + } +} + +/// Records environment variables and run-identity reads while entered. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct CompileInputRecording { + pub(crate) recorder: Mutex>>, +} + +#[pymethods] +impl CompileInputRecording { + pub(crate) fn __enter__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let recorder: Bound<'py, CompileInputReads> = Bound::new(py, CompileInputReads::new())?; + registry(py)?.get().locked().push(recorder.clone().unbind()); + *self.recorder.lock().unwrap_or_else(PoisonError::into_inner) = + Some(recorder.clone().unbind()); + Ok(recorder) + } + + #[pyo3(signature = (*_exception))] + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _exception: &Bound<'_, PyTuple>, + ) -> PyResult { + if let Some(recorder) = self + .recorder + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + { + registry(py)? + .get() + .locked() + .retain(|item| !item.is(&recorder)); + } + Ok(false) + } +} + +/// The store a delta compile reads and fills, opened beside its reuse slots. +#[derive(Debug)] +pub struct DocumentStore { + pub store: NativeStore, + pub path: PathBuf, +} + +impl DocumentStore { + /// Each file's document, decoded from the store or loaded by `load` and stored. + pub fn load_with( + &mut self, + paths: &[String], + absolute: impl Fn(&str) -> PathBuf, + load: impl FnOnce(&[String]) -> Result, E>, + ) -> Result, E> { + let mut outcomes: Vec> = Vec::with_capacity(paths.len()); + let mut missed: Vec = Vec::new(); + for path in paths { + let stored: Option = stored_outcome(&mut self.store, &absolute(path)); + if stored.is_none() { + missed.push(path.clone()); + } + outcomes.push(stored); + } + let mut loaded = if missed.is_empty() { + Vec::new() + } else { + load(&missed)? + } + .into_iter(); + let mut complete: Vec = Vec::with_capacity(paths.len()); + for outcome in outcomes { + let outcome: YamlFileOutcome = match outcome { + Some(outcome) => outcome, + None => { + let Some(outcome) = loaded.next() else { + break; + }; + if let YamlFileOutcome::Loaded { contents, value } = &outcome { + self.store + .put(document_key(contents), encoded_document(value)); + } + outcome + } + }; + complete.push(outcome); + } + Ok(complete) + } +} + +impl std::fmt::Debug for PythonProcessClock<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("PythonProcessClock") + } +} diff --git a/crates/sqlbuild-python-base/src/compile_context/tests.rs b/crates/sqlbuild-python-base/src/compile_context/tests.rs new file mode 100644 index 0000000000..24be33e017 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_yaml_cache.rs"] +mod yaml_cache; diff --git a/crates/sqlbuild-python-base/src/compile_context/tests/helpers.rs b/crates/sqlbuild-python-base/src/compile_context/tests/helpers.rs new file mode 100644 index 0000000000..29a22baf38 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/tests/helpers.rs @@ -0,0 +1,6 @@ +use sqlbuild_config::models::ConfigValue; + +/// A YAML string value. +pub(crate) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} diff --git a/crates/sqlbuild-python-base/src/compile_context/tests/test_types.rs b/crates/sqlbuild-python-base/src/compile_context/tests/test_types.rs new file mode 100644 index 0000000000..4a714afeb0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/tests/test_types.rs @@ -0,0 +1,9 @@ +use sqlbuild_config::models::ConfigValue; + +/// A loaded YAML document kept by the delta compile's document store. +pub(crate) struct DocumentTestCase { + pub(crate) description: &'static str, + pub(crate) document: ConfigValue, + /// How many truncations of the stored document still read back as it. + pub(crate) expected_truncations_read: usize, +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs b/crates/sqlbuild-python-base/src/compile_context/tests/test_yaml_cache.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs rename to crates/sqlbuild-python-base/src/compile_context/tests/test_yaml_cache.rs index 53c46aa186..74a676efe0 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs +++ b/crates/sqlbuild-python-base/src/compile_context/tests/test_yaml_cache.rs @@ -1,8 +1,8 @@ use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; -use crate::bindings::_helpers::compile_context::yaml_cache::{decoded_document, encoded_document}; -use crate::bindings::tests::helpers::text; -use crate::bindings::tests::test_types::DocumentTestCase; +use crate::compile_context::_helpers::yaml_cache::{decoded_document, encoded_document}; +use crate::compile_context::tests::helpers::text; +use crate::compile_context::tests::test_types::DocumentTestCase; #[test] fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { diff --git a/crates/sqlbuild-python-base/src/compile_context/types.rs b/crates/sqlbuild-python-base/src/compile_context/types.rs new file mode 100644 index 0000000000..9831d3301e --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_context/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of compile context. + +/// One model's published `(column, type)` shape. +pub type ModelShape = (String, Vec<(String, String)>); diff --git a/crates/sqlbuild-python-base/src/compile_driver/_helpers/mod.rs b/crates/sqlbuild-python-base/src/compile_driver/_helpers/mod.rs new file mode 100644 index 0000000000..d331510271 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod stage_work; diff --git a/crates/sqlbuild-python-base/src/compile_driver/_helpers/stage_work.rs b/crates/sqlbuild-python-base/src/compile_driver/_helpers/stage_work.rs new file mode 100644 index 0000000000..f4a5784d38 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/_helpers/stage_work.rs @@ -0,0 +1,57 @@ +//! How often each compile stage ran in this process, for tests that assert stage work. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyModuleMethods; +use pyo3::wrap_pyfunction; +use std::collections::BTreeMap; + +use pyo3::prelude::{Py, PyRefMut, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyclass, pyfunction}; + +/// Compile stage runs since the last reset, by stage. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct StageWork { + runs: BTreeMap<&'static str, u64>, +} + +static STAGE_WORK: PyOnceLock> = PyOnceLock::new(); + +fn stage_work(py: Python<'_>) -> PyResult> { + Ok(STAGE_WORK + .get_or_try_init(py, || Py::new(py, StageWork::default()))? + .bind(py) + .borrow_mut()) +} + +/// Add `count` runs of `stage`. +pub(crate) fn count_stage(py: Python<'_>, stage: &'static str, count: u64) -> PyResult<()> { + *stage_work(py)?.runs.entry(stage).or_default() += count; + Ok(()) +} + +/// Report the runs of each compile stage so far, optionally resetting them. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn compile_stage_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult> { + let mut work = stage_work(py)?; + let info: BTreeMap = work + .runs + .iter() + .map(|(stage, count)| ((*stage).to_owned(), *count)) + .collect(); + if reset { + *work = StageWork::default(); + } + Ok(info) +} + +/// Register `compile_stage_work_info` on the native module. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_stage_work_info, module)?) +} diff --git a/crates/sqlbuild-python-base/src/compile_driver/constants.rs b/crates/sqlbuild-python-base/src/compile_driver/constants.rs new file mode 100644 index 0000000000..16f7676841 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/constants.rs @@ -0,0 +1,57 @@ +//! Public constants of compile driver. + +pub const EXPRESSION_SHAPES: &str = "infer_native_expression_source_shapes"; + +pub const EXPRESSION_SHAPE_BATCHES: &str = "expression_shape_batches"; + +pub const EXPRESSION_SHAPES_INFERRED: &str = "expression_shapes_inferred"; + +pub const COLUMN_LINEAGE: &str = "build_project_column_lineage"; + +pub const MODEL_CONTRACTS: &str = "evaluate_model_contracts"; + +pub const PROJECT_DISCOVERIES: &str = "discover_project_inputs"; + +pub const MODEL_ANALYSIS: &str = "analyze_native_model_sql"; + +pub const SEMANTIC_COMPLETION: &str = "complete_semantic_project"; + +pub const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; + +pub const PROMOTION_CONFLICTS: &str = "promotion_conflict_diagnostics"; + +pub const PIVOT_PROOFS: &str = "pivot_proofs"; + +pub const SESSION_PIVOT_PROOFS: &str = "session_pivot_proofs"; + +pub const DECLARATION_SCOPES: &str = "build_declaration_scope"; + +pub const LINEAGE_GRAPH_BUILDS: &str = "lineage_graph_builds"; + +pub const PROJECT_GRAPHS: &str = "build_project_graph"; + +pub const FORMAT_LINT_BATCHES: &str = "format_lint_batches"; + +pub const FORMAT_LINT_FILES: &str = "format_lint_files"; + +pub const FORMAT_CACHE_HITS: &str = "format_cache_hits"; + +pub const FORMAT_CACHE_MISSES: &str = "format_cache_misses"; + +pub const SQL_TEST_ARTIFACT_PLANS: &str = "plan_compiled_sql_test_artifacts"; + +pub const SQL_TEST_MODEL_CHAINS: &str = "sql_test_model_chain_names_by_key"; + +pub const BACKGROUND_SQL_TEST_PLANS: &str = "background_sql_test_plans"; + +/// Models whose header columns are reused from the native header parse. +pub const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; + +/// Models whose header columns are parsed again from their config values. +pub const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; + +/// Config values converted into typed nodes for a key a reader asked for. +pub const TYPED_CONFIG_READS: &str = "typed_config_reads"; + +/// Reads of a model or project that is not a native compiled view, read by attribute instead. +pub const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks"; diff --git a/crates/sqlbuild-python-base/src/compile_driver/main/count_stage.rs b/crates/sqlbuild-python-base/src/compile_driver/main/count_stage.rs new file mode 100644 index 0000000000..427da75623 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/main/count_stage.rs @@ -0,0 +1,11 @@ +//! Add `count` runs of `stage`. + +use crate::compile_driver::_helpers::stage_work; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Add `count` runs of `stage`. +#[inline] +pub fn count_stage(py: Python<'_>, stage: &'static str, count: u64) -> PyResult<()> { + stage_work::count_stage(py, stage, count) +} diff --git a/crates/sqlbuild-python-base/src/compile_driver/main/mod.rs b/crates/sqlbuild-python-base/src/compile_driver/main/mod.rs new file mode 100644 index 0000000000..d762b73875 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/main/mod.rs @@ -0,0 +1,2 @@ +pub mod count_stage; +pub mod register_stage_work; diff --git a/crates/sqlbuild-python-base/src/compile_driver/main/register_stage_work.rs b/crates/sqlbuild-python-base/src/compile_driver/main/register_stage_work.rs new file mode 100644 index 0000000000..42ccf912db --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/main/register_stage_work.rs @@ -0,0 +1,12 @@ +//! Register `compile_stage_work_info` on the native module. + +use crate::compile_driver::_helpers::stage_work; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register `compile_stage_work_info` on the native module. +#[inline] +pub fn register_stage_work(module: &Bound<'_, PyModule>) -> PyResult<()> { + stage_work::register(module) +} diff --git a/crates/sqlbuild-python-base/src/compile_driver/mod.rs b/crates/sqlbuild-python-base/src/compile_driver/mod.rs new file mode 100644 index 0000000000..61af85e0b5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_driver/mod.rs @@ -0,0 +1,5 @@ +//! How often each compile stage ran in this process. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python-base/src/compile_outputs/_helpers/json_report.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs rename to crates/sqlbuild-python-base/src/compile_outputs/_helpers/json_report.rs index 1b648b8e6a..94e1086915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python-base/src/compile_outputs/_helpers/json_report.rs @@ -1,10 +1,11 @@ //! Emit a command's JSON report as orjson would, or as `json.dumps` does when orjson rejects it. +use crate::boundary::main::utf16_units::utf16_units; use pyo3::exceptions::{PyRecursionError, PyTypeError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{ - PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, - PyString, PyStringMethods, PyTuple, PyTypeMethods, + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, PyTypeMethods, }; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::main::dumps::dumps; @@ -12,8 +13,8 @@ use sqlbuild_core::json::models::{ JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, }; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use crate::boundary::main::compiler_guard::compiler_guard; +use crate::boundary::main::native_compiler_error::native_compiler_error; const ORJSON_SURROGATE: &str = "str is not valid UTF-8: surrogates not allowed"; const ORJSON_WIDE_INTEGER: &str = "Integer exceeds 64-bit range"; @@ -25,9 +26,6 @@ const ORJSON_MAX_CONTAINER_DEPTH: usize = 254; const PYTHON_RECURSION_LIMIT: usize = 1_000; const PYTHON_RECURSION_MESSAGE: &str = "maximum recursion depth exceeded"; const SURROGATE_KEY: &str = "a JSON report key holds a lone surrogate, which report keys never do"; -const ENCODE: &str = "encode"; -const UTF16_LE: &str = "utf-16-le"; -const SURROGATE_PASS: &str = "surrogatepass"; const INDENTED: OrjsonOptions = OrjsonOptions { indent_2: true, sort_keys: false, @@ -69,7 +67,7 @@ pub(crate) fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { if let Some(error) = conversion.stdlib_error { return Err(error); } - crate::bindings::_helpers::compile_context::delta::capture(|capture| { + crate::compile_context::main::delta::capture::capture(|capture| { capture.stdlib_json = true; }); emitted( @@ -188,18 +186,7 @@ impl Conversion { return Ok(JsonValue::String(text.to_owned())); } self.orjson_rejects(ORJSON_SURROGATE.to_owned()); - let encoded: Bound<'_, PyBytes> = text - .call_method1(ENCODE, (UTF16_LE, SURROGATE_PASS))? - .downcast_into::()?; - Ok(JsonValue::Utf16( - encoded - .as_bytes() - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) - .collect(), - )) + utf16_units(text).map(JsonValue::Utf16) } /// orjson encodes 64-bit integers; wider ones make it raise, so `json.dumps` writes them. diff --git a/crates/sqlbuild-python-base/src/compile_outputs/_helpers/mod.rs b/crates/sqlbuild-python-base/src/compile_outputs/_helpers/mod.rs new file mode 100644 index 0000000000..5244282350 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_report; diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/emit_json_report.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/emit_json_report.rs new file mode 100644 index 0000000000..52a0c319f8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/emit_json_report.rs @@ -0,0 +1,12 @@ +//! The report as orjson writes it, or as `json.dumps(indent=2)` does when orjson rejects it. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The report as orjson writes it, or as `json.dumps(indent=2)` does when orjson rejects it. +#[inline] +pub fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { + json_report::emit_json_report(report) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/emit_orjson_report.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/emit_orjson_report.rs new file mode 100644 index 0000000000..4c0bfa1a89 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/emit_orjson_report.rs @@ -0,0 +1,12 @@ +//! The report as `orjson.dumps(report, option=OPT_INDENT_2)`, raising orjson's `TypeError`. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The report as `orjson.dumps(report, option=OPT_INDENT_2)`, raising orjson's `TypeError`. +#[inline] +pub fn emit_orjson_report(report: &Bound<'_, PyAny>) -> PyResult { + json_report::emit_orjson_report(report) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/float_key.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/float_key.rs new file mode 100644 index 0000000000..d5de8432d2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/float_key.rs @@ -0,0 +1,12 @@ +//! `json.dumps` writes a float key as `float.__repr__`, or as `NaN` / `Infinity`. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `json.dumps` writes a float key as `float.__repr__`, or as `NaN` / `Infinity`. +#[inline] +pub fn float_key(key: &Bound<'_, PyAny>) -> PyResult { + json_report::float_key(key) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/indented_json.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/indented_json.rs new file mode 100644 index 0000000000..97e30dbd48 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/indented_json.rs @@ -0,0 +1,12 @@ +//! `json.dumps(value, indent=2)`, raising what `json.dumps` raises. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `json.dumps(value, indent=2)`, raising what `json.dumps` raises. +#[inline] +pub fn indented_json(value: &Bound<'_, PyAny>) -> PyResult { + json_report::indented_json(value) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/mod.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/mod.rs new file mode 100644 index 0000000000..422b421ac5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/mod.rs @@ -0,0 +1,8 @@ +pub mod emit_json_report; +pub mod emit_orjson_report; +pub mod float_key; +pub mod indented_json; +pub mod orjson_sorted; +pub mod orjson_value; +pub mod plain_json; +pub mod register_json_report; diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_sorted.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_sorted.rs new file mode 100644 index 0000000000..cef09d1f83 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_sorted.rs @@ -0,0 +1,11 @@ +//! `orjson.dumps(value, option=OPT_SORT_KEYS)` for an already-native payload. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// `orjson.dumps(value, option=OPT_SORT_KEYS)` for an already-native payload. +#[inline] +pub fn orjson_sorted(value: &JsonValue) -> PyResult { + json_report::orjson_sorted(value) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_value.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_value.rs new file mode 100644 index 0000000000..5e09866be6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/orjson_value.rs @@ -0,0 +1,13 @@ +//! A Python value in orjson's supported domain, converted once at its data boundary. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// A Python value in orjson's supported domain, converted once at its data boundary. +#[inline] +pub fn orjson_value(value: &Bound<'_, PyAny>) -> PyResult { + json_report::orjson_value(value) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/plain_json.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/plain_json.rs new file mode 100644 index 0000000000..61e0620062 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/plain_json.rs @@ -0,0 +1,12 @@ +//! `json.dumps(value)`, raising what `json.dumps` raises. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `json.dumps(value)`, raising what `json.dumps` raises. +#[inline] +pub fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { + json_report::plain_json(value) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/main/register_json_report.rs b/crates/sqlbuild-python-base/src/compile_outputs/main/register_json_report.rs new file mode 100644 index 0000000000..4cb2a92659 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/main/register_json_report.rs @@ -0,0 +1,12 @@ +//! Register the json report functions and classes. + +use crate::compile_outputs::_helpers::json_report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the json report functions and classes. +#[inline] +pub fn register_json_report(module: &Bound<'_, PyModule>) -> PyResult<()> { + json_report::register(module) +} diff --git a/crates/sqlbuild-python-base/src/compile_outputs/mod.rs b/crates/sqlbuild-python-base/src/compile_outputs/mod.rs new file mode 100644 index 0000000000..3e0e37ebc6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/compile_outputs/mod.rs @@ -0,0 +1,4 @@ +//! A command's JSON report as orjson or `json.dumps` writes it. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-base/src/contracts/_helpers/mod.rs b/crates/sqlbuild-python-base/src/contracts/_helpers/mod.rs new file mode 100644 index 0000000000..7ba26f9eb6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod resource_identities; +pub(crate) mod state_literals; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python-base/src/contracts/_helpers/resource_identities.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs rename to crates/sqlbuild-python-base/src/contracts/_helpers/resource_identities.rs index 6ebb9758f9..b8ea4ba49e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python-base/src/contracts/_helpers/resource_identities.rs @@ -1,13 +1,13 @@ //! Canonical snake_case resource identity checks and SQL function node types. +use crate::records::main::construct::error_class::error_class; +use crate::records::types::ErrorId; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; const TABLE_FUNCTION_NODE_TYPE: &str = "table_fn"; const UDF_NODE_TYPE: &str = "udf"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python-base/src/contracts/_helpers/state_literals.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs rename to crates/sqlbuild-python-base/src/contracts/_helpers/state_literals.rs index 3d54e11834..51821104ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs +++ b/crates/sqlbuild-python-base/src/contracts/_helpers/state_literals.rs @@ -1,5 +1,6 @@ //! Portable typed SQL literals for warehouse-state columns. +use crate::native_host::main::host_error::host_error; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -15,8 +16,7 @@ const BOOLEAN_TYPE: &str = "boolean"; const TIMESTAMP_TYPE: &str = "timestamp"; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::records::main::fields::field::field as native_field; use sqlbuild_state::freshness::constants::TEXT_TIMESTAMP_STATE_VALUE_TYPE as TEXT_TIMESTAMP_TYPE; const DATE_TYPE: &str = "date"; /// The portable SQL type state timestamps cast to unless the warehouse names a finer one. @@ -158,12 +158,7 @@ fn typed_null(declared_type: &str) -> &'static str { } fn rendering_error(py: Python<'_>, message: &str) -> PyErr { - let raised = || -> PyResult { - let error: Bound<'_, PyAny> = - host_entry(py, ExecutorHost::SqlValueRenderingError)?.call1((message,))?; - Ok(PyErr::from_value(error)) - }; - raised().unwrap_or_else(|error| error) + host_error(py, ExecutorHost::SqlValueRenderingError, message) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python-base/src/contracts/main/mod.rs b/crates/sqlbuild-python-base/src/contracts/main/mod.rs new file mode 100644 index 0000000000..661b87f9c8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/main/mod.rs @@ -0,0 +1,4 @@ +pub mod register_resource_identities; +pub mod register_state_literals; +pub mod render_state_sql_literal; +pub mod validate_resource_identity; diff --git a/crates/sqlbuild-python-base/src/contracts/main/register_resource_identities.rs b/crates/sqlbuild-python-base/src/contracts/main/register_resource_identities.rs new file mode 100644 index 0000000000..a10a37de49 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/main/register_resource_identities.rs @@ -0,0 +1,12 @@ +//! Register the resource identities functions and classes. + +use crate::contracts::_helpers::resource_identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the resource identities functions and classes. +#[inline] +pub fn register_resource_identities(module: &Bound<'_, PyModule>) -> PyResult<()> { + resource_identities::register(module) +} diff --git a/crates/sqlbuild-python-base/src/contracts/main/register_state_literals.rs b/crates/sqlbuild-python-base/src/contracts/main/register_state_literals.rs new file mode 100644 index 0000000000..607717a8ed --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/main/register_state_literals.rs @@ -0,0 +1,12 @@ +//! Register the state literals functions and classes. + +use crate::contracts::_helpers::state_literals; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the state literals functions and classes. +#[inline] +pub fn register_state_literals(module: &Bound<'_, PyModule>) -> PyResult<()> { + state_literals::register(module) +} diff --git a/crates/sqlbuild-python-base/src/contracts/main/render_state_sql_literal.rs b/crates/sqlbuild-python-base/src/contracts/main/render_state_sql_literal.rs new file mode 100644 index 0000000000..d252a2f681 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/main/render_state_sql_literal.rs @@ -0,0 +1,18 @@ +//! `value` as its state column's literal, timestamps cast to `timestamp_type` when given. + +use crate::contracts::_helpers::state_literals; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `value` as its state column's literal, timestamps cast to `timestamp_type` when given. +#[inline] +pub fn render_state_sql_literal( + py: Python<'_>, + value: &Bound<'_, PyAny>, + declared_type: &str, + timestamp_type: Option<&str>, +) -> PyResult { + state_literals::render_state_sql_literal(py, value, declared_type, timestamp_type) +} diff --git a/crates/sqlbuild-python-base/src/contracts/main/validate_resource_identity.rs b/crates/sqlbuild-python-base/src/contracts/main/validate_resource_identity.rs new file mode 100644 index 0000000000..fc818ce955 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/main/validate_resource_identity.rs @@ -0,0 +1,18 @@ +//! Reject one identity unless it uses canonical lowercase snake_case. + +use crate::contracts::_helpers::resource_identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; +use pyo3::types::PyString; + +/// Reject one identity unless it uses canonical lowercase snake_case. +#[inline] +pub fn validate_resource_identity( + name: &Bound<'_, PyString>, + kind: &str, + path: &Bound<'_, PyAny>, + private_identity: bool, +) -> Result<(), PyErr> { + resource_identities::validate_resource_identity(name, kind, path, private_identity) +} diff --git a/crates/sqlbuild-python-base/src/contracts/mod.rs b/crates/sqlbuild-python-base/src/contracts/mod.rs new file mode 100644 index 0000000000..a8fe20c600 --- /dev/null +++ b/crates/sqlbuild-python-base/src/contracts/mod.rs @@ -0,0 +1,4 @@ +//! Resource identity checks and typed SQL literals for warehouse-state columns. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/console_records.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs rename to crates/sqlbuild-python-base/src/diagnostics/_helpers/console_records.rs index a3676b00d7..e38ac4e9f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/console_records.rs @@ -1,5 +1,6 @@ //! The `--debug` stderr line of one Python log record. +use crate::records::main::fields::field::field as native_field; use pyo3::exceptions::PyOverflowError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyInt, PyString}; @@ -8,7 +9,6 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::diagnostic_routing::constants::SQL_TEXT_FIELD; use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The record field holding an operation's duration in milliseconds. const DURATION_FIELD: &str = "sqlbuild_duration_ms"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/getters.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/diagnostics/getters.rs rename to crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs index 5304244325..271ac0ee01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/getters.rs +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::diagnostics::models::BuildPhaseTimingTracker; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_build_phase_timing_tracker, -}]; - -fn read_build_phase_timing_tracker<'py>( +pub(crate) fn read_build_phase_timing_tracker<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs rename to crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs index 8d1c737795..78801b5487 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs @@ -25,12 +25,12 @@ use sqlbuild_core::diagnostic_routing::models::{RecordFacts, RouteOptions, Third use sqlbuild_core::redaction::main::redact_text::redact_text; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::diagnostics::redaction::redacted_value; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use crate::boundary::main::imported::imported; +use crate::boundary::models::Request; +use crate::diagnostics::_helpers::redaction::redacted_value; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_items::mapping_items; const PRIVATE_SQL_MARKER: ExecutorHost = ExecutorHost::SqlPrivateRecordMarker; diff --git a/crates/sqlbuild-python-base/src/diagnostics/_helpers/mod.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/mod.rs new file mode 100644 index 0000000000..bfb17b9fef --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod console_records; +pub(crate) mod getters; +pub(crate) mod log_records; +pub(crate) mod redaction; +pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/redaction.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs rename to crates/sqlbuild-python-base/src/diagnostics/_helpers/redaction.rs index deab38118e..e61cb1ccef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/redaction.rs @@ -9,8 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::redaction::main::redact_text::redact_text; use sqlbuild_core::redaction::main::secret_field_name::secret_field_name; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use crate::boundary::main::imported::imported; +use crate::records::main::fields::mapping_items::mapping_items; const REDACTED: &str = "[REDACTED]"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python-base/src/diagnostics/_helpers/trackers.rs similarity index 53% rename from crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs rename to crates/sqlbuild-python-base/src/diagnostics/_helpers/trackers.rs index 0f083f387a..c8a325fcd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python-base/src/diagnostics/_helpers/trackers.rs @@ -1,21 +1,19 @@ //! Invocation-scoped partial build phase timings and process resource deltas. -use std::sync::{Mutex, PoisonError}; +use crate::diagnostics::models::BuildPhaseTimingTracker; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyType}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyclass, pymethods}; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_python_base::boundary::main::context_variable::context_variable; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::types::RecordId; +use crate::boundary::main::context_variable::context_variable; +use crate::boundary::main::imported::imported; +use crate::native_host::main::host_entry::host_entry; const TRACKER_VARIABLE: &str = "sqlbuild_build_phase_timing_tracker"; -const PHASES: [&str; 6] = [ +pub(crate) const PHASES: [&str; 6] = [ "compile_seconds", "planning_seconds", "connection_preparation_seconds", @@ -27,7 +25,7 @@ const PHASES: [&str; 6] = [ static CURRENT_TRACKER: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the active build timing tracker. -fn current_variable(py: Python<'_>) -> PyResult> { +pub(crate) fn current_variable(py: Python<'_>) -> PyResult> { context_variable(py, &CURRENT_TRACKER, TRACKER_VARIABLE) } @@ -37,7 +35,7 @@ pub(crate) fn current_tracker(py: Python<'_>) -> PyResult> { } /// `monotonic`, or `time.monotonic` when none is given. -fn clock<'py>( +pub(crate) fn clock<'py>( py: Python<'py>, monotonic: Option>, ) -> PyResult> { @@ -48,98 +46,19 @@ fn clock<'py>( } /// `max(0.0, later - earlier)`. -fn non_negative(later: f64, earlier: f64) -> f64 { +pub(crate) fn non_negative(later: f64, earlier: f64) -> f64 { (later - earlier).max(0.0) } -/// Retain disjoint monotonic phase durations across exceptional exits. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct BuildPhaseTimingTracker { - monotonic: Py, - started_at: f64, - #[pyo3(get, set)] - pub(crate) compile_seconds: Option, - #[pyo3(get, set)] - pub(crate) planning_seconds: Option, - #[pyo3(get, set)] - pub(crate) connection_preparation_seconds: Option, - #[pyo3(get, set)] - pub(crate) schema_preparation_seconds: Option, - #[pyo3(get, set)] - pub(crate) execution_seconds: Option, - #[pyo3(get, set)] - pub(crate) cost_collection_seconds: Option, - tokens: Mutex>>, -} - -#[pymethods] -impl BuildPhaseTimingTracker { - #[new] - #[pyo3(signature = (*, monotonic = None))] - pub(crate) fn new(py: Python<'_>, monotonic: Option>) -> PyResult { - let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; - let started_at: f64 = monotonic.call0()?.extract()?; - Ok(Self { - monotonic: monotonic.unbind(), - started_at, - compile_seconds: None, - planning_seconds: None, - connection_preparation_seconds: None, - schema_preparation_seconds: None, - execution_seconds: None, - cost_collection_seconds: None, - tokens: Mutex::new(Vec::new()), - }) - } - - /// Return currently available phase durations and total wall time. - fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { - let now: f64 = self.monotonic.bind(py).call0()?.extract()?; - let values: [Option; 6] = [ - self.compile_seconds, - self.planning_seconds, - self.connection_preparation_seconds, - self.schema_preparation_seconds, - self.execution_seconds, - self.cost_collection_seconds, - ]; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - for (phase, value) in PHASES.into_iter().zip(values) { - keywords.set_item(phase, value)?; - } - keywords.set_item("total_seconds", non_negative(now, self.started_at))?; - new_record(py, RecordId::PartialBuildPhaseTimings, (), Some(&keywords)) - } - - /// Return the active build timing tracker. - #[classmethod] - fn current<'py>(cls: &Bound<'py, PyType>) -> PyResult> { - current_variable(cls.py())?.call_method0("get") - } - - /// Install this tracker for one command invocation and reset it afterward. - fn scope(slf: &Bound<'_, Self>) -> PhaseTimingScope { - PhaseTimingScope { - tracker: slf.clone().unbind(), - } - } -} - -impl BuildPhaseTimingTracker { - fn tokens(&self) -> std::sync::MutexGuard<'_, Vec>> { - self.tokens.lock().unwrap_or_else(PoisonError::into_inner) - } -} - /// Makes one tracker the active build timing tracker while entered. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct PhaseTimingScope { - tracker: Py, + pub(crate) tracker: Py, } #[pymethods] impl PhaseTimingScope { - fn __enter__(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn __enter__(&self, py: Python<'_>) -> PyResult<()> { let token: Bound<'_, PyAny> = current_variable(py)?.call_method1("set", (self.tracker.bind(py),))?; self.tracker.bind(py).borrow().tokens().push(token.unbind()); @@ -147,7 +66,11 @@ impl PhaseTimingScope { } #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _exception: &Bound<'_, PyTuple>, + ) -> PyResult { let token: Option> = self.tracker.bind(py).borrow().tokens().pop(); if let Some(token) = token { current_variable(py)?.call_method1("reset", (token,))?; @@ -170,7 +93,7 @@ pub(crate) struct ProcessResourceTracker { impl ProcessResourceTracker { #[new] #[pyo3(signature = (*, monotonic = None, resource_reader = None))] - fn new( + pub(crate) fn new( py: Python<'_>, monotonic: Option>, resource_reader: Option>, diff --git a/crates/sqlbuild-python-base/src/diagnostics/constants.rs b/crates/sqlbuild-python-base/src/diagnostics/constants.rs new file mode 100644 index 0000000000..1be65e8f02 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of diagnostics. + +use crate::diagnostics::_helpers::getters::read_build_phase_timing_tracker; +use crate::diagnostics::models::BuildPhaseTimingTracker; +use crate::records::main::registry::class_of::class_of; +use crate::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_build_phase_timing_tracker, +}]; diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/current_tracker.rs b/crates/sqlbuild-python-base/src/diagnostics/main/current_tracker.rs new file mode 100644 index 0000000000..001b51104c --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/current_tracker.rs @@ -0,0 +1,13 @@ +//! The active build timing tracker, or None. + +use crate::diagnostics::_helpers::trackers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The active build timing tracker, or None. +#[inline] +pub fn current_tracker(py: Python<'_>) -> PyResult> { + trackers::current_tracker(py) +} diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/mod.rs b/crates/sqlbuild-python-base/src/diagnostics/main/mod.rs new file mode 100644 index 0000000000..e83908a122 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/mod.rs @@ -0,0 +1,5 @@ +pub mod current_tracker; +pub mod register_console_records; +pub mod register_log_records; +pub mod register_redaction; +pub mod register_trackers; diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/register_console_records.rs b/crates/sqlbuild-python-base/src/diagnostics/main/register_console_records.rs new file mode 100644 index 0000000000..ed812f5572 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/register_console_records.rs @@ -0,0 +1,12 @@ +//! Register the console records functions and classes. + +use crate::diagnostics::_helpers::console_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the console records functions and classes. +#[inline] +pub fn register_console_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + console_records::register(module) +} diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/register_log_records.rs b/crates/sqlbuild-python-base/src/diagnostics/main/register_log_records.rs new file mode 100644 index 0000000000..919ff48528 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/register_log_records.rs @@ -0,0 +1,12 @@ +//! Register the log records functions and classes. + +use crate::diagnostics::_helpers::log_records; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the log records functions and classes. +#[inline] +pub fn register_log_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + log_records::register(module) +} diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/register_redaction.rs b/crates/sqlbuild-python-base/src/diagnostics/main/register_redaction.rs new file mode 100644 index 0000000000..e1338bc274 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/register_redaction.rs @@ -0,0 +1,12 @@ +//! Register the redaction functions and classes. + +use crate::diagnostics::_helpers::redaction; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the redaction functions and classes. +#[inline] +pub fn register_redaction(module: &Bound<'_, PyModule>) -> PyResult<()> { + redaction::register(module) +} diff --git a/crates/sqlbuild-python-base/src/diagnostics/main/register_trackers.rs b/crates/sqlbuild-python-base/src/diagnostics/main/register_trackers.rs new file mode 100644 index 0000000000..52053cf184 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/main/register_trackers.rs @@ -0,0 +1,12 @@ +//! Register the trackers functions and classes. + +use crate::diagnostics::_helpers::trackers; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the trackers functions and classes. +#[inline] +pub fn register_trackers(module: &Bound<'_, PyModule>) -> PyResult<()> { + trackers::register(module) +} diff --git a/crates/sqlbuild-python-base/src/diagnostics/mod.rs b/crates/sqlbuild-python-base/src/diagnostics/mod.rs new file mode 100644 index 0000000000..625d029d8f --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/mod.rs @@ -0,0 +1,6 @@ +//! Diagnostic log records, console lines, redaction and build phase trackers. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/diagnostics/models.rs b/crates/sqlbuild-python-base/src/diagnostics/models.rs new file mode 100644 index 0000000000..99de762ec9 --- /dev/null +++ b/crates/sqlbuild-python-base/src/diagnostics/models.rs @@ -0,0 +1,102 @@ +//! Public records of diagnostics. + +use crate::diagnostics::_helpers::trackers::PHASES; +use crate::diagnostics::_helpers::trackers::PhaseTimingScope; +use crate::diagnostics::_helpers::trackers::clock; +use crate::diagnostics::_helpers::trackers::current_variable; +use crate::diagnostics::_helpers::trackers::non_negative; +use crate::records::main::construct::new_record::new_record; +use crate::records::types::RecordId; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyType; +use std::sync::Mutex; +use std::sync::PoisonError; + +/// Retain disjoint monotonic phase durations across exceptional exits. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct BuildPhaseTimingTracker { + pub(crate) monotonic: Py, + pub(crate) started_at: f64, + #[pyo3(get, set)] + pub compile_seconds: Option, + #[pyo3(get, set)] + pub planning_seconds: Option, + #[pyo3(get, set)] + pub connection_preparation_seconds: Option, + #[pyo3(get, set)] + pub schema_preparation_seconds: Option, + #[pyo3(get, set)] + pub execution_seconds: Option, + #[pyo3(get, set)] + pub cost_collection_seconds: Option, + pub(crate) tokens: Mutex>>, +} + +#[pymethods] +impl BuildPhaseTimingTracker { + #[new] + #[pyo3(signature = (*, monotonic = None))] + pub fn new(py: Python<'_>, monotonic: Option>) -> PyResult { + let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; + let started_at: f64 = monotonic.call0()?.extract()?; + Ok(Self { + monotonic: monotonic.unbind(), + started_at, + compile_seconds: None, + planning_seconds: None, + connection_preparation_seconds: None, + schema_preparation_seconds: None, + execution_seconds: None, + cost_collection_seconds: None, + tokens: Mutex::new(Vec::new()), + }) + } + + /// Return currently available phase durations and total wall time. + pub(crate) fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { + let now: f64 = self.monotonic.bind(py).call0()?.extract()?; + let values: [Option; 6] = [ + self.compile_seconds, + self.planning_seconds, + self.connection_preparation_seconds, + self.schema_preparation_seconds, + self.execution_seconds, + self.cost_collection_seconds, + ]; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (phase, value) in PHASES.into_iter().zip(values) { + keywords.set_item(phase, value)?; + } + keywords.set_item("total_seconds", non_negative(now, self.started_at))?; + new_record(py, RecordId::PartialBuildPhaseTimings, (), Some(&keywords)) + } + + /// Return the active build timing tracker. + #[classmethod] + pub(crate) fn current<'py>(cls: &Bound<'py, PyType>) -> PyResult> { + current_variable(cls.py())?.call_method0("get") + } + + /// Install this tracker for one command invocation and reset it afterward. + pub(crate) fn scope(slf: &Bound<'_, Self>) -> PhaseTimingScope { + PhaseTimingScope { + tracker: slf.clone().unbind(), + } + } +} + +impl BuildPhaseTimingTracker { + pub(crate) fn tokens(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.tokens.lock().unwrap_or_else(PoisonError::into_inner) + } +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/_helpers/mod.rs b/crates/sqlbuild-python-base/src/event_exporting/_helpers/mod.rs new file mode 100644 index 0000000000..b25725a0ef --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod scope; +pub(crate) mod values; diff --git a/crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs b/crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs new file mode 100644 index 0000000000..a5e4ea4411 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs @@ -0,0 +1,237 @@ +//! `EventExporterCommandScope`: typed sink binding, its provider session and the active command scope. + +use crate::event_exporting::models::EventExporterCommandScope; +use crate::event_exporting::models::EventExporterScopeContext; +use std::collections::HashMap; +use std::sync::Mutex; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyclass, + pyfunction, pymethods, wrap_pyfunction, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use sqlbuild_host::entries::models::ExecutorHost; + +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::context_variable::context_variable; +use crate::event_exporting::constants::SCOPE_CONTEXT_VARIABLE; +use crate::event_exporting::models::BoundEventExporter; +use crate::native_host::main::host_entry::host_entry; +use crate::native_host::main::write_console::write_console; +use crate::orchestration_reads::main::provider_state::provider_state; +use crate::orchestration_reads::models::ProviderState; +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_get::mapping_get; + +pub(crate) static CURRENT_SCOPE: PyOnceLock> = PyOnceLock::new(); + +/// Discovered extensions and the provider session one command bound. +#[derive(Debug)] +pub(crate) struct ScopeState { + pub(crate) configured: bool, + pub(crate) provider_close_registered: bool, + pub(crate) provider_session: Option>, + pub(crate) project_dir: Option>, + pub(crate) providers: Option>, + pub(crate) event_exporters: Option>, + pub(crate) command_output_sinks: Option>, +} + +/// Releases and force-closes a provider session on the closing thread once sink delivery is idle. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ProviderSessionCloser { + pub(crate) session: Py, +} + +pub(crate) fn empty_tuple(py: Python<'_>) -> Py { + PyTuple::empty(py).into_any().unbind() +} + +pub(crate) fn tuple_or_empty(py: Python<'_>, value: Option<&Py>) -> Py { + value.map_or_else(|| empty_tuple(py), |value| value.clone_ref(py)) +} + +pub(crate) fn optional(py: Python<'_>, value: Option<&Py>) -> Py { + value.map_or_else(|| py.None(), |value| value.clone_ref(py)) +} + +/// `{usage.parameter_name: session.get(usage.provider_name)}` for one declaration. +fn provider_arguments<'py>( + session: &Bound<'py, PyAny>, + declaration: &Bound<'py, PyAny>, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(session.py()); + for usage in native_field(declaration, "provider_usages")?.try_iter()? { + let usage: Bound<'py, PyAny> = usage?; + arguments.set_item( + native_field(&usage, "parameter_name")?, + mapping_get(session, native_field(&usage, "provider_name")?)?, + )?; + } + Ok(arguments) +} + +/// Each discovered provider whose required secret settings are unset, with their variables. +pub(crate) fn providers_missing_secrets( + providers: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut missing: HashMap> = HashMap::new(); + for provider in providers.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + if let ProviderState::MissingSecrets(variables) = + provider_state(&native_field(&provider, "provider_class")?)? + { + missing.insert(native_field(&provider, "name")?.extract()?, variables); + } + } + Ok(missing) +} + +/// Whether `declaration` can bind; when a provider it takes is missing secrets, a notice says it is left unbound. +fn bindable( + declaration: &Bound<'_, PyAny>, + missing: &HashMap>, +) -> PyResult { + for usage in native_field(declaration, "provider_usages")?.try_iter()? { + let provider: String = native_field(&usage?, "provider_name")?.extract()?; + if let Some(variables) = missing.get(&provider) { + write_console( + declaration.py(), + "stderr", + &format!( + "sink '{}' is not bound: provider '{provider}' is missing {}\n", + native_field(declaration, "name")?.str()?, + variables.join(", ") + ), + )?; + return Ok(false); + } + } + Ok(true) +} + +/// Bound lifecycle exporters and command-output sinks over `session`, except those using `missing` providers. +pub(crate) fn bindings<'py>( + session: &Bound<'py, PyAny>, + (event_exporters, command_output_sinks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + missing: &HashMap>, +) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyTuple>)> { + let py: Python<'py> = session.py(); + let mut exporters: Vec> = Vec::new(); + for exporter in event_exporters.try_iter()? { + let exporter: Bound<'py, PyAny> = exporter?; + if !bindable(&exporter, missing)? { + continue; + } + let arguments: Bound<'py, PyDict> = provider_arguments(session, &exporter)?; + exporters.push( + Bound::new( + py, + BoundEventExporter::create( + native_field(&exporter, "name")?.extract()?, + ( + native_field(&exporter, "function")?.unbind(), + arguments.into_any().unbind(), + ), + ( + Some(&native_field(&exporter, "event_kinds")?), + Some(&native_field(&exporter, "min_severity")?), + ), + )?, + )? + .into_any(), + ); + } + let sink_class: Bound<'py, PyAny> = host_entry(py, ExecutorHost::BoundCommandOutputSink)?; + let mut sinks: Vec> = Vec::new(); + for sink in command_output_sinks.try_iter()? { + let sink: Bound<'py, PyAny> = sink?; + if !bindable(&sink, missing)? { + continue; + } + let arguments: Bound<'py, PyDict> = provider_arguments(session, &sink)?; + sinks.push(call_with( + &sink_class, + &[ + ("name", native_field(&sink, "name")?), + ("function", native_field(&sink, "function")?), + ("provider_arguments", arguments.into_any()), + ("streams", native_field(&sink, "streams")?), + ], + )?); + } + Ok((PyTuple::new(py, exporters)?, PyTuple::new(py, sinks)?)) +} + +#[pymethods] +impl ProviderSessionCloser { + pub(crate) fn __call__(&self, py: Python<'_>) -> PyResult<()> { + let session: &Bound<'_, PyAny> = self.session.bind(py); + let released = session.call_method0("release"); + call_with( + &native_field(session, "close")?, + &[("force", PyBool::new(py, true).to_owned().into_any())], + )?; + released?; + Ok(()) + } +} + +/// Install one command-owned exporter scope. +#[pyfunction] +pub(crate) fn event_exporter_command_scope(scope: Py) -> EventExporterScopeContext { + EventExporterScopeContext { + scope, + token: Mutex::new(None), + } +} + +/// The active command-owned exporter scope, or `None`. +pub(crate) fn current_scope(py: Python<'_>) -> PyResult> { + context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)?.call_method0("get") +} + +/// Return the active command-owned exporter scope, if any. +#[pyfunction] +fn current_event_exporter_command_scope(py: Python<'_>) -> PyResult> { + current_scope(py) +} + +/// Configure the active exporter scope from validated discovery inputs. +pub(crate) fn configure_discovered_event_exporters(discovered: &Bound<'_, PyAny>) -> PyResult<()> { + let scope: Bound<'_, PyAny> = current_scope(discovered.py())?; + if !scope.is_none() { + scope.call_method1("configure", (discovered,))?; + } + Ok(()) +} + +/// Return startup-discovered extensions for the active project root. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn cached_event_exporter_extensions<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let scope: Bound<'py, PyAny> = current_scope(py)?; + if scope.is_none() { + return Ok(scope); + } + call_with( + &native_field(&scope, "cached_extensions")?, + &[("project_dir", project_dir.clone())], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(event_exporter_command_scope, module)?)?; + module.add_function(wrap_pyfunction!( + current_event_exporter_command_scope, + module + )?)?; + module.add_function(wrap_pyfunction!(cached_event_exporter_extensions, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs b/crates/sqlbuild-python-base/src/event_exporting/_helpers/values.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs rename to crates/sqlbuild-python-base/src/event_exporting/_helpers/values.rs index fe059dc88e..baf1543815 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs +++ b/crates/sqlbuild-python-base/src/event_exporting/_helpers/values.rs @@ -1,5 +1,6 @@ //! Event exporter errors, severities, reprs and the lifecycle export policy catalogue across the Python boundary. +use crate::native_host::main::host_error::host_error; use std::collections::HashMap; use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; @@ -13,9 +14,8 @@ use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_V use sqlbuild_core::lifecycle_events::main::catalog::catalog; use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::types::RecordId; +use crate::records::main::construct::enum_from_value::enum_from_value; +use crate::records::types::RecordId; /// `BoundEventExporter` parameters, in positional order. const BOUND_EXPORTER_PARAMETERS: [&str; 5] = [ @@ -31,10 +31,7 @@ static POLICIES: PyOnceLock> = PyOnceLock::new(); /// The exception class `name` of the event exporter exceptions module, raised with `message`. pub(crate) fn exporter_error(py: Python<'_>, name: UserCodeHost, message: &str) -> PyErr { - match host_entry(py, name).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, name, message) } /// `EventExporterStateError(message)`. diff --git a/crates/sqlbuild-python-base/src/event_exporting/constants.rs b/crates/sqlbuild-python-base/src/event_exporting/constants.rs new file mode 100644 index 0000000000..9a195b54d6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/constants.rs @@ -0,0 +1,38 @@ +//! Public constants of event exporting. + +pub(crate) const SCOPE_CONTEXT_VARIABLE: &str = "sqlbuild_event_exporter_command_scope"; + +pub const DISPATCHER_THREAD_NAME: &str = "sqlbuild-event-exporter-dispatcher"; + +pub const NOTIFIER_THREAD_NAME: &str = "sqlbuild-event-exporter-notifier"; + +pub const HEALTH_THREAD_NAME: &str = "sqlbuild-event-exporter-health"; + +pub const OUTPUT_SINK_THREAD_NAME: &str = "sqlbuild-command-output-sink"; + +/// Prefix of each exporter's long-lived delivery thread name. +pub const INVOCATION_THREAD_PREFIX: &str = "sqlbuild-event-exporter-"; + +/// Attribute naming the worker of a long-lived dispatcher thread, for the exit hook. +pub const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; + +/// The longest single wait of a polling delivery loop, as the Python dispatcher polled. +pub const POLL_SECONDS: f64 = 0.01; + +/// The most one dispatch round charges a sink call, so the dispatcher's GIL waits are not counted. +pub const CHARGE_LIMIT_SECONDS: f64 = 2.0 * POLL_SECONDS; + +/// Extra time `shutdown` waits for the dispatcher thread beyond the drain deadline. +pub const JOIN_GRACE_SECONDS: f64 = 0.05; + +/// How long shutdown and interpreter exit wait for each stopped dispatcher thread. +pub const EXIT_JOIN_SECONDS: f64 = 1.0; + +/// The most attempts one exporter's delivery thread holds, at most the queue capacity; more stay queued. +pub const LANE_WINDOW: usize = 64; + +/// The longest a finalizer waits for sink calls still running after shutdown before it runs anyway. +pub const IDLE_WAIT_SECONDS: f64 = 1.0; + +/// No time left before a deadline. +pub const NO_SECONDS: f64 = 0.0; diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/configure_discovered_event_exporters.rs b/crates/sqlbuild-python-base/src/event_exporting/main/configure_discovered_event_exporters.rs new file mode 100644 index 0000000000..ae6bc97957 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/configure_discovered_event_exporters.rs @@ -0,0 +1,12 @@ +//! Configure the active exporter scope from validated discovery inputs. + +use crate::event_exporting::_helpers::scope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Configure the active exporter scope from validated discovery inputs. +#[inline] +pub fn configure_discovered_event_exporters(discovered: &Bound<'_, PyAny>) -> PyResult<()> { + scope::configure_discovered_event_exporters(discovered) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/current_scope.rs b/crates/sqlbuild-python-base/src/event_exporting/main/current_scope.rs new file mode 100644 index 0000000000..198e46eb4c --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/current_scope.rs @@ -0,0 +1,13 @@ +//! The active command-owned exporter scope, or `None`. + +use crate::event_exporting::_helpers::scope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The active command-owned exporter scope, or `None`. +#[inline] +pub fn current_scope(py: Python<'_>) -> PyResult> { + scope::current_scope(py) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/error_type_of.rs b/crates/sqlbuild-python-base/src/event_exporting/main/error_type_of.rs new file mode 100644 index 0000000000..d89fc44fb4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/error_type_of.rs @@ -0,0 +1,11 @@ +//! The sanitized class name of a raised Python exception. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// The sanitized class name of a raised Python exception. +#[inline] +pub fn error_type_of(py: Python<'_>, error: &PyErr) -> String { + values::error_type_of(py, error) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/event_exporter_command_scope.rs b/crates/sqlbuild-python-base/src/event_exporting/main/event_exporter_command_scope.rs new file mode 100644 index 0000000000..22a2796477 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/event_exporter_command_scope.rs @@ -0,0 +1,12 @@ +//! Install one command-owned exporter scope. + +use crate::event_exporting::_helpers::scope; +use crate::event_exporting::models::EventExporterScopeContext; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; + +/// Install one command-owned exporter scope. +#[inline] +pub fn event_exporter_command_scope(scope: Py) -> EventExporterScopeContext { + scope::event_exporter_command_scope(scope) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/export_policy.rs b/crates/sqlbuild-python-base/src/event_exporting/main/export_policy.rs new file mode 100644 index 0000000000..bc70b5aca6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/export_policy.rs @@ -0,0 +1,12 @@ +//! The export policy of a validated event's type; `KeyError` for an uncatalogued type. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_core::event_export::models::ExportPolicy; + +/// The export policy of a validated event's type; `KeyError` for an uncatalogued type. +#[inline] +pub fn export_policy(py: Python<'_>, event_type: &str) -> PyResult { + values::export_policy(py, event_type) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/input_error.rs b/crates/sqlbuild-python-base/src/event_exporting/main/input_error.rs new file mode 100644 index 0000000000..b6f04581c8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/input_error.rs @@ -0,0 +1,11 @@ +//! `EventExporterInputError(message)`. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// `EventExporterInputError(message)`. +#[inline] +pub fn input_error(py: Python<'_>, message: &str) -> PyErr { + values::input_error(py, message) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/mod.rs b/crates/sqlbuild-python-base/src/event_exporting/main/mod.rs new file mode 100644 index 0000000000..d9c23b7f2a --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/mod.rs @@ -0,0 +1,10 @@ +pub mod configure_discovered_event_exporters; +pub mod current_scope; +pub mod error_type_of; +pub mod event_exporter_command_scope; +pub mod export_policy; +pub mod input_error; +pub mod register_scope; +pub mod severity_from; +pub mod severity_object; +pub mod state_error; diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/register_scope.rs b/crates/sqlbuild-python-base/src/event_exporting/main/register_scope.rs new file mode 100644 index 0000000000..797e29e493 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/register_scope.rs @@ -0,0 +1,12 @@ +//! Register the scope functions and classes. + +use crate::event_exporting::_helpers::scope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the scope functions and classes. +#[inline] +pub fn register_scope(module: &Bound<'_, PyModule>) -> PyResult<()> { + scope::register(module) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/severity_from.rs b/crates/sqlbuild-python-base/src/event_exporting/main/severity_from.rs new file mode 100644 index 0000000000..41e03126ba --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/severity_from.rs @@ -0,0 +1,13 @@ +//! An `EventExportSeverity` or its string value, as `EventExportSeverity(value)` accepts it. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::event_export::models::ExportSeverity; + +/// An `EventExportSeverity` or its string value, as `EventExportSeverity(value)` accepts it. +#[inline] +pub fn severity_from(value: &Bound<'_, PyAny>) -> PyResult { + values::severity_from(value) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/severity_object.rs b/crates/sqlbuild-python-base/src/event_exporting/main/severity_object.rs new file mode 100644 index 0000000000..99085079f6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/severity_object.rs @@ -0,0 +1,14 @@ +//! The Python `EventExportSeverity` member of `severity`. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_core::event_export::models::ExportSeverity; + +/// The Python `EventExportSeverity` member of `severity`. +#[inline] +pub fn severity_object(py: Python<'_>, severity: ExportSeverity) -> PyResult> { + values::severity_object(py, severity) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/main/state_error.rs b/crates/sqlbuild-python-base/src/event_exporting/main/state_error.rs new file mode 100644 index 0000000000..a2a7ea9645 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/main/state_error.rs @@ -0,0 +1,11 @@ +//! `EventExporterStateError(message)`. + +use crate::event_exporting::_helpers::values; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// `EventExporterStateError(message)`. +#[inline] +pub fn state_error(py: Python<'_>, message: &str) -> PyErr { + values::state_error(py, message) +} diff --git a/crates/sqlbuild-python-base/src/event_exporting/mod.rs b/crates/sqlbuild-python-base/src/event_exporting/mod.rs new file mode 100644 index 0000000000..b9bca26d03 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/mod.rs @@ -0,0 +1,6 @@ +//! The event exporter command scope, exporter errors, severities and the export policy catalogue. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/event_exporting/models.rs b/crates/sqlbuild-python-base/src/event_exporting/models.rs new file mode 100644 index 0000000000..6815c2e197 --- /dev/null +++ b/crates/sqlbuild-python-base/src/event_exporting/models.rs @@ -0,0 +1,645 @@ +//! Public records of event exporting. + +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::context_variable::context_variable; +use crate::boundary::models::KeywordArguments; +use crate::event_exporting::_helpers::scope::CURRENT_SCOPE; +use crate::event_exporting::_helpers::scope::ProviderSessionCloser; +use crate::event_exporting::_helpers::scope::ScopeState; +use crate::event_exporting::_helpers::scope::bindings; +use crate::event_exporting::_helpers::scope::optional; +use crate::event_exporting::_helpers::scope::providers_missing_secrets; +use crate::event_exporting::_helpers::scope::tuple_or_empty; +use crate::event_exporting::_helpers::values::accounting_repr; +use crate::event_exporting::_helpers::values::bound_exporter_arguments; +use crate::event_exporting::_helpers::values::counts_repr; +use crate::event_exporting::_helpers::values::python_repr; +use crate::event_exporting::_helpers::values::severity_from; +use crate::event_exporting::_helpers::values::severity_object; +use crate::event_exporting::_helpers::values::text_repr; +use crate::event_exporting::constants::SCOPE_CONTEXT_VARIABLE; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::fields::field::field as native_field; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyFrozenSet; +use pyo3::types::PyTuple; +use sqlbuild_core::event_export::constants::EVENT_EXPORT_KINDS; +use sqlbuild_core::event_export::models::ExportSeverity; +use sqlbuild_core::event_export::models::ExportSnapshot; +use sqlbuild_core::event_export::models::ExporterCounts; +use sqlbuild_core::event_export::models::ExporterFilter; +use sqlbuild_host::entries::models::UserCodeHost; +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::MutexGuard; + +/// Frozen attempt counters for one exporter or their aggregate. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventExporterCounts { + pub counts: ExporterCounts, +} + +/// Frozen accounting snapshot for one named exporter. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventExporterAccounting { + pub(crate) exporter_name: String, + pub(crate) counts: ExporterCounts, +} + +/// Final or point-in-time best-effort delivery accounting. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventExportSummary { + pub snapshot: ExportSnapshot, +} + +/// Sanitized exporter failure information safe for diagnostics. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventExporterFailure { + pub exporter_name: String, + pub error_type: String, + pub event_kind: String, + pub event_severity: ExportSeverity, +} + +/// One lifecycle-event sink bound to command-scoped provider instances. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct BoundEventExporter { + pub name: String, + pub function: Py, + pub provider_arguments: Py, + pub filter: ExporterFilter, +} + +#[pymethods] +impl EventExporterCounts { + #[getter] + pub fn accepted(&self) -> u64 { + self.counts.accepted + } + + #[getter] + pub fn filtered(&self) -> u64 { + self.counts.filtered + } + + #[getter] + pub fn delivered(&self) -> u64 { + self.counts.delivered + } + + #[getter] + pub fn failed(&self) -> u64 { + self.counts.failed + } + + #[getter] + pub fn dropped(&self) -> u64 { + self.counts.dropped + } + + pub(crate) fn __repr__(&self) -> String { + counts_repr(&self.counts) + } +} + +#[pymethods] +impl EventExporterAccounting { + #[getter] + pub fn exporter_name(&self) -> &str { + &self.exporter_name + } + + #[getter] + pub fn counts(&self) -> EventExporterCounts { + EventExporterCounts { + counts: self.counts, + } + } + + pub(crate) fn __repr__(&self, py: Python<'_>) -> PyResult { + accounting_repr(py, (&self.exporter_name, &self.counts)) + } +} + +#[pymethods] +impl EventExportSummary { + #[getter] + pub fn aggregate(&self) -> EventExporterCounts { + EventExporterCounts { + counts: self.snapshot.aggregate, + } + } + + #[getter] + pub fn per_exporter<'py>(&self, py: Python<'py>) -> PyResult> { + let items: Vec = self + .snapshot + .per_exporter + .iter() + .map(|(name, counts)| EventExporterAccounting { + exporter_name: name.clone(), + counts: *counts, + }) + .collect(); + PyTuple::new(py, items) + } + + #[getter] + pub fn queue_depth(&self) -> usize { + self.snapshot.queue_depth + } + + #[getter] + pub fn queue_capacity(&self) -> usize { + self.snapshot.queue_capacity + } + + #[getter] + pub fn flush_complete(&self) -> bool { + self.snapshot.flush_complete + } + + #[getter] + pub fn accepted(&self) -> u64 { + self.snapshot.aggregate.accepted + } + + #[getter] + pub fn filtered(&self) -> u64 { + self.snapshot.aggregate.filtered + } + + #[getter] + pub fn delivered(&self) -> u64 { + self.snapshot.aggregate.delivered + } + + #[getter] + pub fn failed(&self) -> u64 { + self.snapshot.aggregate.failed + } + + #[getter] + pub fn dropped(&self) -> u64 { + self.snapshot.aggregate.dropped + } + + pub(crate) fn __repr__(&self, py: Python<'_>) -> PyResult { + let per_exporter: Vec = self + .snapshot + .per_exporter + .iter() + .map(|(name, counts)| accounting_repr(py, (name, counts))) + .collect::>>()?; + let per_exporter: String = match per_exporter.as_slice() { + [single] => format!("({single},)"), + items => format!("({})", items.join(", ")), + }; + Ok(format!( + "EventExportSummary(aggregate={}, per_exporter={per_exporter}, queue_depth={}, \ + queue_capacity={}, flush_complete={})", + counts_repr(&self.snapshot.aggregate), + self.snapshot.queue_depth, + self.snapshot.queue_capacity, + if self.snapshot.flush_complete { + "True" + } else { + "False" + } + )) + } +} + +#[pymethods] +impl EventExporterFailure { + #[getter] + pub fn exporter_name(&self) -> &str { + &self.exporter_name + } + + #[getter] + pub fn error_type(&self) -> &str { + &self.error_type + } + + #[getter] + pub fn event_kind(&self) -> &str { + &self.event_kind + } + + #[getter] + pub fn event_severity<'py>(&self, py: Python<'py>) -> PyResult> { + severity_object(py, self.event_severity) + } + + pub(crate) fn __repr__(&self, py: Python<'_>) -> PyResult { + Ok(format!( + "EventExporterFailure(exporter_name={}, error_type={}, event_kind={}, event_severity={})", + text_repr(py, &self.exporter_name)?, + text_repr(py, &self.error_type)?, + text_repr(py, &self.event_kind)?, + python_repr(&severity_object(py, self.event_severity)?)? + )) + } +} + +#[pymethods] +impl BoundEventExporter { + #[new] + #[pyo3(signature = (*arguments, **options))] + pub(crate) fn new( + arguments: &Bound<'_, PyTuple>, + options: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let [ + name, + function, + provider_arguments, + event_kinds, + min_severity, + ] = bound_exporter_arguments(arguments, options)?; + let (Some(name), Some(function), Some(provider_arguments)) = + (name, function, provider_arguments) + else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "BoundEventExporter() missing required arguments", + )); + }; + Self::create( + name.extract()?, + (function.unbind(), provider_arguments.unbind()), + (event_kinds.as_ref(), min_severity.as_ref()), + ) + } + + #[getter] + pub fn name(&self) -> &str { + &self.name + } + + #[getter] + pub fn function(&self, py: Python<'_>) -> Py { + self.function.clone_ref(py) + } + + #[getter] + pub fn provider_arguments(&self, py: Python<'_>) -> Py { + self.provider_arguments.clone_ref(py) + } + + #[getter] + pub fn event_kinds<'py>(&self, py: Python<'py>) -> PyResult> { + PyFrozenSet::new(py, self.filter.event_kinds.iter()) + } + + #[getter] + pub fn min_severity<'py>(&self, py: Python<'py>) -> PyResult> { + severity_object(py, self.filter.min_severity) + } +} + +impl BoundEventExporter { + /// A sink bound to `provider_arguments`, filtered by `event_kinds` and `min_severity` when given. + pub fn create( + name: String, + (function, provider_arguments): (Py, Py), + (event_kinds, min_severity): (Option<&Bound<'_, PyAny>>, Option<&Bound<'_, PyAny>>), + ) -> PyResult { + let event_kinds = match event_kinds { + Some(kinds) => kinds + .try_iter()? + .map(|kind| kind.and_then(|kind| kind.extract::())) + .collect::>()?, + None => EVENT_EXPORT_KINDS + .iter() + .map(|kind| (*kind).to_owned()) + .collect(), + }; + let min_severity: ExportSeverity = match min_severity { + Some(severity) => severity_from(severity)?, + None => ExportSeverity::Debug, + }; + Ok(Self { + name, + function, + provider_arguments, + filter: ExporterFilter { + event_kinds, + min_severity, + }, + }) + } +} + +impl EventExportSummary { + /// A Python summary of `snapshot`. + pub fn object(py: Python<'_>, snapshot: ExportSnapshot) -> PyResult> { + Py::new(py, Self { snapshot }) + } +} + +/// Bind discovered typed sinks and own their command-scoped provider session. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct EventExporterCommandScope { + pub(crate) dispatcher: Py, + pub(crate) state: Mutex, +} + +/// Installs one command-owned exporter scope while entered. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct EventExporterScopeContext { + pub(crate) scope: Py, + pub(crate) token: Mutex>>, +} + +impl EventExporterCommandScope { + pub(crate) fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned")) + } + + /// Bind discovered sinks over one command-scoped provider session, once. + pub(crate) fn bind_extensions<'py>( + &self, + py: Python<'py>, + project_dir: Option>, + (providers, event_exporters, command_output_sinks): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ), + ) -> PyResult<()> { + { + let mut state = self.locked()?; + if state.configured { + return Ok(()); + } + state.configured = true; + } + let project_dir: Option> = project_dir + .map(|project_dir| project_dir.call_method0("resolve")) + .transpose()?; + { + let mut state = self.locked()?; + state.project_dir = project_dir.map(Bound::unbind); + state.providers = Some(providers.clone().unbind()); + state.event_exporters = Some(event_exporters.clone().unbind()); + state.command_output_sinks = Some(command_output_sinks.clone().unbind()); + } + if !event_exporters.is_truthy()? && !command_output_sinks.is_truthy()? { + let empty = PyTuple::empty(py).into_any(); + return self.bind_dispatcher(py, (empty.clone(), empty)); + } + let missing: HashMap> = providers_missing_secrets(&providers)?; + let session: Bound<'_, PyAny> = call_with( + &host_entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + session.call_method0("hold_open")?; + let (exporters, sinks) = match bindings( + &session, + (&event_exporters, &command_output_sinks), + &missing, + ) { + Ok(bound) => bound, + Err(error) => { + session.call_method0("release")?; + call_with( + &native_field(&session, "close")?, + &[("force", PyBool::new(py, true).to_owned().into_any())], + )?; + return Err(error); + } + }; + self.locked()?.provider_session = Some(session.unbind()); + self.bind_dispatcher(py, (exporters.into_any(), sinks.into_any())) + } + + pub(crate) fn bind_dispatcher( + &self, + py: Python<'_>, + (exporters, sinks): (Bound<'_, PyAny>, Bound<'_, PyAny>), + ) -> PyResult<()> { + let dispatcher: &Bound<'_, PyAny> = self.dispatcher.bind(py); + dispatcher.call_method1("bind", (exporters,))?; + dispatcher.call_method1("bind_command_output_sinks", (sinks,))?; + Ok(()) + } +} + +#[pymethods] +impl EventExporterCommandScope { + #[new] + #[pyo3(signature = (*, dispatcher))] + pub fn new(dispatcher: Py) -> Self { + Self { + dispatcher, + state: Mutex::new(ScopeState { + configured: false, + provider_close_registered: false, + provider_session: None, + project_dir: None, + providers: None, + event_exporters: None, + command_output_sinks: None, + }), + } + } + + #[getter] + pub fn dispatcher(&self, py: Python<'_>) -> Py { + self.dispatcher.clone_ref(py) + } + + #[getter] + pub fn provider_session(&self, py: Python<'_>) -> PyResult> { + Ok(optional(py, self.locked()?.provider_session.as_ref())) + } + + #[getter] + pub fn project_dir(&self, py: Python<'_>) -> PyResult> { + Ok(optional(py, self.locked()?.project_dir.as_ref())) + } + + #[getter] + pub fn providers(&self, py: Python<'_>) -> PyResult> { + Ok(tuple_or_empty(py, self.locked()?.providers.as_ref())) + } + + #[getter] + pub fn event_exporters(&self, py: Python<'_>) -> PyResult> { + Ok(tuple_or_empty(py, self.locked()?.event_exporters.as_ref())) + } + + #[getter] + pub fn command_output_sinks(&self, py: Python<'_>) -> PyResult> { + Ok(tuple_or_empty( + py, + self.locked()?.command_output_sinks.as_ref(), + )) + } + + /// Bind once after normal project discovery has validated declarations. + pub(crate) fn configure( + &self, + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.bind_extensions( + py, + None, + ( + native_field(discovered_inputs, "providers")?, + native_field(discovered_inputs, "event_exporters")?, + native_field(discovered_inputs, "command_output_sinks")?, + ), + ) + } + + /// Bind declarations discovered at the outer command startup boundary. + #[pyo3(signature = (**options))] + pub(crate) fn configure_extensions( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult<()> { + let options: KeywordArguments<'_> = KeywordArguments::read( + py, + "configure_extensions", + options, + ( + &["providers", "event_exporters"], + &["project_dir", "command_output_sinks"], + ), + )?; + let project_dir: Bound<'_, PyAny> = options.value("project_dir")?; + let command_output_sinks: Bound<'_, PyAny> = options.value("command_output_sinks")?; + let command_output_sinks: Bound<'_, PyAny> = if command_output_sinks.is_none() { + PyTuple::empty(py).into_any() + } else { + command_output_sinks + }; + self.bind_extensions( + py, + (!project_dir.is_none()).then_some(project_dir), + ( + options.value("providers")?, + options.value("event_exporters")?, + command_output_sinks, + ), + ) + } + + /// Return startup declarations only for the same project root. + #[pyo3(signature = (*, project_dir))] + pub(crate) fn cached_extensions( + &self, + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, + ) -> PyResult> { + let (own, providers, exporters, sinks) = { + let state = self.locked()?; + let Some(own) = state.project_dir.as_ref() else { + return Ok(py.None()); + }; + ( + own.clone_ref(py), + optional(py, state.providers.as_ref()), + optional(py, state.event_exporters.as_ref()), + optional(py, state.command_output_sinks.as_ref()), + ) + }; + if project_dir.call_method0("resolve")?.ne(own.bind(py))? { + return Ok(py.None()); + } + Ok(PyTuple::new(py, [providers, exporters, sinks])? + .into_any() + .unbind()) + } + + /// Flush sink delivery before reverse-order provider teardown. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult> { + let summary: Py = self.dispatcher.bind(py).call_method0("shutdown")?.unbind(); + let session: Py = { + let mut state = self.locked()?; + let Some(session) = state + .provider_session + .as_ref() + .map(|session| session.clone_ref(py)) + else { + return Ok(summary); + }; + if state.provider_close_registered { + return Ok(summary); + } + state.provider_close_registered = true; + session + }; + let closer: Py = Py::new(py, ProviderSessionCloser { session })?; + self.dispatcher + .bind(py) + .call_method1("finalize_when_idle", (closer,))?; + Ok(summary) + } + + /// Reuse bound sink functions and provider instances for output records. + pub(crate) fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { + self.dispatcher + .bind(py) + .call_method1("export_output", (records,))?; + Ok(()) + } +} + +#[pymethods] +impl EventExporterScopeContext { + pub(crate) fn __enter__(&self, py: Python<'_>) -> PyResult> { + let token = context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? + .call_method1("set", (self.scope.bind(py),))?; + *self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? = + Some(token.unbind()); + Ok(self.scope.clone_ref(py)) + } + + #[pyo3(signature = (*_exc_info))] + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _exc_info: &Bound<'_, PyTuple>, + ) -> PyResult { + let token = self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? + .take(); + if let Some(token) = token { + context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? + .call_method1("reset", (token,))?; + } + Ok(false) + } +} diff --git a/crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs b/crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs new file mode 100644 index 0000000000..a8e2fc780b --- /dev/null +++ b/crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs @@ -0,0 +1,157 @@ +//! Encode a frontier object graph as canonical JSON, sharing large subtrees by digest. + +use crate::boundary::main::utf16_units::utf16_units; + +use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyString, PyStringMethods, PyTuple}; + +use sqlbuild_cache::hosted_shapes::main::captured_shape_name::captured_shape_name; +use sqlbuild_cache::stage_capture::constants::SKIPPED_SLOTS; +use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_items::mapping_items; + +/// An instance's attributes, by name in first-set order. +type Attributes<'py> = Vec<(String, Bound<'py, PyAny>)>; + +/// Nesting at which encoding stops, as Python's recursion limit stops the Python encoder. +pub(crate) const MAX_DEPTH: usize = 900; +pub(crate) const RECURSION_MESSAGE: &str = "maximum recursion depth exceeded"; +const SURROGATE_MESSAGE: &str = "'utf-8' codec can't encode characters: surrogates not allowed"; + +/// The Python error a capture serializer failure stands for. +pub(crate) fn emit_error(error: JsonEmitError) -> PyErr { + match error { + JsonEmitError::LoneSurrogate => PyUnicodeEncodeError::new_err(SURROGATE_MESSAGE), + JsonEmitError::NestingTooDeep => PyRecursionError::new_err(RECURSION_MESSAGE), + other => PyValueError::new_err(format!("{other:?}")), + } +} + +/// `{"key": value}`. +pub(crate) fn tagged(key: &str, value: JsonValue) -> JsonValue { + JsonValue::Object(vec![(key.to_owned(), value)]) +} + +/// Set `key` in `entries`, keeping the position of a key already present, as a dict does. +pub(crate) fn with_entry( + mut entries: Vec<(String, T)>, + key: String, + value: T, +) -> Vec<(String, T)> { + match entries.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => entries.push((key, value)), + } + entries +} + +/// `table[name]` of a per-type table. +pub(crate) fn lookup<'a, T: Copy>(table: &'a [(&'a str, T)], name: &str) -> Option { + table + .iter() + .find(|(type_name, _)| *type_name == name) + .map(|(_, value)| *value) +} + +/// The `module:qualname` a captured class is written as, natively hosted shapes keeping their Python name. +pub(crate) fn captured_type_name(class: &Bound<'_, PyAny>) -> PyResult { + let name: String = qualified_name(class)?; + Ok(captured_shape_name(&name).map_or(name, str::to_owned)) +} + +/// `module:qualname` of a callable or class, as Python's `qualified_name` writes it. +pub(crate) fn qualified_name(value: &Bound<'_, PyAny>) -> PyResult { + let module: Bound<'_, PyAny> = attribute_or_none(value, "__module__")?; + let mut name: Bound<'_, PyAny> = attribute_or_none(value, "__qualname__")?; + if !name.is_truthy()? { + name = attribute_or_none(value, "__name__")?; + } + if name.is_none() { + return qualified_name(&value.get_type().into_any()); + } + let name: String = name.str()?.to_str()?.to_owned(); + match module.downcast::() { + Ok(module) => Ok(format!("{}:{name}", module.to_str()?)), + Err(_) => Ok(name), + } +} + +/// `getattr(value, name, None)`. +pub(crate) fn attribute_or_none<'py>( + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + let py: Python<'py> = value.py(); + match native_field(value, name) { + Ok(attribute) => Ok(attribute), + Err(error) if error.is_instance_of::(py) => { + Ok(py.None().into_bound(py)) + } + Err(error) => Err(error), + } +} + +/// A Python `str` as JSON text; one holding a lone surrogate as its UTF-16 code units. +pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(text) => Ok(JsonValue::String(text.to_owned())), + Err(_) => utf16_units(text).map(JsonValue::Utf16), + } +} + +/// `items` sorted by their masked, key-sorted JSON text, as Python's `sorted` orders them. +pub(crate) fn sorted_items(items: Vec) -> PyResult> { + let mut keyed: Vec<(String, JsonValue)> = items + .into_iter() + .map(|item| Ok((capture_sort_key(&item).map_err(emit_error)?, item))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, item)| item).collect()) +} + +/// `_instance_attributes`: the instance `__dict__` and the slots set on `value`, or `None`. +pub(crate) fn instance_attributes<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult>> { + let instance_dict: Bound<'py, PyAny> = attribute_or_none(value, "__dict__")?; + let has_dict: bool = instance_dict.is_exact_instance_of::() + || instance_dict.is_instance_of::(); + let mut attributes: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + if has_dict { + for item in mapping_items(&instance_dict)?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let name: String = item.get_item(0)?.str()?.to_str()?.to_owned(); + attributes = with_entry(attributes, name, item.get_item(1)?); + } + } + for cls in value.get_type().getattr("__mro__")?.try_iter()? { + let slots: Bound<'py, PyAny> = cls? + .getattr("__dict__")? + .call_method1("get", ("__slots__", PyTuple::empty(value.py())))?; + let names: Vec> = if slots.is_instance_of::() { + vec![slots] + } else { + slots.try_iter()?.collect::>()? + }; + for name in names { + let Ok(text) = name.downcast::() else { + continue; + }; + let text: String = text.to_str()?.to_owned(); + if SKIPPED_SLOTS.contains(&text.as_str()) || !value.hasattr(text.as_str())? { + continue; + } + let item: Bound<'py, PyAny> = native_field(value, text.as_str())?; + attributes = with_entry(attributes, text, item); + } + } + if attributes.is_empty() && !has_dict { + return Ok(None); + } + Ok(Some(attributes)) +} diff --git a/crates/sqlbuild-python-base/src/frontier/_helpers/mod.rs b/crates/sqlbuild-python-base/src/frontier/_helpers/mod.rs new file mode 100644 index 0000000000..6fb95df55c --- /dev/null +++ b/crates/sqlbuild-python-base/src/frontier/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod encoder; +pub(crate) mod python_types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs b/crates/sqlbuild-python-base/src/frontier/_helpers/python_types.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs rename to crates/sqlbuild-python-base/src/frontier/_helpers/python_types.rs index 2a47cae481..922973a755 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs +++ b/crates/sqlbuild-python-base/src/frontier/_helpers/python_types.rs @@ -5,11 +5,12 @@ use pyo3::types::{PyBytes, PyBytesMethods, PyStringMethods, PyTuple}; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; -use crate::bindings::_helpers::frontier::encoder::qualified_name; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::boundary::main::imported::imported; +use crate::frontier::_helpers::encoder::qualified_name; +use crate::records::main::fields::field::field as native_field; /// The Python classes and functions the encoder checks values against. +#[derive(Debug)] pub(crate) struct PythonTypes<'py> { pub(crate) enumeration: Bound<'py, PyAny>, pub(crate) mapping: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-base/src/frontier/main/emit_error.rs b/crates/sqlbuild-python-base/src/frontier/main/emit_error.rs new file mode 100644 index 0000000000..2e0abdda41 --- /dev/null +++ b/crates/sqlbuild-python-base/src/frontier/main/emit_error.rs @@ -0,0 +1,11 @@ +//! The Python error a capture serializer failure stands for. + +use crate::frontier::_helpers::encoder; +use pyo3::prelude::PyErr; +use sqlbuild_core::json::errors::JsonEmitError; + +/// The Python error a capture serializer failure stands for. +#[inline] +pub fn emit_error(error: JsonEmitError) -> PyErr { + encoder::emit_error(error) +} diff --git a/crates/sqlbuild-python-base/src/frontier/main/mod.rs b/crates/sqlbuild-python-base/src/frontier/main/mod.rs new file mode 100644 index 0000000000..5d43b1b0c6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/frontier/main/mod.rs @@ -0,0 +1 @@ +pub mod emit_error; diff --git a/crates/sqlbuild-python-base/src/frontier/mod.rs b/crates/sqlbuild-python-base/src/frontier/mod.rs new file mode 100644 index 0000000000..4705d9ba6f --- /dev/null +++ b/crates/sqlbuild-python-base/src/frontier/mod.rs @@ -0,0 +1,5 @@ +//! Canonical JSON of frontier object graphs and the Python types the encoder distinguishes. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs b/crates/sqlbuild-python-base/src/frontier/models.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs rename to crates/sqlbuild-python-base/src/frontier/models.rs index 80fdb7908a..1e654b0749 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs +++ b/crates/sqlbuild-python-base/src/frontier/models.rs @@ -1,139 +1,67 @@ -//! Encode a frontier object graph as canonical JSON, sharing large subtrees by digest. +//! Public records of frontier. -use std::collections::{HashMap, HashSet}; - -use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{ - PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyFloat, PyInt, PyList, PyString, - PyStringMethods, PyTuple, -}; - -use sqlbuild_cache::hosted_shapes::main::captured_shape_name::captured_shape_name; -use sqlbuild_cache::stage_capture::constants::{ - CANONICAL_TYPES, DECODED_SEQUENCES, OMITTED_ATTRIBUTES, PROJECTED_ATTRIBUTES, RESERVED_KEYS, - SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, UNORDERED_ATTRIBUTES, -}; -use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; +use crate::boundary::main::imported::imported; +use crate::frontier::_helpers::encoder::MAX_DEPTH; +use crate::frontier::_helpers::encoder::RECURSION_MESSAGE; +use crate::frontier::_helpers::encoder::attribute_or_none; +use crate::frontier::_helpers::encoder::captured_type_name; +use crate::frontier::_helpers::encoder::emit_error; +use crate::frontier::_helpers::encoder::instance_attributes; +use crate::frontier::_helpers::encoder::lookup; +use crate::frontier::_helpers::encoder::qualified_name; +use crate::frontier::_helpers::encoder::sorted_items; +use crate::frontier::_helpers::encoder::tagged; +use crate::frontier::_helpers::encoder::text_value; +use crate::frontier::_helpers::encoder::with_entry; +use crate::frontier::_helpers::python_types::PythonTypes; +use crate::records::main::fields::field::field as native_field; +use crate::records::main::fields::mapping_items::mapping_items; +use pyo3::exceptions::PyRecursionError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyBool; +use pyo3::types::PyBoolMethods; +use pyo3::types::PyFloat; +use pyo3::types::PyInt; +use pyo3::types::PyList; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use pyo3::types::PyTuple; +use sqlbuild_cache::stage_capture::constants::CANONICAL_TYPES; +use sqlbuild_cache::stage_capture::constants::DECODED_SEQUENCES; +use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; +use sqlbuild_cache::stage_capture::constants::PROJECTED_ATTRIBUTES; +use sqlbuild_cache::stage_capture::constants::RESERVED_KEYS; +use sqlbuild_cache::stage_capture::constants::SHARED_MARKER; +use sqlbuild_cache::stage_capture::constants::SHARED_MIN_CHARS; +use sqlbuild_cache::stage_capture::constants::UNORDERED_ATTRIBUTES; use sqlbuild_cache::stage_capture::main::compact_capture_json::compact_capture_json; use sqlbuild_cache::stage_capture::main::shared_node_digest::shared_node_digest; use sqlbuild_cache::stage_capture::models::CaptureDocument; -use sqlbuild_core::json::errors::JsonEmitError; -use sqlbuild_core::json::models::{JsonInteger, JsonValue}; - -use crate::bindings::_helpers::frontier::python_types::PythonTypes; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; - -/// An instance's attributes, by name in first-set order. -type Attributes<'py> = Vec<(String, Bound<'py, PyAny>)>; - -/// Nesting at which encoding stops, as Python's recursion limit stops the Python encoder. -const MAX_DEPTH: usize = 900; -const RECURSION_MESSAGE: &str = "maximum recursion depth exceeded"; -const SURROGATE_MESSAGE: &str = "'utf-8' codec can't encode characters: surrogates not allowed"; - -/// The Python error a capture serializer failure stands for. -pub(crate) fn emit_error(error: JsonEmitError) -> PyErr { - match error { - JsonEmitError::LoneSurrogate => PyUnicodeEncodeError::new_err(SURROGATE_MESSAGE), - JsonEmitError::NestingTooDeep => PyRecursionError::new_err(RECURSION_MESSAGE), - other => PyValueError::new_err(format!("{other:?}")), - } -} - -/// `{"key": value}`. -fn tagged(key: &str, value: JsonValue) -> JsonValue { - JsonValue::Object(vec![(key.to_owned(), value)]) -} - -/// Set `key` in `entries`, keeping the position of a key already present, as a dict does. -fn with_entry(mut entries: Vec<(String, T)>, key: String, value: T) -> Vec<(String, T)> { - match entries.iter_mut().find(|(existing, _)| *existing == key) { - Some(entry) => entry.1 = value, - None => entries.push((key, value)), - } - entries -} - -/// `table[name]` of a per-type table. -fn lookup<'a, T: Copy>(table: &'a [(&'a str, T)], name: &str) -> Option { - table - .iter() - .find(|(type_name, _)| *type_name == name) - .map(|(_, value)| *value) -} - -/// The `module:qualname` a captured class is written as, natively hosted shapes keeping their Python name. -fn captured_type_name(class: &Bound<'_, PyAny>) -> PyResult { - let name: String = qualified_name(class)?; - Ok(captured_shape_name(&name).map_or(name, str::to_owned)) -} - -/// `module:qualname` of a callable or class, as Python's `qualified_name` writes it. -pub(crate) fn qualified_name(value: &Bound<'_, PyAny>) -> PyResult { - let module: Bound<'_, PyAny> = attribute_or_none(value, "__module__")?; - let mut name: Bound<'_, PyAny> = attribute_or_none(value, "__qualname__")?; - if !name.is_truthy()? { - name = attribute_or_none(value, "__name__")?; - } - if name.is_none() { - return qualified_name(&value.get_type().into_any()); - } - let name: String = name.str()?.to_str()?.to_owned(); - match module.downcast::() { - Ok(module) => Ok(format!("{}:{name}", module.to_str()?)), - Err(_) => Ok(name), - } -} - -/// `getattr(value, name, None)`. -fn attribute_or_none<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { - let py: Python<'py> = value.py(); - match native_field(value, name) { - Ok(attribute) => Ok(attribute), - Err(error) if error.is_instance_of::(py) => { - Ok(py.None().into_bound(py)) - } - Err(error) => Err(error), - } -} - -/// A Python `str` as JSON text; one holding a lone surrogate as its UTF-16 code units. -fn text_value(text: &Bound<'_, PyString>) -> PyResult { - if let Ok(text) = text.to_str() { - return Ok(JsonValue::String(text.to_owned())); - } - let encoded: Bound<'_, PyBytes> = text - .call_method1("encode", ("utf-16-le", "surrogatepass"))? - .downcast_into::()?; - Ok(JsonValue::Utf16( - encoded - .as_bytes() - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) - .collect(), - )) -} +use sqlbuild_core::json::models::JsonInteger; +use sqlbuild_core::json::models::JsonValue; +use std::collections::HashMap; +use std::collections::HashSet; /// The capture encoder with the document its shared nodes are written to. -pub(crate) struct CaptureEncoder<'py> { - types: PythonTypes<'py>, - active: HashSet, - cycles: usize, - encoded: HashMap, JsonValue)>, - digests: HashSet, - depth: usize, - pub(crate) document: CaptureDocument, +#[derive(Debug)] +pub struct CaptureEncoder<'py> { + pub(crate) types: PythonTypes<'py>, + pub(crate) active: HashSet, + pub(crate) cycles: usize, + pub(crate) encoded: HashMap, JsonValue)>, + pub(crate) digests: HashSet, + pub(crate) depth: usize, + pub document: CaptureDocument, /// The document text written so far: the shared nodes, as Python streams them. - pub(crate) written: String, + pub written: String, } impl<'py> CaptureEncoder<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { + pub fn new(py: Python<'py>) -> PyResult { Ok(Self { types: PythonTypes::load(py)?, active: HashSet::new(), @@ -147,7 +75,7 @@ impl<'py> CaptureEncoder<'py> { } /// The JSON value of `value`, keeping insertion order and sorting only sets. - pub(crate) fn encode(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + pub fn encode(&mut self, value: &Bound<'py, PyAny>) -> PyResult { if self.depth >= MAX_DEPTH { return Err(PyRecursionError::new_err(RECURSION_MESSAGE)); } @@ -157,7 +85,7 @@ impl<'py> CaptureEncoder<'py> { encoded } - fn encode_value(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + pub(crate) fn encode_value(&mut self, value: &Bound<'py, PyAny>) -> PyResult { if value.is_instance(&self.types.enumeration)? { let name: String = native_field(value, "name")?.str()?.to_str()?.to_owned(); return Ok(tagged( @@ -227,7 +155,7 @@ impl<'py> CaptureEncoder<'py> { } /// The shared-node marker for a large object, array or string, writing the node once. - fn share(&mut self, encoded: JsonValue) -> PyResult { + pub(crate) fn share(&mut self, encoded: JsonValue) -> PyResult { if !matches!( encoded, JsonValue::Object(_) | JsonValue::Array(_) | JsonValue::String(_) | JsonValue::Utf16(_) @@ -246,7 +174,7 @@ impl<'py> CaptureEncoder<'py> { Ok(tagged(SHARED_MARKER, JsonValue::String(digest))) } - fn encode_container(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + pub(crate) fn encode_container(&mut self, value: &Bound<'py, PyAny>) -> PyResult { let type_name: String = captured_type_name(&value.get_type().into_any())?; if DECODED_SEQUENCES.contains(&type_name.as_str()) && value.is_instance(&self.types.sequence)? @@ -288,7 +216,7 @@ impl<'py> CaptureEncoder<'py> { self.encode_instance(value, type_name) } - fn encode_items(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + pub(crate) fn encode_items(&mut self, value: &Bound<'py, PyAny>) -> PyResult { let mut items: Vec = Vec::new(); for item in value.try_iter()? { items.push(self.encode(&item?)?); @@ -296,7 +224,7 @@ impl<'py> CaptureEncoder<'py> { Ok(JsonValue::Array(items)) } - fn encode_dataclass( + pub(crate) fn encode_dataclass( &mut self, value: &Bound<'py, PyAny>, type_name: String, @@ -316,7 +244,7 @@ impl<'py> CaptureEncoder<'py> { Ok(JsonValue::Object(entries)) } - fn encode_mapping(&mut self, value: &Bound<'py, PyAny>) -> PyResult { + pub(crate) fn encode_mapping(&mut self, value: &Bound<'py, PyAny>) -> PyResult { let mut plain: bool = true; for key in value.try_iter()? { if !key?.is_instance_of::() { @@ -349,7 +277,7 @@ impl<'py> CaptureEncoder<'py> { Ok(JsonValue::Object(entries)) } - fn mapping_pairs(&mut self, value: &Bound<'py, PyAny>) -> PyResult> { + pub(crate) fn mapping_pairs(&mut self, value: &Bound<'py, PyAny>) -> PyResult> { let mut pairs: Vec = Vec::new(); for item in mapping_items(value)?.try_iter()? { let item: Bound<'py, PyAny> = item?; @@ -360,7 +288,7 @@ impl<'py> CaptureEncoder<'py> { Ok(pairs) } - fn encode_instance( + pub(crate) fn encode_instance( &mut self, value: &Bound<'py, PyAny>, type_name: String, @@ -402,53 +330,3 @@ impl<'py> CaptureEncoder<'py> { Ok(JsonValue::Object(entries)) } } - -/// `items` sorted by their masked, key-sorted JSON text, as Python's `sorted` orders them. -fn sorted_items(items: Vec) -> PyResult> { - let mut keyed: Vec<(String, JsonValue)> = items - .into_iter() - .map(|item| Ok((capture_sort_key(&item).map_err(emit_error)?, item))) - .collect::>()?; - keyed.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(keyed.into_iter().map(|(_, item)| item).collect()) -} - -/// `_instance_attributes`: the instance `__dict__` and the slots set on `value`, or `None`. -fn instance_attributes<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { - let instance_dict: Bound<'py, PyAny> = attribute_or_none(value, "__dict__")?; - let has_dict: bool = instance_dict.is_exact_instance_of::() - || instance_dict.is_instance_of::(); - let mut attributes: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - if has_dict { - for item in mapping_items(&instance_dict)?.try_iter()? { - let item: Bound<'py, PyAny> = item?; - let name: String = item.get_item(0)?.str()?.to_str()?.to_owned(); - attributes = with_entry(attributes, name, item.get_item(1)?); - } - } - for cls in value.get_type().getattr("__mro__")?.try_iter()? { - let slots: Bound<'py, PyAny> = cls? - .getattr("__dict__")? - .call_method1("get", ("__slots__", PyTuple::empty(value.py())))?; - let names: Vec> = if slots.is_instance_of::() { - vec![slots] - } else { - slots.try_iter()?.collect::>()? - }; - for name in names { - let Ok(text) = name.downcast::() else { - continue; - }; - let text: String = text.to_str()?.to_owned(); - if SKIPPED_SLOTS.contains(&text.as_str()) || !value.hasattr(text.as_str())? { - continue; - } - let item: Bound<'py, PyAny> = native_field(value, text.as_str())?; - attributes = with_entry(attributes, text, item); - } - } - if attributes.is_empty() && !has_dict { - return Ok(None); - } - Ok(Some(attributes)) -} diff --git a/crates/sqlbuild-python-base/src/lib.rs b/crates/sqlbuild-python-base/src/lib.rs index 862f421350..cc330292ef 100644 --- a/crates/sqlbuild-python-base/src/lib.rs +++ b/crates/sqlbuild-python-base/src/lib.rs @@ -3,6 +3,26 @@ #![forbid(unsafe_code)] pub mod boundary; +pub mod build_command; +pub mod build_progress; +pub mod cache; +pub mod cli; +pub mod command_runtime; +pub mod compile_context; +pub mod compile_driver; +pub mod compile_outputs; +pub mod contracts; +pub mod diagnostics; +pub mod event_exporting; +pub mod frontier; pub mod native_host; +pub mod native_threads; +pub mod orchestration_reads; +pub mod planner_identity; +pub mod presentation_text; +pub mod python_values; pub mod records; +pub mod result_output; +pub mod type_system; pub mod types; +pub mod warehouse_access; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/connections.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs rename to crates/sqlbuild-python-base/src/native_host/_helpers/connections.rs index 0bbc963ca8..bcb0d4a856 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/connections.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use crate::native_host::main::host_entry::host_entry; /// The coded error to raise for a failed `adapter.connect`, or the failure itself when coded. pub(crate) fn connection_failure(adapter: &Bound<'_, PyAny>, error: &PyErr) -> PyErr { diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs index dfd6cf90cb..dfa367e8b4 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs @@ -138,3 +138,10 @@ pub(crate) fn within_identity_pass<'py, T>( let identity_pass: Bound<'py, PyAny> = entry(py, UserCodeHost::PythonIdentityPass)?.call0()?; within(&identity_pass, |_| body()) } + +/// The host exception class `id` raised with `message`, or the error looking it up or raising it. +pub(crate) fn host_error(py: Python<'_>, id: impl Into, message: &str) -> PyErr { + entry(py, id) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs new file mode 100644 index 0000000000..ca192c2c4e --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs @@ -0,0 +1,25 @@ +//! Python's default compilation Ctrl+C handler, retained across unraisable callbacks. + +use std::cell::Cell; + +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{PyResult, Python}; + +thread_local! { + pub(crate) static INTERRUPTED: Cell = const { Cell::new(false) }; +} +pub(crate) const SIGINT: i64 = 2; + +/// Whether the default handler observed Ctrl+C during the current command. +pub(crate) fn interrupted() -> bool { + INTERRUPTED.get() +} + +/// Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. +pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { + py.check_signals()?; + if interrupted() { + return Err(PyKeyboardInterrupt::new_err(())); + } + Ok(()) +} diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs index e5c0bc02ed..7af1af0538 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs @@ -1 +1,4 @@ +pub(crate) mod connections; pub(crate) mod host; +pub(crate) mod interrupt; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs rename to crates/sqlbuild-python-base/src/native_host/_helpers/results.rs index 94f7650092..55e05d3c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs @@ -1,5 +1,6 @@ //! Read executor dataclasses once into the host contract. Only opaque payloads retain Python values. +use crate::native_host::types::ReadResult; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -21,14 +22,9 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::types::RecordId; - -/// Extraction at the permanent Python executor edge, including nested result records. -pub(crate) trait ReadResult<'py>: Sized { - fn read(value: &Bound<'py, PyAny>) -> PyResult; -} +use crate::records::main::construct::enum_from_value::enum_from_value; +use crate::records::main::fields::field::field; +use crate::records::types::RecordId; macro_rules! scalar { ($($kind:ty),+ $(,)?) => {$ ( diff --git a/crates/sqlbuild-python-base/src/native_host/constants.rs b/crates/sqlbuild-python-base/src/native_host/constants.rs new file mode 100644 index 0000000000..f80d45b375 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/constants.rs @@ -0,0 +1,29 @@ +//! Public constants of native host. + +/// The calendar's evaluation window is supplied by the host, not a provider. +pub const CALENDAR_WINDOW_PARAMETER: &str = "window"; + +pub const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; + +pub const SOURCE_STALE_CONDITION: &str = "source_stale"; + +/// The sensor context is supplied by the host, not a provider. +pub const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; + +pub const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; + +/// Frames of these modules construct declarations on the author's behalf. +pub const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; + +/// The authoring classes publication imports calendar and sensor modules for. +pub const CALENDAR_CLASS: &str = "Calendar"; + +pub const SENSOR_CLASS: &str = "Sensor"; + +/// Pages one bounded state-history SELECT streams before the scan continues from its cursor. +pub const HISTORY_STREAM_PAGES: usize = 50; + +/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. +pub const ALIAS_PREFIX_TARGET: &str = "alias"; + +pub const ALL_PREFIX_TARGET: &str = "all"; diff --git a/crates/sqlbuild-python-base/src/native_host/main/check_signals.rs b/crates/sqlbuild-python-base/src/native_host/main/check_signals.rs new file mode 100644 index 0000000000..df38b132f0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/main/check_signals.rs @@ -0,0 +1,11 @@ +//! Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. + +use crate::native_host::_helpers::interrupt; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. +#[inline] +pub fn check_signals(py: Python<'_>) -> PyResult<()> { + interrupt::check_signals(py) +} diff --git a/crates/sqlbuild-python-base/src/native_host/main/connection_failure.rs b/crates/sqlbuild-python-base/src/native_host/main/connection_failure.rs new file mode 100644 index 0000000000..4e75c79f8e --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/main/connection_failure.rs @@ -0,0 +1,12 @@ +//! The coded error to raise for a failed `adapter.connect`, or the failure itself when coded. + +use crate::native_host::_helpers::connections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; + +/// The coded error to raise for a failed `adapter.connect`, or the failure itself when coded. +#[inline] +pub fn connection_failure(adapter: &Bound<'_, PyAny>, error: &PyErr) -> PyErr { + connections::connection_failure(adapter, error) +} diff --git a/crates/sqlbuild-python-base/src/native_host/main/host_error.rs b/crates/sqlbuild-python-base/src/native_host/main/host_error.rs new file mode 100644 index 0000000000..b6d9fbe0d3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/main/host_error.rs @@ -0,0 +1,12 @@ +//! The host exception class `id` raised with `message`, or the error looking it up or raising it. + +use crate::native_host::_helpers::host; +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_host::entries::models::HostId; + +/// The host exception class `id` raised with `message`, or the error looking it up or raising it. +#[inline] +pub fn host_error(py: Python<'_>, id: impl Into, message: &str) -> PyErr { + host::host_error(py, id, message) +} diff --git a/crates/sqlbuild-python-base/src/native_host/main/interrupted.rs b/crates/sqlbuild-python-base/src/native_host/main/interrupted.rs new file mode 100644 index 0000000000..b626745df4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/main/interrupted.rs @@ -0,0 +1,9 @@ +//! Whether the default handler observed Ctrl+C during the current command. + +use crate::native_host::_helpers::interrupt; + +/// Whether the default handler observed Ctrl+C during the current command. +#[inline] +pub fn interrupted() -> bool { + interrupt::interrupted() +} diff --git a/crates/sqlbuild-python-base/src/native_host/main/mod.rs b/crates/sqlbuild-python-base/src/native_host/main/mod.rs index 6198f548bc..b30fd0da60 100644 --- a/crates/sqlbuild-python-base/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-base/src/native_host/main/mod.rs @@ -1,5 +1,9 @@ +pub mod check_signals; +pub mod connection_failure; pub mod flush_console; pub mod host_entry; +pub mod host_error; +pub mod interrupted; pub mod log_debug_message; pub mod log_warning_message; pub mod module_installed; diff --git a/crates/sqlbuild-python-base/src/native_host/mod.rs b/crates/sqlbuild-python-base/src/native_host/mod.rs index c954c0691c..3e6cabe3ea 100644 --- a/crates/sqlbuild-python-base/src/native_host/mod.rs +++ b/crates/sqlbuild-python-base/src/native_host/mod.rs @@ -1,4 +1,7 @@ //! Calls into the permanent Python host entries of `sqlbuild._native_host`. pub(crate) mod _helpers; +pub mod constants; pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-base/src/native_host/models.rs b/crates/sqlbuild-python-base/src/native_host/models.rs new file mode 100644 index 0000000000..32544d9016 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/models.rs @@ -0,0 +1,145 @@ +//! Public records of native host. + +use crate::native_host::_helpers::interrupt::INTERRUPTED; +use crate::native_host::_helpers::interrupt::SIGINT; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyCFunction; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyTuple; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +/// Factory results, source locations and the cached public audit tuple are Python-owned edges. +pub struct AuditProjection { + pub custom: Option>>, + pub generated: Vec>, + pub location: Py, + pub records: PyOnceLock>, +} + +/// A supplied constructor and its public family records; native callers retain the family specs. +pub struct FamilyProjection { + pub(crate) class: Py, + pub custom: bool, + pub(crate) records: PyOnceLock>, +} + +impl FamilyProjection { + pub fn new(class: &Bound<'_, PyAny>, custom: bool) -> Self { + Self { + class: class.clone().unbind(), + custom, + records: PyOnceLock::new(), + } + } + + pub fn project<'py>( + &self, + py: Python<'py>, + specs: &[DynamicFamily], + ) -> PyResult> { + self.records + .get_or_try_init(py, || { + let mut records: Vec> = Vec::with_capacity(specs.len()); + for family in specs { + let fields = PyDict::new(py); + fields.set_item("name", &family.name)?; + fields.set_item("pivot_column", &family.pivot_column)?; + fields.set_item("value_column", &family.value_column)?; + fields.set_item("aggregate", &family.aggregate)?; + fields.set_item("type", &family.data_type)?; + fields.set_item("name_pattern", &family.name_pattern)?; + records.push(self.class.bind(py).call((), Some(&fields))?); + } + PyTuple::new(py, records).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } +} + +/// The previous Python handler and the command's replacement, restored on every exit path. +#[derive(Debug)] +pub struct InterruptHandler { + pub(crate) previous: Py, + pub(crate) installed: Py, +} + +impl InterruptHandler { + /// Capture only Python's default main-thread handler; leave embedding applications' handlers alone. + pub fn install(py: Python<'_>) -> PyResult> { + let threading = py.import("threading")?; + if !threading + .call_method0("current_thread")? + .is(threading.call_method0("main_thread")?) + { + return Ok(None); + } + let signal = py.import("signal")?; + let previous = signal.call_method1("getsignal", (SIGINT,))?; + if !previous.is(signal.getattr("default_int_handler")?) { + return Ok(None); + } + let installed = PyCFunction::new_closure( + py, + None, + None, + |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + INTERRUPTED.set(true); + Err(PyKeyboardInterrupt::new_err(())) + }, + )?; + INTERRUPTED.set(false); + signal.call_method1("signal", (SIGINT, &installed))?; + Ok(Some(Self { + previous: previous.unbind(), + installed: installed.into_any().unbind(), + })) + } +} + +impl Drop for InterruptHandler { + fn drop(&mut self) { + Python::attach(|py| { + let restored: PyResult<()> = (|| { + let signal = py.import("signal")?; + if signal + .call_method1("getsignal", (SIGINT,))? + .is(self.installed.bind(py)) + { + signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; + } + Ok(()) + })(); + drop(restored); + }); + INTERRUPTED.set(false); + } +} + +impl std::fmt::Debug for AuditProjection { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AuditProjection") + .field("custom", &self.custom) + .field("generated", &self.generated) + .field("location", &self.location) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for FamilyProjection { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("FamilyProjection") + .field("class", &self.class) + .field("custom", &self.custom) + .finish_non_exhaustive() + } +} diff --git a/crates/sqlbuild-python-base/src/native_host/types.rs b/crates/sqlbuild-python-base/src/native_host/types.rs new file mode 100644 index 0000000000..5d64f25885 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/types.rs @@ -0,0 +1,10 @@ +//! Public traits and type aliases of native host. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Extraction at the permanent Python executor edge, including nested result records. +pub trait ReadResult<'py>: Sized { + fn read(value: &Bound<'py, PyAny>) -> PyResult; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs rename to crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/interruptible_join.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs rename to crates/sqlbuild-python-base/src/native_threads/_helpers/interruptible_join.rs index 29aecc94b9..964871c1b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs +++ b/crates/sqlbuild-python-base/src/native_threads/_helpers/interruptible_join.rs @@ -1,21 +1,16 @@ //! Joining a native thread from Python's main thread without hiding Ctrl+C. +use crate::native_threads::models::Joined; use std::thread::JoinHandle; use std::time::Duration; -use pyo3::prelude::{PyErr, Python}; +use pyo3::prelude::Python; -use crate::bindings::_helpers::native_host::interrupt::check_signals; +use crate::native_host::main::check_signals::check_signals; /// How often a join checks for pending signals such as Ctrl+C. const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); -/// A joined thread's outcome, or the signal raised while waiting for it. -pub(crate) enum Joined { - Finished(std::thread::Result), - Interrupted(PyErr), -} - /// Join `thread` without the GIL; on a signal call `interrupt`, finish the join and return the signal. pub(crate) fn join_interruptibly( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs rename to crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs index b98894c8c2..6f88f92bf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs +++ b/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod constants; pub(crate) mod exit_signal; pub(crate) mod interruptible_join; pub(crate) mod reentrant_lock; diff --git a/crates/sqlbuild-python-base/src/native_threads/_helpers/reentrant_lock.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/reentrant_lock.rs new file mode 100644 index 0000000000..2ae597ae3a --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/_helpers/reentrant_lock.rs @@ -0,0 +1,29 @@ +//! A native reentrant lock whose contended waits release the GIL and honor Python signals. + +use crate::native_threads::models::ReentrantLock; +use std::sync::PoisonError; +use std::thread::ThreadId; +use std::time::Duration; + +pub(crate) const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); +pub(crate) const POISONED: &str = "native reentrant lock is poisoned"; + +#[derive(Debug, Default)] +pub(crate) struct LockState { + pub(crate) owner: Option, + pub(crate) depth: usize, +} + +/// Release one acquisition even when the protected callback raises or panics. +pub(crate) struct LockGuard<'a>(pub(crate) &'a ReentrantLock); + +impl Drop for LockGuard<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock().unwrap_or_else(PoisonError::into_inner); + state.depth -= 1; + if state.depth == 0 { + state.owner = None; + self.0.available.notify_one(); + } + } +} diff --git a/crates/sqlbuild-python-base/src/native_threads/constants.rs b/crates/sqlbuild-python-base/src/native_threads/constants.rs new file mode 100644 index 0000000000..a8ff53f00e --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of native threads. + +/// Marks a thread whose writes command-output capture ignores, such as one running a command-output sink. +pub const OUTPUT_SUPPRESSED_ATTRIBUTE: &str = "_sqlbuild_command_output_suppressed"; diff --git a/crates/sqlbuild-python-base/src/native_threads/main/interpreter_exiting.rs b/crates/sqlbuild-python-base/src/native_threads/main/interpreter_exiting.rs new file mode 100644 index 0000000000..1168c4c623 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/interpreter_exiting.rs @@ -0,0 +1,9 @@ +//! Whether an exit hook has started stopping native daemon threads. + +use crate::native_threads::_helpers::exit_signal; + +/// Whether an exit hook has started stopping native daemon threads. +#[inline] +pub fn interpreter_exiting() -> bool { + exit_signal::interpreter_exiting() +} diff --git a/crates/sqlbuild-python-base/src/native_threads/main/join_interruptibly.rs b/crates/sqlbuild-python-base/src/native_threads/main/join_interruptibly.rs new file mode 100644 index 0000000000..608fa0c69f --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/join_interruptibly.rs @@ -0,0 +1,16 @@ +//! Join `thread` without the GIL; on a signal call `interrupt`, finish the join and return the signal. + +use crate::native_threads::_helpers::interruptible_join; +use crate::native_threads::models::Joined; +use pyo3::prelude::Python; +use std::thread::JoinHandle; + +/// Join `thread` without the GIL; on a signal call `interrupt`, finish the join and return the signal. +#[inline] +pub fn join_interruptibly( + py: Python<'_>, + thread: JoinHandle, + interrupt: impl FnOnce(), +) -> Joined { + interruptible_join::join_interruptibly(py, thread, interrupt) +} diff --git a/crates/sqlbuild-python-base/src/native_threads/main/mod.rs b/crates/sqlbuild-python-base/src/native_threads/main/mod.rs new file mode 100644 index 0000000000..fcc7ac3c20 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/mod.rs @@ -0,0 +1,3 @@ +pub mod interpreter_exiting; +pub mod join_interruptibly; +pub mod signal_interpreter_exit; diff --git a/crates/sqlbuild-python-base/src/native_threads/main/signal_interpreter_exit.rs b/crates/sqlbuild-python-base/src/native_threads/main/signal_interpreter_exit.rs new file mode 100644 index 0000000000..26ece7ea08 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/signal_interpreter_exit.rs @@ -0,0 +1,9 @@ +//! Tell every native wait to stop waiting on Python work that may outlive the interpreter. + +use crate::native_threads::_helpers::exit_signal; + +/// Tell every native wait to stop waiting on Python work that may outlive the interpreter. +#[inline] +pub fn signal_interpreter_exit() { + exit_signal::signal_interpreter_exit() +} diff --git a/crates/sqlbuild-python-base/src/native_threads/mod.rs b/crates/sqlbuild-python-base/src/native_threads/mod.rs new file mode 100644 index 0000000000..6958d2c544 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/mod.rs @@ -0,0 +1,6 @@ +//! Native threads and locks that cooperate with Python signals and interpreter exit. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/native_threads/models.rs b/crates/sqlbuild-python-base/src/native_threads/models.rs new file mode 100644 index 0000000000..b5a07a5d3f --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/models.rs @@ -0,0 +1,57 @@ +//! Public records of native threads. + +use crate::native_threads::_helpers::reentrant_lock::LockGuard; +use crate::native_threads::_helpers::reentrant_lock::LockState; +use crate::native_threads::_helpers::reentrant_lock::POISONED; +use crate::native_threads::_helpers::reentrant_lock::SIGNAL_POLL_INTERVAL; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::sync::Condvar; +use std::sync::Mutex; +use std::thread::ThreadId; +use std::thread::current; + +/// A joined thread's outcome, or the signal raised while waiting for it. +#[derive(Debug)] +pub enum Joined { + Finished(std::thread::Result), + Interrupted(PyErr), +} + +/// Serialize native operations that may reenter through Python callbacks. +#[derive(Debug, Default)] +pub struct ReentrantLock { + pub(crate) state: Mutex, + pub(crate) available: Condvar, +} + +impl ReentrantLock { + /// Run `body` under the lock, checking pending Python signals while another thread owns it. + pub fn with_lock(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + while !py + .detach(|| self.acquire_or_wait()) + .map_err(PyRuntimeError::new_err)? + { + py.check_signals()?; + } + let _guard = LockGuard(self); + body() + } + + pub(crate) fn acquire_or_wait(&self) -> Result { + let owner: ThreadId = current().id(); + let mut state = self.state.lock().map_err(|_| POISONED)?; + if state.owner.is_none() || state.owner == Some(owner) { + state.owner = Some(owner); + state.depth += 1; + return Ok(true); + } + let (_state, _) = self + .available + .wait_timeout(state, SIGNAL_POLL_INTERVAL) + .map_err(|_| POISONED)?; + Ok(false) + } +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/mod.rs b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/mod.rs new file mode 100644 index 0000000000..159c07a075 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod orchestration_declarations; +pub(crate) mod orchestration_environment; +pub(crate) mod provider_settings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs rename to crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs index 95cdeaec2c..52fed366ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs @@ -1,58 +1,27 @@ //! The public authoring records enter native compilation here, never scheduler execution. +use crate::orchestration_reads::models::ImportedDeclarations; +use crate::orchestration_reads::models::RunControls; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyModuleMethods; +use pyo3::wrap_pyfunction; use std::collections::{BTreeMap, HashSet}; use std::path::{Component, Path}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use pyo3::{FromPyObject, pyfunction}; use serde_json::Value; use sqlbuild_discovery::orchestration_files::models::{self as files, RunFile}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap, RunResources}; -use crate::bindings::_helpers::native_host::constants::{ - AUTHORING_MODULE, CALENDAR_CLASS, CONSTRUCTING_MODULES, SENSOR_CLASS, -}; -use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; - -#[derive(FromPyObject)] -pub(crate) struct AuthoredSensorCondition { - pub(crate) kind: String, - #[pyo3(default)] - pub(crate) kinds: Vec, - #[pyo3(default)] - pub(crate) job: Option, - #[pyo3(default)] - pub(crate) executed_successfully: Option>, - #[pyo3(default)] - pub(crate) select: Option, - #[pyo3(default, from_py_with = authored_setting)] - pub(crate) after_seconds: Value, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredSensor { - pub(crate) name: String, - pub(crate) description: String, - pub(crate) when: Option, - pub(crate) function: Option>, - pub(crate) run: String, - #[pyo3(from_py_with = authored_setting)] - pub(crate) minimum_interval_seconds: Value, - pub(crate) default_status: String, - pub(crate) targets: Vec, - #[pyo3(attribute("_declared_at"))] - pub(crate) declared_at: Option<(String, u32)>, -} - -/// A job's or group's owner, overlap and failure policy, as publication records them. -pub(crate) struct RunControls { - pub(crate) owner: Option, - pub(crate) overlap: Overlap, - pub(crate) on_failure: Option, -} +use crate::native_host::constants::AUTHORING_MODULE; +use crate::native_host::constants::CALENDAR_CLASS; +use crate::native_host::constants::CONSTRUCTING_MODULES; +use crate::native_host::constants::SENSOR_CLASS; +use crate::native_host::main::host_entry::host_entry as entry; /// The file and line of the author's code constructing a declaration; `None` for code without a source file. #[pyfunction] @@ -150,13 +119,6 @@ pub(crate) fn run_resources( Ok(resources) } -/// The calendars, by name with their descriptions, and the sensors that importing the modules defines. -pub(crate) struct ImportedDeclarations<'py> { - pub(crate) calendars: BTreeMap>, - pub(crate) calendar_descriptions: BTreeMap, - pub(crate) sensors: Vec, -} - /// Sort imported declarations into calendars and sensors, refusing a calendar name defined twice. pub(crate) fn imported_declarations<'py>( py: Python<'py>, @@ -223,3 +185,8 @@ pub(crate) fn module_declarations<'py>( } Ok(found) } + +/// Register `declaration_site` on the native module. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declaration_site, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_environment.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs rename to crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_environment.rs index 26cfa2bf67..c3862ca729 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs +++ b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_environment.rs @@ -10,12 +10,16 @@ use sqlbuild_discovery::tree::main::rglob::rglob; use sqlbuild_discovery::tree::models::ProjectTree; /// A lockfile binds declared dependencies; without one, bind installed distribution versions. -pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { +pub(crate) fn environment_hash( + py: Python<'_>, + project_dir: &Path, + sqlbuild_version: &str, +) -> PyResult { let mut hash = Sha256::new(); let version = py.version_info(); hash.update(format!( "sqlbuild:{}\npython:{}.{}.{}{}\n", - env!("CARGO_PKG_VERSION"), + sqlbuild_version, version.major, version.minor, version.patch, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/provider_settings.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs rename to crates/sqlbuild-python-base/src/orchestration_reads/_helpers/provider_settings.rs index 6b0831c671..a7d43a6556 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs +++ b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/provider_settings.rs @@ -1,29 +1,22 @@ //! A provider class's declared settings and whether the environment sets its required secrets. +use crate::orchestration_reads::models::ProviderState; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use serde_json::{Value, json}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::native_host::constants::{ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::imported::imported; +use crate::native_host::constants::ALIAS_PREFIX_TARGET; +use crate::native_host::constants::ALL_PREFIX_TARGET; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::fields::mapping_get_or::mapping_get_or; const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); const MISSING_ERROR_TYPE: &str = "missing"; -/// How a provider class instantiates from the environment. -pub(crate) enum ProviderState<'py> { - Ready(Bound<'py, PyAny>), - /// Only required secret settings are unset: their environment variables, sorted. - MissingSecrets(Vec), - /// Another setting is invalid: pydantic's validation error. - Invalid(PyErr), -} - /// The class's description and declared setting fields, read without any setting value. pub(crate) fn provider_class_settings(provider_class: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = provider_class.py(); diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/callback_code_hash.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/callback_code_hash.rs new file mode 100644 index 0000000000..6f6ff18d4c --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/callback_code_hash.rs @@ -0,0 +1,12 @@ +//! Include the callable's code even for public in-memory declaration compilation. + +use crate::orchestration_reads::_helpers::orchestration_environment; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Include the callable's code even for public in-memory declaration compilation. +#[inline] +pub fn callback_code_hash(function: &Bound<'_, PyAny>, sources: &str) -> PyResult { + orchestration_environment::callback_code_hash(function, sources) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/declared_at.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/declared_at.rs new file mode 100644 index 0000000000..9eb81c002c --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/declared_at.rs @@ -0,0 +1,15 @@ +//! A declaration's file relative to the project; code outside the project keeps its location private. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use pyo3::prelude::PyResult; +use sqlbuild_orchestration::declarations::models::DeclaredAt; +use std::path::Path; + +/// A declaration's file relative to the project; code outside the project keeps its location private. +#[inline] +pub fn declared_at( + project_dir: &Path, + written: Option<(String, u32)>, +) -> PyResult> { + orchestration_declarations::declared_at(project_dir, written) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/environment_hash.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/environment_hash.rs new file mode 100644 index 0000000000..4c99fd91ae --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/environment_hash.rs @@ -0,0 +1,16 @@ +//! A lockfile binds declared dependencies; without one, bind installed distribution versions. + +use crate::orchestration_reads::_helpers::orchestration_environment; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::Path; + +/// A lockfile binds declared dependencies; without one, bind installed distribution versions. +#[inline] +pub fn environment_hash( + py: Python<'_>, + project_dir: &Path, + sqlbuild_version: &str, +) -> PyResult { + orchestration_environment::environment_hash(py, project_dir, sqlbuild_version) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/imported_declarations.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/imported_declarations.rs new file mode 100644 index 0000000000..4cac5c070a --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/imported_declarations.rs @@ -0,0 +1,17 @@ +//! Sort imported declarations into calendars and sensors, refusing a calendar name defined twice. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use crate::orchestration_reads::models::ImportedDeclarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Sort imported declarations into calendars and sensors, refusing a calendar name defined twice. +#[inline] +pub fn imported_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult> { + orchestration_declarations::imported_declarations(py, declarations) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/missing_secrets.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/missing_secrets.rs new file mode 100644 index 0000000000..106c624f74 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/missing_secrets.rs @@ -0,0 +1,12 @@ +//! The environment variables of the class's required secrets the environment leaves unset; an invalid setting is an error. + +use crate::orchestration_reads::_helpers::provider_settings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The environment variables of the class's required secrets the environment leaves unset; an invalid setting is an error. +#[inline] +pub fn missing_secrets(provider_class: &Bound<'_, PyAny>) -> PyResult> { + provider_settings::missing_secrets(provider_class) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs new file mode 100644 index 0000000000..0ae0a077a2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs @@ -0,0 +1,14 @@ +pub mod callback_code_hash; +pub mod declared_at; +pub mod environment_hash; +pub mod imported_declarations; +pub mod missing_secrets; +pub mod module_declarations; +pub mod orchestration_code_hash; +pub mod provider_class_settings; +pub mod provider_state; +pub mod register_orchestration_declarations; +pub mod run_controls; +pub mod run_resources; +pub mod secret_variables; +pub mod setting_rows; diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/module_declarations.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/module_declarations.rs new file mode 100644 index 0000000000..2f468afe98 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/module_declarations.rs @@ -0,0 +1,18 @@ +//! The calendars and sensors one module defines; only module execution crosses into Python. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use std::path::Path; + +/// The calendars and sensors one module defines; only module execution crosses into Python. +#[inline] +pub fn module_declarations<'py>( + py: Python<'py>, + file_path: &Path, + project_dir: &Path, +) -> PyResult>> { + orchestration_declarations::module_declarations(py, file_path, project_dir) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/orchestration_code_hash.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/orchestration_code_hash.rs new file mode 100644 index 0000000000..6d696b6de2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/orchestration_code_hash.rs @@ -0,0 +1,11 @@ +//! Hash declaration/helper sources once per compilation, never once per sensor. + +use crate::orchestration_reads::_helpers::orchestration_environment; +use pyo3::prelude::PyResult; +use std::path::Path; + +/// Hash declaration/helper sources once per compilation, never once per sensor. +#[inline] +pub fn orchestration_code_hash(project_dir: &Path) -> PyResult { + orchestration_environment::orchestration_code_hash(project_dir) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_class_settings.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_class_settings.rs new file mode 100644 index 0000000000..c5032b0ed4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_class_settings.rs @@ -0,0 +1,13 @@ +//! The class's description and declared setting fields, read without any setting value. + +use crate::orchestration_reads::_helpers::provider_settings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use serde_json::Value; + +/// The class's description and declared setting fields, read without any setting value. +#[inline] +pub fn provider_class_settings(provider_class: &Bound<'_, PyAny>) -> PyResult { + provider_settings::provider_class_settings(provider_class) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_state.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_state.rs new file mode 100644 index 0000000000..43029d4695 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/provider_state.rs @@ -0,0 +1,13 @@ +//! Instantiate the class, telling unset required secrets apart from other invalid settings. + +use crate::orchestration_reads::_helpers::provider_settings; +use crate::orchestration_reads::models::ProviderState; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Instantiate the class, telling unset required secrets apart from other invalid settings. +#[inline] +pub fn provider_state<'py>(provider_class: &Bound<'py, PyAny>) -> PyResult> { + provider_settings::provider_state(provider_class) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/register_orchestration_declarations.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/register_orchestration_declarations.rs new file mode 100644 index 0000000000..0f66723356 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/register_orchestration_declarations.rs @@ -0,0 +1,12 @@ +//! Register `declaration_site` on the native module. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register `declaration_site` on the native module. +#[inline] +pub fn register_orchestration_declarations(module: &Bound<'_, PyModule>) -> PyResult<()> { + orchestration_declarations::register(module) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/run_controls.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/run_controls.rs new file mode 100644 index 0000000000..4eab07988d --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/run_controls.rs @@ -0,0 +1,11 @@ +//! The controls of a TOML job or group; discovery has already checked the owner and the retry bounds. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use crate::orchestration_reads::models::RunControls; +use sqlbuild_discovery::orchestration_files::models::RunFile; + +/// The controls of a TOML job or group; discovery has already checked the owner and the retry bounds. +#[inline] +pub fn run_controls(run: &RunFile) -> RunControls { + orchestration_declarations::run_controls(run) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/run_resources.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/run_resources.rs new file mode 100644 index 0000000000..6a15f26de7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/run_resources.rs @@ -0,0 +1,15 @@ +//! Refuse an unknown limit, a memory size Docker cannot read, and CPUs or a process limit that is not positive. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use pyo3::prelude::PyResult; +use serde_json::Value; +use sqlbuild_orchestration::declarations::models::RunResources; + +/// Refuse an unknown limit, a memory size Docker cannot read, and CPUs or a process limit that is not positive. +#[inline] +pub fn run_resources( + name: &str, + written: &serde_json::Map, +) -> PyResult { + orchestration_declarations::run_resources(name, written) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/secret_variables.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/secret_variables.rs new file mode 100644 index 0000000000..e7c43914a6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/secret_variables.rs @@ -0,0 +1,12 @@ +//! The environment variables of the class's secret settings. + +use crate::orchestration_reads::_helpers::provider_settings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The environment variables of the class's secret settings. +#[inline] +pub fn secret_variables(provider_class: &Bound<'_, PyAny>) -> PyResult> { + provider_settings::secret_variables(provider_class) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/setting_rows.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/setting_rows.rs new file mode 100644 index 0000000000..b64a29df67 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/setting_rows.rs @@ -0,0 +1,10 @@ +//! Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. + +use crate::orchestration_reads::_helpers::provider_settings; +use serde_json::Value; + +/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. +#[inline] +pub fn setting_rows(class: &Value) -> Vec { + provider_settings::setting_rows(class) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/mod.rs b/crates/sqlbuild-python-base/src/orchestration_reads/mod.rs new file mode 100644 index 0000000000..17c506070a --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/mod.rs @@ -0,0 +1,5 @@ +//! Orchestration declarations, provider settings and the location environment read from Python. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/models.rs b/crates/sqlbuild-python-base/src/orchestration_reads/models.rs new file mode 100644 index 0000000000..501f3cb5d1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/models.rs @@ -0,0 +1,67 @@ +//! Public records of orchestration reads. + +use crate::orchestration_reads::_helpers::orchestration_declarations::authored_setting; +use pyo3::FromPyObject; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyErr; +use serde_json::Value; +use sqlbuild_orchestration::declarations::models::OnFailure; +use sqlbuild_orchestration::declarations::models::Overlap; +use std::collections::BTreeMap; + +#[derive(Debug, FromPyObject)] +pub struct AuthoredSensorCondition { + pub kind: String, + #[pyo3(default)] + pub kinds: Vec, + #[pyo3(default)] + pub job: Option, + #[pyo3(default)] + pub executed_successfully: Option>, + #[pyo3(default)] + pub select: Option, + #[pyo3(default, from_py_with = authored_setting)] + pub after_seconds: Value, +} + +#[derive(Debug, FromPyObject)] +pub struct AuthoredSensor { + pub name: String, + pub description: String, + pub when: Option, + pub function: Option>, + pub run: String, + #[pyo3(from_py_with = authored_setting)] + pub minimum_interval_seconds: Value, + pub default_status: String, + pub targets: Vec, + #[pyo3(attribute("_declared_at"))] + pub declared_at: Option<(String, u32)>, +} + +/// A job's or group's owner, overlap and failure policy, as publication records them. +#[derive(Debug)] +pub struct RunControls { + pub owner: Option, + pub overlap: Overlap, + pub on_failure: Option, +} + +/// The calendars, by name with their descriptions, and the sensors that importing the modules defines. +#[derive(Debug)] +pub struct ImportedDeclarations<'py> { + pub calendars: BTreeMap>, + pub calendar_descriptions: BTreeMap, + pub sensors: Vec, +} + +/// How a provider class instantiates from the environment. +#[derive(Debug)] +pub enum ProviderState<'py> { + Ready(Bound<'py, PyAny>), + /// Only required secret settings are unset: their environment variables, sorted. + MissingSecrets(Vec), + /// Another setting is invalid: pydantic's validation error. + Invalid(PyErr), +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs b/crates/sqlbuild-python-base/src/planner_identity/_helpers/json_values.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs rename to crates/sqlbuild-python-base/src/planner_identity/_helpers/json_values.rs index 05b2897621..b13dc0b0ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs +++ b/crates/sqlbuild-python-base/src/planner_identity/_helpers/json_values.rs @@ -7,18 +7,13 @@ use pyo3::types::{ PyTypeMethods, }; +use crate::records::main::fields::field::field as native_field; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::main::python_float_repr::python_float_repr; use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; const BUILTINS_MODULE: &str = "builtins"; -/// Compact `(",", ":")` separators. -pub(crate) const COMPACT: (&str, &str) = (",", ":"); -/// Python's default `(", ", ": ")` separators. -pub(crate) const SPACED: (&str, &str) = (", ", ": "); - /// `json.dumps(value, sort_keys=True, separators=separators)`, with `default=str` when `default_str`. pub(crate) fn sorted_dumps( value: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-base/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-base/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..291e98e01b --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_values; diff --git a/crates/sqlbuild-python-base/src/planner_identity/constants.rs b/crates/sqlbuild-python-base/src/planner_identity/constants.rs new file mode 100644 index 0000000000..b92b1b16bd --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/constants.rs @@ -0,0 +1,7 @@ +//! Public constants of planner identity. + +/// Compact `(",", ":")` separators. +pub const COMPACT: (&str, &str) = (",", ":"); + +/// Python's default `(", ", ": ")` separators. +pub const SPACED: (&str, &str) = (", ", ": "); diff --git a/crates/sqlbuild-python-base/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-base/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..41850fd61a --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/main/mod.rs @@ -0,0 +1,2 @@ +pub mod sorted_dumps; +pub mod unicode_sorted_dumps; diff --git a/crates/sqlbuild-python-base/src/planner_identity/main/sorted_dumps.rs b/crates/sqlbuild-python-base/src/planner_identity/main/sorted_dumps.rs new file mode 100644 index 0000000000..1f884cca75 --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/main/sorted_dumps.rs @@ -0,0 +1,16 @@ +//! `json.dumps(value, sort_keys=True, separators=separators)`, with `default=str` when `default_str`. + +use crate::planner_identity::_helpers::json_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `json.dumps(value, sort_keys=True, separators=separators)`, with `default=str` when `default_str`. +#[inline] +pub fn sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), + default_str: bool, +) -> PyResult { + json_values::sorted_dumps(value, separators, default_str) +} diff --git a/crates/sqlbuild-python-base/src/planner_identity/main/unicode_sorted_dumps.rs b/crates/sqlbuild-python-base/src/planner_identity/main/unicode_sorted_dumps.rs new file mode 100644 index 0000000000..de286c0434 --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/main/unicode_sorted_dumps.rs @@ -0,0 +1,15 @@ +//! `json.dumps(value, ensure_ascii=False, sort_keys=True, separators=separators)`. + +use crate::planner_identity::_helpers::json_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `json.dumps(value, ensure_ascii=False, sort_keys=True, separators=separators)`. +#[inline] +pub fn unicode_sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), +) -> PyResult { + json_values::unicode_sorted_dumps(value, separators) +} diff --git a/crates/sqlbuild-python-base/src/planner_identity/mod.rs b/crates/sqlbuild-python-base/src/planner_identity/mod.rs new file mode 100644 index 0000000000..deed01d739 --- /dev/null +++ b/crates/sqlbuild-python-base/src/planner_identity/mod.rs @@ -0,0 +1,5 @@ +//! Python values as sorted `json.dumps` writes them, for planner identities. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/document.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs rename to crates/sqlbuild-python-base/src/presentation_text/_helpers/document.rs index 07a4a354d9..ba9b326ffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs +++ b/crates/sqlbuild-python-base/src/presentation_text/_helpers/document.rs @@ -7,7 +7,7 @@ use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliDocument as Document; -use crate::bindings::_helpers::presentation_text::functions::style_of; +use crate::presentation_text::_helpers::functions::style_of; /// Build simple static CLI output with semantic styling. #[pyclass(module = "sqlbuild._native", name = "CliDocument", frozen)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs rename to crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs index 54c647cd97..77222e523d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs +++ b/crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs @@ -13,8 +13,8 @@ use sqlbuild_core::cli_text::main::resolve_name_column_width::resolve_name_colum use sqlbuild_core::cli_text::main::split_count_header::split_count_header; use sqlbuild_core::cli_text::models::{AlignedRow, CliStyle, CompletionState, SummaryCount}; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::boundary::models::KeywordArguments; +use crate::records::main::fields::field::field as native_field; const ALIGNED_OPTIONS: (&[&str], &[&str]) = ( &["plain_name", "styled_name", "value", "name_column_width"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/getters.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/presentation_text/getters.rs rename to crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs index f05c9656c0..dfcf2441cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/getters.rs +++ b/crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::presentation_text::style::PythonCliStyle; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::presentation_text::models::PythonCliStyle; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_python_cli_style, -}]; - -fn read_python_cli_style<'py>( +pub(crate) fn read_python_cli_style<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-base/src/presentation_text/_helpers/mod.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/mod.rs new file mode 100644 index 0000000000..14b7ef753c --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod document; +pub(crate) mod functions; +pub(crate) mod getters; +pub(crate) mod style; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-python-base/src/presentation_text/_helpers/style.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/style.rs new file mode 100644 index 0000000000..d34c07bac3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/_helpers/style.rs @@ -0,0 +1,8 @@ +//! Semantic styling for human CLI output, as a Python class. + +use crate::presentation_text::models::PythonCliStyle; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs b/crates/sqlbuild-python-base/src/presentation_text/_helpers/terminal.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs rename to crates/sqlbuild-python-base/src/presentation_text/_helpers/terminal.rs diff --git a/crates/sqlbuild-python-base/src/presentation_text/constants.rs b/crates/sqlbuild-python-base/src/presentation_text/constants.rs new file mode 100644 index 0000000000..fc3ee55458 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of presentation text. + +use crate::presentation_text::_helpers::getters::read_python_cli_style; +use crate::presentation_text::models::PythonCliStyle; +use crate::records::main::registry::class_of::class_of; +use crate::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_python_cli_style, +}]; diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/mod.rs b/crates/sqlbuild-python-base/src/presentation_text/main/mod.rs new file mode 100644 index 0000000000..c8ef42a901 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/mod.rs @@ -0,0 +1,6 @@ +pub mod register_document; +pub mod register_functions; +pub mod register_style; +pub mod register_terminal; +pub mod stdout_is_tty; +pub mod stdout_supports_color; diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/register_document.rs b/crates/sqlbuild-python-base/src/presentation_text/main/register_document.rs new file mode 100644 index 0000000000..e0220428c1 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/register_document.rs @@ -0,0 +1,12 @@ +//! Register the document functions and classes. + +use crate::presentation_text::_helpers::document; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the document functions and classes. +#[inline] +pub fn register_document(module: &Bound<'_, PyModule>) -> PyResult<()> { + document::register(module) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/register_functions.rs b/crates/sqlbuild-python-base/src/presentation_text/main/register_functions.rs new file mode 100644 index 0000000000..0bc3707fd3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/register_functions.rs @@ -0,0 +1,12 @@ +//! Register the functions functions and classes. + +use crate::presentation_text::_helpers::functions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the functions functions and classes. +#[inline] +pub fn register_functions(module: &Bound<'_, PyModule>) -> PyResult<()> { + functions::register(module) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/register_style.rs b/crates/sqlbuild-python-base/src/presentation_text/main/register_style.rs new file mode 100644 index 0000000000..2e9e3c6cb3 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/register_style.rs @@ -0,0 +1,12 @@ +//! Register the style functions and classes. + +use crate::presentation_text::_helpers::style; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the style functions and classes. +#[inline] +pub fn register_style(module: &Bound<'_, PyModule>) -> PyResult<()> { + style::register(module) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/register_terminal.rs b/crates/sqlbuild-python-base/src/presentation_text/main/register_terminal.rs new file mode 100644 index 0000000000..799f8baee0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/register_terminal.rs @@ -0,0 +1,12 @@ +//! Register the terminal functions and classes. + +use crate::presentation_text::_helpers::terminal; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the terminal functions and classes. +#[inline] +pub fn register_terminal(module: &Bound<'_, PyModule>) -> PyResult<()> { + terminal::register(module) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/stdout_is_tty.rs b/crates/sqlbuild-python-base/src/presentation_text/main/stdout_is_tty.rs new file mode 100644 index 0000000000..aeff9cc654 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/stdout_is_tty.rs @@ -0,0 +1,11 @@ +//! Whether `sys.stdout` is a terminal. + +use crate::presentation_text::_helpers::terminal; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Whether `sys.stdout` is a terminal. +#[inline] +pub fn stdout_is_tty(py: Python<'_>) -> PyResult { + terminal::stdout_is_tty(py) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/main/stdout_supports_color.rs b/crates/sqlbuild-python-base/src/presentation_text/main/stdout_supports_color.rs new file mode 100644 index 0000000000..214f91b7dd --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/main/stdout_supports_color.rs @@ -0,0 +1,11 @@ +//! Whether `sys.stdout` is a terminal and `NO_COLOR` is unset. + +use crate::presentation_text::_helpers::terminal; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Whether `sys.stdout` is a terminal and `NO_COLOR` is unset. +#[inline] +pub fn stdout_supports_color(py: Python<'_>) -> PyResult { + terminal::stdout_supports_color(py) +} diff --git a/crates/sqlbuild-python-base/src/presentation_text/mod.rs b/crates/sqlbuild-python-base/src/presentation_text/mod.rs new file mode 100644 index 0000000000..76b89dacc2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/mod.rs @@ -0,0 +1,6 @@ +//! CLI text: styles, static documents, shared text helpers and terminal colour support. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-base/src/presentation_text/models.rs b/crates/sqlbuild-python-base/src/presentation_text/models.rs new file mode 100644 index 0000000000..f2921f615d --- /dev/null +++ b/crates/sqlbuild-python-base/src/presentation_text/models.rs @@ -0,0 +1,130 @@ +//! Public records of presentation text. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::pyclass; +use pyo3::pymethods; +use sqlbuild_core::cli_text::models::CliStyle as NativeStyle; + +/// Semantic CLI styling facade used by human-output formatters. +#[pyclass(name = "CliStyle", module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct PythonCliStyle { + #[pyo3(get)] + pub use_color: bool, +} + +impl PythonCliStyle { + pub(crate) fn native(&self) -> NativeStyle { + NativeStyle { + use_color: self.use_color, + } + } +} + +#[pymethods] +impl PythonCliStyle { + #[new] + #[pyo3(signature = (*, use_color))] + pub fn new(use_color: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + use_color: use_color.is_truthy()?, + }) + } + + pub(crate) fn title(&self, text: &str) -> String { + self.native().title(text) + } + + pub(crate) fn section(&self, text: &str) -> String { + self.native().section(text) + } + + pub(crate) fn label(&self, text: &str) -> String { + self.native().label(text) + } + + pub(crate) fn value(&self, text: &str) -> String { + self.native().value(text) + } + + pub(crate) fn accent(&self, text: &str) -> String { + self.native().accent(text) + } + + pub(crate) fn accent_strong(&self, text: &str) -> String { + self.native().accent_strong(text) + } + + pub(crate) fn plan_section(&self, text: &str) -> String { + self.native().plan_section(text) + } + + pub(crate) fn object_name(&self, text: &str) -> String { + self.native().object_name(text) + } + + pub(crate) fn command(&self, text: &str) -> String { + self.native().command(text) + } + + pub(crate) fn muted(&self, text: &str) -> String { + self.native().muted(text) + } + + pub(crate) fn success(&self, text: &str) -> String { + self.native().success(text) + } + + pub(crate) fn success_strong(&self, text: &str) -> String { + self.native().success_strong(text) + } + + pub(crate) fn warning(&self, text: &str) -> String { + self.native().warning(text) + } + + pub(crate) fn warning_strong(&self, text: &str) -> String { + self.native().warning_strong(text) + } + + pub(crate) fn error(&self, text: &str) -> String { + self.native().error(text) + } + + pub(crate) fn error_strong(&self, text: &str) -> String { + self.native().error_strong(text) + } + + pub(crate) fn error_muted(&self, text: &str) -> String { + self.native().error_muted(text) + } + + pub(crate) fn log_label(&self, text: &str) -> String { + self.native().log_label(text) + } + + pub(crate) fn dbt_section(&self, text: &str) -> String { + self.native().dbt_section(text) + } + + pub(crate) fn dbt_label(&self, text: &str) -> String { + self.native().dbt_label(text) + } + + pub(crate) fn dbt_object_name(&self, text: &str) -> String { + self.native().dbt_object_name(text) + } + + pub(crate) fn dbt_execution_label(&self, text: &str) -> String { + self.native().dbt_execution_label(text) + } + + /// Style status text according to the status word. + #[pyo3(signature = (*, status, text = None))] + pub(crate) fn status(&self, status: &str, text: Option<&str>) -> String { + self.native().status_text(status, text.unwrap_or(status)) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs b/crates/sqlbuild-python-base/src/python_values/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/python_values/mod.rs rename to crates/sqlbuild-python-base/src/python_values/_helpers/mod.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs b/crates/sqlbuild-python-base/src/python_values/_helpers/values.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs rename to crates/sqlbuild-python-base/src/python_values/_helpers/values.rs index 24a2ab767a..b03a288364 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs +++ b/crates/sqlbuild-python-base/src/python_values/_helpers/values.rs @@ -1,16 +1,17 @@ //! Python values as `json.dumps` would encode them, optionally after `dataclasses.asdict`. +use crate::boundary::main::utf16_units::utf16_units; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ - PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, - PyString, PyStringMethods, PyTuple, PyTypeMethods, + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, PyTypeMethods, }; use sqlbuild_analysis::python_values::models::{PythonKey, PythonValue}; use sqlbuild_core::json::models::JsonInteger; -use crate::bindings::_helpers::compile_outputs::json_report::float_key; -use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::boundary::main::native_compiler_error::native_compiler_error; +use crate::compile_outputs::main::float_key::float_key; +use crate::records::main::fields::field::field as native_field; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; const SURROGATE_KEY: &str = "a JSON artifact mapping key holds a lone surrogate, which the native \ @@ -23,21 +24,10 @@ fn type_name(value: &Bound<'_, PyAny>) -> PyResult { /// A `str`, as UTF-16 code units when it holds lone surrogates. pub(crate) fn text_value(text: &Bound<'_, PyString>) -> PyResult { - if let Ok(text) = text.to_str() { - return Ok(PythonValue::String(text.to_owned())); - } - let encoded: Bound<'_, PyBytes> = text - .call_method1("encode", ("utf-16-le", "surrogatepass"))? - .downcast_into::()?; - Ok(PythonValue::Utf16( - encoded - .as_bytes() - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) - .collect(), - )) + match text.to_str() { + Ok(text) => Ok(PythonValue::String(text.to_owned())), + Err(_) => utf16_units(text).map(PythonValue::Utf16), + } } fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python-base/src/python_values/main/mod.rs b/crates/sqlbuild-python-base/src/python_values/main/mod.rs new file mode 100644 index 0000000000..1365cab48e --- /dev/null +++ b/crates/sqlbuild-python-base/src/python_values/main/mod.rs @@ -0,0 +1 @@ +pub mod python_value; diff --git a/crates/sqlbuild-python-base/src/python_values/main/python_value.rs b/crates/sqlbuild-python-base/src/python_values/main/python_value.rs new file mode 100644 index 0000000000..a32283dd48 --- /dev/null +++ b/crates/sqlbuild-python-base/src/python_values/main/python_value.rs @@ -0,0 +1,13 @@ +//! The value `json.dumps` encodes, copied by `asdict` first when `dataclasses` is set. + +use crate::python_values::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::python_values::models::PythonValue; + +/// The value `json.dumps` encodes, copied by `asdict` first when `dataclasses` is set. +#[inline] +pub fn python_value(value: &Bound<'_, PyAny>, dataclasses: bool) -> PyResult { + values::python_value(value, dataclasses) +} diff --git a/crates/sqlbuild-python-base/src/python_values/mod.rs b/crates/sqlbuild-python-base/src/python_values/mod.rs new file mode 100644 index 0000000000..0d72e03112 --- /dev/null +++ b/crates/sqlbuild-python-base/src/python_values/mod.rs @@ -0,0 +1,4 @@ +//! Python values as `json.dumps` encodes them. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs index ab9001ca12..c87c99f7e1 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs @@ -41,6 +41,13 @@ pub(crate) fn error_class(py: Python<'_>, id: ErrorId) -> PyResult, id: ErrorId, message: &str) -> PyErr { + error_class(py, id) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + /// A raisable native `id` error carrying `message` and the diagnostic `code`. pub(crate) fn new_error(py: Python<'_>, id: ErrorId, message: String, code: &str) -> PyErr { let raised = (|| { diff --git a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs index 5f66c8f2a0..fb2bfee1e3 100644 --- a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs +++ b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs @@ -4,6 +4,7 @@ pub mod error_class; pub mod new_annotated_error; pub mod new_error; pub mod new_helped_error; +pub mod new_plain_error; pub mod new_record; pub mod new_record_with; pub mod record_class; diff --git a/crates/sqlbuild-python-base/src/records/main/construct/new_plain_error.rs b/crates/sqlbuild-python-base/src/records/main/construct/new_plain_error.rs new file mode 100644 index 0000000000..17d5333a3a --- /dev/null +++ b/crates/sqlbuild-python-base/src/records/main/construct/new_plain_error.rs @@ -0,0 +1,12 @@ +//! The native `id` exception raised with `message`, or the error looking it up or raising it. + +use crate::records::_helpers::native_classes::construct; +use crate::records::types::ErrorId; +use pyo3::PyErr; +use pyo3::prelude::Python; + +/// The native `id` exception raised with `message`, or the error looking it up or raising it. +#[inline] +pub fn new_plain_error(py: Python<'_>, id: ErrorId, message: &str) -> PyErr { + construct::new_plain_error(py, id, message) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs b/crates/sqlbuild-python-base/src/result_output/_helpers/integration_values.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs rename to crates/sqlbuild-python-base/src/result_output/_helpers/integration_values.rs index be5d364a67..53ebfc67ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs +++ b/crates/sqlbuild-python-base/src/result_output/_helpers/integration_values.rs @@ -1,5 +1,7 @@ //! Python values as integration JSON, and contract failures as `ObservabilityValidationError`. +use crate::native_host::main::host_error::host_error; +use crate::result_output::types::Checked; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, @@ -10,19 +12,13 @@ use sqlbuild_core::integration_results::main::encode_integration_value::encode_i use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; - -/// A contract failure, or a Python failure while reading the values. -pub(crate) type Checked = PyResult>; - /// The `ObservabilityValidationError` Python raised for the same contract failure. pub(crate) fn validation_error(py: Python<'_>, error: &IntegrationRecordError) -> PyErr { - match host_entry(py, ExecutorHost::ObservabilityValidationError) - .and_then(|class| class.call1((error.message.as_str(),))) - { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } + host_error( + py, + ExecutorHost::ObservabilityValidationError, + error.message.as_str(), + ) } /// A plain JSON Python value, or `None` when it holds something JSON cannot carry. diff --git a/crates/sqlbuild-python-base/src/result_output/_helpers/mod.rs b/crates/sqlbuild-python-base/src/result_output/_helpers/mod.rs new file mode 100644 index 0000000000..99adf1f471 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod integration_values; diff --git a/crates/sqlbuild-python-base/src/result_output/main/bounded_metadata.rs b/crates/sqlbuild-python-base/src/result_output/main/bounded_metadata.rs new file mode 100644 index 0000000000..be6d2f3742 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/bounded_metadata.rs @@ -0,0 +1,13 @@ +//! `_bounded_metadata`: the value when it encodes within the metadata bounds, else `{}`. + +use crate::result_output::_helpers::integration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// `_bounded_metadata`: the value when it encodes within the metadata bounds, else `{}`. +#[inline] +pub fn bounded_metadata(value: &Bound<'_, PyAny>) -> PyResult { + integration_values::bounded_metadata(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/json_value.rs b/crates/sqlbuild-python-base/src/result_output/main/json_value.rs new file mode 100644 index 0000000000..08cf8ebf4a --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/json_value.rs @@ -0,0 +1,13 @@ +//! A plain JSON Python value, or `None` when it holds something JSON cannot carry. + +use crate::result_output::_helpers::integration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// A plain JSON Python value, or `None` when it holds something JSON cannot carry. +#[inline] +pub fn json_value(value: &Bound<'_, PyAny>) -> PyResult> { + integration_values::json_value(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/mod.rs b/crates/sqlbuild-python-base/src/result_output/main/mod.rs new file mode 100644 index 0000000000..543f922edb --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/mod.rs @@ -0,0 +1,7 @@ +pub mod bounded_metadata; +pub mod json_value; +pub mod optional_text; +pub mod required_json; +pub mod text; +pub mod text_or_empty; +pub mod validation_error; diff --git a/crates/sqlbuild-python-base/src/result_output/main/optional_text.rs b/crates/sqlbuild-python-base/src/result_output/main/optional_text.rs new file mode 100644 index 0000000000..a8d272b4ef --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/optional_text.rs @@ -0,0 +1,13 @@ +//! `value` when it is a string, else `null`. + +use crate::result_output::_helpers::integration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// `value` when it is a string, else `null`. +#[inline] +pub fn optional_text(value: &Bound<'_, PyAny>) -> PyResult { + integration_values::optional_text(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/required_json.rs b/crates/sqlbuild-python-base/src/result_output/main/required_json.rs new file mode 100644 index 0000000000..66f9d94aa2 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/required_json.rs @@ -0,0 +1,13 @@ +//! A plain JSON Python value, or the contract failure of a non-JSON value. + +use crate::result_output::_helpers::integration_values; +use crate::result_output::types::Checked; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use sqlbuild_core::json::models::JsonValue; + +/// A plain JSON Python value, or the contract failure of a non-JSON value. +#[inline] +pub fn required_json(value: &Bound<'_, PyAny>) -> Checked { + integration_values::required_json(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/text.rs b/crates/sqlbuild-python-base/src/result_output/main/text.rs new file mode 100644 index 0000000000..58b061e598 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/text.rs @@ -0,0 +1,10 @@ +//! A JSON string. + +use crate::result_output::_helpers::integration_values; +use sqlbuild_core::json::models::JsonValue; + +/// A JSON string. +#[inline] +pub fn text(value: &str) -> JsonValue { + integration_values::text(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/text_or_empty.rs b/crates/sqlbuild-python-base/src/result_output/main/text_or_empty.rs new file mode 100644 index 0000000000..56ff94ca37 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/text_or_empty.rs @@ -0,0 +1,13 @@ +//! `str(value or "")`. + +use crate::result_output::_helpers::integration_values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::json::models::JsonValue; + +/// `str(value or "")`. +#[inline] +pub fn text_or_empty(value: &Bound<'_, PyAny>) -> PyResult { + integration_values::text_or_empty(value) +} diff --git a/crates/sqlbuild-python-base/src/result_output/main/validation_error.rs b/crates/sqlbuild-python-base/src/result_output/main/validation_error.rs new file mode 100644 index 0000000000..4ba3f9ac37 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/main/validation_error.rs @@ -0,0 +1,12 @@ +//! The `ObservabilityValidationError` Python raised for the same contract failure. + +use crate::result_output::_helpers::integration_values; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; + +/// The `ObservabilityValidationError` Python raised for the same contract failure. +#[inline] +pub fn validation_error(py: Python<'_>, error: &IntegrationRecordError) -> PyErr { + integration_values::validation_error(py, error) +} diff --git a/crates/sqlbuild-python-base/src/result_output/mod.rs b/crates/sqlbuild-python-base/src/result_output/mod.rs new file mode 100644 index 0000000000..568f20cd59 --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/mod.rs @@ -0,0 +1,5 @@ +//! Python values as integration JSON and their contract failures. + +pub(crate) mod _helpers; +pub mod main; +pub mod types; diff --git a/crates/sqlbuild-python-base/src/result_output/types.rs b/crates/sqlbuild-python-base/src/result_output/types.rs new file mode 100644 index 0000000000..76855ae47a --- /dev/null +++ b/crates/sqlbuild-python-base/src/result_output/types.rs @@ -0,0 +1,7 @@ +//! Public traits and type aliases of result output. + +use pyo3::prelude::PyResult; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; + +/// A contract failure, or a Python failure while reading the values. +pub type Checked = PyResult>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs b/crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs rename to crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs index 7bfc1351e7..22056cb7f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs +++ b/crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs @@ -10,10 +10,10 @@ use sqlbuild_analysis::type_system::main::same_type::same_type; use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::type_system::normalization::normalization_error; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::compiler_guard::compiler_guard; +use crate::native_host::main::host_entry::host_entry; +use crate::type_system::_helpers::normalization::normalization_error; const TYPE_NORMALIZATION_LOGGER: &str = "sqlbuild.adapter"; const TYPE_PARSE_FALLBACK_MESSAGE: &str = "type normalization parse failed; falling back"; diff --git a/crates/sqlbuild-python-base/src/type_system/_helpers/mod.rs b/crates/sqlbuild-python-base/src/type_system/_helpers/mod.rs new file mode 100644 index 0000000000..ecd0ccdcd8 --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod equality; +pub(crate) mod normalization; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs b/crates/sqlbuild-python-base/src/type_system/_helpers/normalization.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs rename to crates/sqlbuild-python-base/src/type_system/_helpers/normalization.rs index 639fecc61f..916e67c13c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs +++ b/crates/sqlbuild-python-base/src/type_system/_helpers/normalization.rs @@ -6,19 +6,35 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::type_system::errors::TypeNormalizationError; use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; -use sqlbuild_python_base::positional_record; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; - -positional_record! { - into; - /// `(name, family, precision, scale, length)` of one normalized type. - struct NormalizedRow { - name: String, - family: &'static str, - precision: Option, - scale: Option, - length: Option, + +use crate::boundary::main::compiler_guard::compiler_guard; + +/// `(name, family, precision, scale, length)` of one normalized type. +struct NormalizedRow { + name: String, + family: &'static str, + precision: Option, + scale: Option, + length: Option, +} + +impl<'py> pyo3::IntoPyObject<'py> for NormalizedRow { + type Target = pyo3::types::PyTuple; + type Output = Bound<'py, pyo3::types::PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> PyResult { + use pyo3::IntoPyObjectExt; + pyo3::types::PyTuple::new( + py, + [ + self.name.into_bound_py_any(py)?, + self.family.into_bound_py_any(py)?, + self.precision.into_bound_py_any(py)?, + self.scale.into_bound_py_any(py)?, + self.length.into_bound_py_any(py)?, + ], + ) } } diff --git a/crates/sqlbuild-python-base/src/type_system/main/mod.rs b/crates/sqlbuild-python-base/src/type_system/main/mod.rs new file mode 100644 index 0000000000..99861214f0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/main/mod.rs @@ -0,0 +1,3 @@ +pub mod normalization_error; +pub mod register_normalization; +pub mod types_equal; diff --git a/crates/sqlbuild-python-base/src/type_system/main/normalization_error.rs b/crates/sqlbuild-python-base/src/type_system/main/normalization_error.rs new file mode 100644 index 0000000000..f32fba28dd --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/main/normalization_error.rs @@ -0,0 +1,11 @@ +//! Python's exception for a type that cannot be normalized. + +use crate::type_system::_helpers::normalization; +use pyo3::prelude::PyErr; +use sqlbuild_analysis::type_system::errors::TypeNormalizationError; + +/// Python's exception for a type that cannot be normalized. +#[inline] +pub fn normalization_error(error: &TypeNormalizationError) -> PyErr { + normalization::normalization_error(error) +} diff --git a/crates/sqlbuild-python-base/src/type_system/main/register_normalization.rs b/crates/sqlbuild-python-base/src/type_system/main/register_normalization.rs new file mode 100644 index 0000000000..c7e9d22226 --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/main/register_normalization.rs @@ -0,0 +1,12 @@ +//! Register the normalization functions and classes. + +use crate::type_system::_helpers::normalization; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the normalization functions and classes. +#[inline] +pub fn register_normalization(module: &Bound<'_, PyModule>) -> PyResult<()> { + normalization::register(module) +} diff --git a/crates/sqlbuild-python-base/src/type_system/main/types_equal.rs b/crates/sqlbuild-python-base/src/type_system/main/types_equal.rs new file mode 100644 index 0000000000..40ce445459 --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/main/types_equal.rs @@ -0,0 +1,12 @@ +//! Whether two type strings are semantically equal under `dialect` (a `TypeDialect`, name or `None`). + +use crate::type_system::_helpers::equality; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Whether two type strings are semantically equal under `dialect` (a `TypeDialect`, name or `None`). +#[inline] +pub fn types_equal(dialect: &Bound<'_, PyAny>, left: &str, right: &str) -> PyResult { + equality::types_equal(dialect, left, right) +} diff --git a/crates/sqlbuild-python-base/src/type_system/mod.rs b/crates/sqlbuild-python-base/src/type_system/mod.rs new file mode 100644 index 0000000000..88d385432d --- /dev/null +++ b/crates/sqlbuild-python-base/src/type_system/mod.rs @@ -0,0 +1,4 @@ +//! Native type normalization and semantic type equality. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs new file mode 100644 index 0000000000..36e4f4ba72 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod sqlite_connection; +pub(crate) mod state_column_evolution; +pub(crate) mod synced_copy; +pub(crate) mod warehouse_catalog; +pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/sqlite_connection.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/sqlite_connection.rs new file mode 100644 index 0000000000..9e331fc5c0 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/sqlite_connection.rs @@ -0,0 +1,49 @@ +//! Rust-owned SQLite connection and its rows; Python views exist only for connection overrides. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; +use rusqlite::types::Value; +use rusqlite::{Error, ErrorCode}; +use sqlbuild_state::execution_history::models::SqlParam; + +use crate::boundary::main::imported::imported; + +/// Preserve DB-API exception classes at the public storage-error boundary. +pub(crate) fn database_error(py: Python<'_>, error: Error) -> PyErr { + let class: &str = match &error { + Error::SqliteFailure(error, _) if error.code == ErrorCode::ConstraintViolation => { + "IntegrityError" + } + Error::SqliteFailure(error, _) + if matches!( + error.code, + ErrorCode::DatabaseCorrupt | ErrorCode::NotADatabase + ) => + { + "DatabaseError" + } + _ => "OperationalError", + }; + match imported(py, "sqlite3", class).and_then(|class| class.call1((error.to_string(),))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Parameters of history SQL contain no Python-owned values. +pub(crate) fn sqlite_params(params: &[SqlParam]) -> PyResult> { + params + .iter() + .map(|value| { + Ok(match value { + SqlParam::Null => Value::Null, + SqlParam::Text(value) => Value::Text(value.clone()), + SqlParam::Integer(value) => Value::Integer(*value), + SqlParam::Flag(value) => Value::Integer(i64::from(*value)), + SqlParam::Timestamp(_) | SqlParam::TextList(_) => { + return Err(PyValueError::new_err("Postgres-only history parameter")); + } + }) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/state_column_evolution.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs rename to crates/sqlbuild-python-base/src/warehouse_access/_helpers/state_column_evolution.rs index d9461ad1f9..11036cd094 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/state_column_evolution.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyString, PyTuple}; use sqlbuild_host::entries::models::AdapterHost; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field; +use crate::boundary::main::call_with::call_with; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::fields::field::field; /// Missing callbacks retain the existing renderer-only API for already-current/new schemas. pub(crate) fn ensure_event_id_column<'py>( diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs new file mode 100644 index 0000000000..a96d0b15f7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs @@ -0,0 +1,109 @@ +//! A plan preview's adapter: warehouse reads come from the location's mirrored copy, nothing connects. + +use pyo3::PyErr; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyModuleMethods; + +use pyo3::create_exception; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyResult, Python, pyclass, pymethods}; +use serde::Deserialize; + +create_exception!( + sqlbuild._native, + WarehouseAccessRefusedError, + PyRuntimeError, + "Raised when a plan preview reaches past the synced copy for the warehouse." +); + +/// Adapter methods that only describe the dialect, never reaching the warehouse. +pub(crate) const PASSTHROUGH_NAMES: [&str; 12] = [ + "_render_source_relation_cast_subquery_with_columns", + "adapter_name", + "default_database", + "default_schema", + "default_table_promotion_mode", + "expression_inference_profile", + "metadata_inspection_concurrency", + "python_functions_inherit_default_namespace", + "requires_derived_table_aliases", + "sql_analysis_dialect", + "sql_analysis_dialect_name", + "star_exclude_keyword", +]; +/// Rendering and capability methods pass through; every other unknown attribute is refused. +pub(crate) const PASSTHROUGH_PREFIXES: [&str; 2] = ["render_", "supports_"]; +/// Local table holding the copy's fingerprints; previews read it through per-location views. +pub(crate) const FINGERPRINT_STORE: &str = "__sqlbuild_synced_fingerprints"; +pub(crate) const FINGERPRINT_STORE_COLUMNS: [&str; 2] = ["state_database", "state_schema"]; +pub(crate) const MODEL_NODE_TYPE: &str = "model"; + +/// Items a preview reports as known only once the run plans live. +pub(crate) const PREVIEW_UNKNOWNS: [&str; 4] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", +]; + +/// A source's described columns, keyed to the exact SQL that was described. +#[derive(Deserialize)] +pub(crate) struct DescribedColumns { + pub(crate) sql_hash: String, + pub(crate) columns: Vec, +} + +pub(crate) fn refused(access: &str) -> pyo3::PyErr { + WarehouseAccessRefusedError::new_err(format!( + "plan previews read only the synced copy; refused warehouse access: {access}" + )) +} + +pub(crate) fn same(left: Option<&str>, right: Option<&str>) -> bool { + left.unwrap_or_default().to_lowercase() == right.unwrap_or_default().to_lowercase() +} + +/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. +pub(crate) fn any_same(wanted: Option<&[String]>, value: &str) -> bool { + let Some(wanted) = wanted else { + return true; + }; + for item in wanted { + if same(Some(value), Some(item)) { + return true; + } + } + false +} + +pub(crate) fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// A connection that refuses every use; like the adapter, it lives in the adapter contract's module. +#[pyclass( + frozen, + module = "sqlbuild.adapter.contract", + name = "SyncedCopyConnection" +)] +pub(crate) struct SyncedCopyConnection; + +#[pymethods] +impl SyncedCopyConnection { + pub(crate) fn __getattr__(&self, name: &str) -> PyResult<()> { + Err(refused(&format!("connection.{name}"))) + } +} + +/// Register `WarehouseAccessRefusedError` on the native module. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "WarehouseAccessRefusedError", + module.py().get_type::(), + ) +} + +/// Whether `error` is a plan preview reaching past the synced copy for the warehouse. +pub(crate) fn is_warehouse_access_refused(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs rename to crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs index e65e66dff8..2501d41f88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs @@ -10,8 +10,8 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryRecord, }; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field; +use crate::boundary::main::call_with::call_with; +use crate::records::main::fields::field::field; pub(crate) fn read_catalog( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs rename to crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs index 6fb405f2ff..544adb787c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs @@ -8,8 +8,8 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ StatisticsRead, WarehouseHistoryPage, WarehouseHistoryRecord, WarehouseStatisticsRecord, }; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field; +use crate::boundary::main::call_with::call_with; +use crate::records::main::fields::field::field; pub(crate) fn read_statistics( py: Python<'_>, diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/ensure_event_id_column.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/ensure_event_id_column.rs new file mode 100644 index 0000000000..31fd28b1f5 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/ensure_event_id_column.rs @@ -0,0 +1,26 @@ +//! Missing callbacks retain the existing renderer-only API for already-current/new schemas. + +use crate::warehouse_access::_helpers::state_column_evolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Missing callbacks retain the existing renderer-only API for already-current/new schemas. +#[inline] +pub fn ensure_event_id_column<'py>( + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (table, qualified): (&str, &str), + (read_columns, render_add, render_type): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<()> { + state_column_evolution::ensure_event_id_column( + (connection, execute), + (database, schema), + (table, qualified), + (read_columns, render_add, render_type), + ) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs new file mode 100644 index 0000000000..fe53edaac4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs @@ -0,0 +1,11 @@ +//! Whether `error` is a plan preview reaching past the synced copy for the warehouse. + +use crate::warehouse_access::_helpers::synced_copy; +use pyo3::PyErr; +use pyo3::prelude::Python; + +/// Whether `error` is a plan preview reaching past the synced copy for the warehouse. +#[inline] +pub fn is_warehouse_access_refused(py: Python<'_>, error: &PyErr) -> bool { + synced_copy::is_warehouse_access_refused(py, error) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs new file mode 100644 index 0000000000..b0a2d9b65e --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs @@ -0,0 +1,6 @@ +pub mod ensure_event_id_column; +pub mod is_warehouse_access_refused; +pub mod read_catalog; +pub mod read_statistics; +pub mod register_synced_copy; +pub mod sqlite_params; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs new file mode 100644 index 0000000000..5a8a51f1d6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs @@ -0,0 +1,19 @@ +//! Read catalog. + +use crate::warehouse_access::_helpers::warehouse_catalog; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::warehouse_metadata::models::CatalogRead; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryPage; + +/// Read catalog. +#[inline] +pub fn read_catalog( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &CatalogRead, +) -> PyResult { + warehouse_catalog::read_catalog(py, (adapter, connection), request) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs new file mode 100644 index 0000000000..b87f6fe040 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs @@ -0,0 +1,19 @@ +//! Read statistics. + +use crate::warehouse_access::_helpers::warehouse_statistics; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::warehouse_metadata::models::StatisticsRead; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryPage; + +/// Read statistics. +#[inline] +pub fn read_statistics( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &StatisticsRead, +) -> PyResult { + warehouse_statistics::read_statistics(py, (adapter, connection), request) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs new file mode 100644 index 0000000000..1f72828bb4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs @@ -0,0 +1,12 @@ +//! Register `WarehouseAccessRefusedError` on the native module. + +use crate::warehouse_access::_helpers::synced_copy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register `WarehouseAccessRefusedError` on the native module. +#[inline] +pub fn register_synced_copy(module: &Bound<'_, PyModule>) -> PyResult<()> { + synced_copy::register(module) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/sqlite_params.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/sqlite_params.rs new file mode 100644 index 0000000000..5c791277b6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/sqlite_params.rs @@ -0,0 +1,12 @@ +//! Parameters of history SQL contain no Python-owned values. + +use crate::warehouse_access::_helpers::sqlite_connection; +use pyo3::prelude::PyResult; +use rusqlite::types::Value; +use sqlbuild_state::execution_history::models::SqlParam; + +/// Parameters of history SQL contain no Python-owned values. +#[inline] +pub fn sqlite_params(params: &[SqlParam]) -> PyResult> { + sqlite_connection::sqlite_params(params) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/mod.rs new file mode 100644 index 0000000000..d4bb633bca --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse catalog, statistics and state-column reads, the native SQLite connection and the synced copy. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python-base/src/warehouse_access/models.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs rename to crates/sqlbuild-python-base/src/warehouse_access/models.rs index e3ec44cf2f..c788096175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/models.rs @@ -1,117 +1,294 @@ -//! A plan preview's adapter: warehouse reads come from the location's mirrored copy, nothing connects. - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Mutex, PoisonError}; - +//! Public records of warehouse access. + +use crate::boundary::main::call_with::call_with; +use crate::boundary::main::imported::imported; +use crate::boundary::models::KeywordArguments; +use crate::native_host::main::host_entry::host_entry; +use crate::records::main::fields::field::field as native_field; +use crate::warehouse_access::_helpers::sqlite_connection::database_error; +use crate::warehouse_access::_helpers::synced_copy::DescribedColumns; +use crate::warehouse_access::_helpers::synced_copy::FINGERPRINT_STORE; +use crate::warehouse_access::_helpers::synced_copy::FINGERPRINT_STORE_COLUMNS; +use crate::warehouse_access::_helpers::synced_copy::MODEL_NODE_TYPE; +use crate::warehouse_access::_helpers::synced_copy::PASSTHROUGH_NAMES; +use crate::warehouse_access::_helpers::synced_copy::PASSTHROUGH_PREFIXES; +use crate::warehouse_access::_helpers::synced_copy::PREVIEW_UNKNOWNS; +use crate::warehouse_access::_helpers::synced_copy::SyncedCopyConnection; +use crate::warehouse_access::_helpers::synced_copy::any_same; +use crate::warehouse_access::_helpers::synced_copy::literal; +use crate::warehouse_access::_helpers::synced_copy::refused; +use crate::warehouse_access::_helpers::synced_copy::same; use chrono::SecondsFormat; use pyo3::IntoPyObject; -use pyo3::create_exception; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyIndexError; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use serde::Deserialize; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyBytes; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use rusqlite::Connection; +use rusqlite::OpenFlags; +use rusqlite::params_from_iter; +use rusqlite::types::Value; use sqlbuild_analysis::fingerprint_storage::constants::FINGERPRINT_COLUMNS; use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; use sqlbuild_cache::digest::main::base64_text::base64_text; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; -use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::STATE_TABLE_NAMES; use sqlbuild_host::entries::models::AdapterHost; -use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord}; -use sqlbuild_state::consumption::models::{ - ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, -}; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopy; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseCatalogRecord; +use sqlbuild_state::consumption::models::ConsumedInput; +use sqlbuild_state::consumption::models::ConsumptionAppend; +use sqlbuild_state::consumption::models::ConsumptionRow; +use sqlbuild_state::consumption::models::NodeKey; +use sqlbuild_state::consumption::models::RelationKey; use sqlbuild_state::execution_history::constants::MEMORY_PATH; -use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; - -use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -create_exception!( - sqlbuild._native, - WarehouseAccessRefusedError, - PyRuntimeError, - "Raised when a plan preview reaches past the synced copy for the warehouse." -); - -/// Adapter methods that only describe the dialect, never reaching the warehouse. -const PASSTHROUGH_NAMES: [&str; 12] = [ - "_render_source_relation_cast_subquery_with_columns", - "adapter_name", - "default_database", - "default_schema", - "default_table_promotion_mode", - "expression_inference_profile", - "metadata_inspection_concurrency", - "python_functions_inherit_default_namespace", - "requires_derived_table_aliases", - "sql_analysis_dialect", - "sql_analysis_dialect_name", - "star_exclude_keyword", -]; -/// Rendering and capability methods pass through; every other unknown attribute is refused. -const PASSTHROUGH_PREFIXES: [&str; 2] = ["render_", "supports_"]; -/// Local table holding the copy's fingerprints; previews read it through per-location views. -const FINGERPRINT_STORE: &str = "__sqlbuild_synced_fingerprints"; -const FINGERPRINT_STORE_COLUMNS: [&str; 2] = ["state_database", "state_schema"]; -const MODEL_NODE_TYPE: &str = "model"; - -/// Items a preview reports as known only once the run plans live. -pub(crate) const PREVIEW_UNKNOWNS: [&str; 4] = [ - "source freshness", - "retention", - "column rename support", - "sources outside mirrored schemas", -]; - -/// A source's described columns, keyed to the exact SQL that was described. -#[derive(Deserialize)] -struct DescribedColumns { - sql_hash: String, - columns: Vec, +use sqlbuild_state::observations::constants::CURSOR_MAX_KIND; +use sqlbuild_state::observations::constants::SOURCE_COLUMNS_KIND; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::VecDeque; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteConnection")] +#[derive(Debug)] +pub struct SqliteConnection { + pub(crate) connection: Mutex>, } -fn refused(access: &str) -> pyo3::PyErr { - WarehouseAccessRefusedError::new_err(format!( - "plan previews read only the synced copy; refused warehouse access: {access}" - )) +impl SqliteConnection { + /// Open literal filenames, using absolute paths because bundled SQLite enables URI handling globally. + pub fn open(py: Python<'_>, path: &str) -> PyResult { + let flags = OpenFlags::default() & !OpenFlags::SQLITE_OPEN_URI; + let database: PathBuf = if path == MEMORY_PATH { + PathBuf::from(path) + } else { + std::env::current_dir()?.join(path) + }; + let connection = py + .detach(|| Connection::open_with_flags(database, flags)) + .map_err(|error| database_error(py, error))?; + Ok(Self { + connection: Mutex::new(Some(connection)), + }) + } + + pub(crate) fn with_connection( + &self, + py: Python<'_>, + body: impl FnOnce(&Connection) -> rusqlite::Result + Send, + ) -> PyResult { + let result = py.detach(|| { + let guard = self + .connection + .lock() + .unwrap_or_else(PoisonError::into_inner); + guard.as_ref().map(body) + }); + match result { + Some(result) => result.map_err(|error| database_error(py, error)), + None => Err(pyo3::prelude::PyErr::from_value( + imported(py, "sqlite3", "ProgrammingError")? + .call1(("Cannot operate on a closed database.",))?, + )), + } + } + + pub fn execute_native( + &self, + py: Python<'_>, + sql: &str, + params: &[Value], + ) -> PyResult { + self.with_connection(py, |connection| { + let mut statement = connection.prepare_cached(sql)?; + let columns = Arc::new( + statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(), + ); + let count: usize = statement.column_count(); + let mut cursor = statement.query(params_from_iter(params))?; + let mut rows: VecDeque = VecDeque::new(); + while let Some(row) = cursor.next()? { + rows.push_back(SqliteRow { + columns: Arc::clone(&columns), + values: (0..count) + .map(|index| row.get(index)) + .collect::>()?, + }); + } + Ok(SqliteCursor { + rows: Mutex::new(rows), + last_row_id: connection.last_insert_rowid(), + }) + }) + } } -fn same(left: Option<&str>, right: Option<&str>) -> bool { - left.unwrap_or_default().to_lowercase() == right.unwrap_or_default().to_lowercase() +#[pymethods] +impl SqliteConnection { + #[pyo3(signature = (sql, parameters = None))] + pub(crate) fn execute( + &self, + py: Python<'_>, + sql: &str, + parameters: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let mut params: Vec = Vec::new(); + if let Some(parameters) = parameters { + for value in parameters.try_iter()? { + let value = value?; + params.push(if value.is_none() { + Value::Null + } else if let Ok(value) = value.extract::() { + Value::Integer(value) + } else { + Value::Text(value.extract()?) + }); + } + } + self.execute_native(py, sql, ¶ms) + } + + pub(crate) fn executemany( + &self, + py: Python<'_>, + sql: &str, + parameters: &Bound<'_, PyAny>, + ) -> PyResult { + let mut result = SqliteCursor { + rows: Mutex::default(), + last_row_id: 0, + }; + for row in parameters.try_iter()? { + result = self.execute(py, sql, Some(&row?))?; + } + Ok(result) + } + + #[getter] + pub fn in_transaction(&self, py: Python<'_>) -> PyResult { + self.with_connection(py, |connection| Ok(!connection.is_autocommit())) + } + + pub fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + self.connection + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + }); + Ok(()) + } +} + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] +#[derive(Debug)] +pub struct SqliteRow { + pub columns: Arc>, + pub values: Vec, +} + +impl SqliteRow { + pub fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { + match self.values.get(index) { + Some(Value::Null) => Ok(py.None().into_bound(py)), + Some(Value::Integer(value)) => value.into_bound_py_any(py), + Some(Value::Real(value)) => value.into_bound_py_any(py), + Some(Value::Text(value)) => value.into_bound_py_any(py), + Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), + None => Err(PyIndexError::new_err("tuple index out of range")), + } + } + + pub fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let index: usize = self + .columns + .iter() + .position(|column| column.eq_ignore_ascii_case(name)) + .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; + self.value(py, index) + } } -/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. -fn any_same(wanted: Option<&[String]>, value: &str) -> bool { - let Some(wanted) = wanted else { - return true; - }; - for item in wanted { - if same(Some(value), Some(item)) { - return true; +#[pymethods] +impl SqliteRow { + pub(crate) fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(index) = key.extract::() { + let index = if index < 0 { + self.values.len().checked_add_signed(index) + } else { + Some(index as usize) + }; + return self.value(key.py(), index.unwrap_or(usize::MAX)); + } + if let Ok(name) = key.extract::() { + return self.column(key.py(), &name); } + Err(PyTypeError::new_err("Index must be int or string")) } - false } -fn literal(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] +#[derive(Debug)] +pub struct SqliteCursor { + pub rows: Mutex>, + pub last_row_id: i64, } -/// A connection that refuses every use; like the adapter, it lives in the adapter contract's module. -#[pyclass( - frozen, - module = "sqlbuild.adapter.contract", - name = "SyncedCopyConnection" -)] -pub(crate) struct SyncedCopyConnection; +impl SqliteCursor { + pub fn next_row(&self) -> Option { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop_front() + } +} #[pymethods] -impl SyncedCopyConnection { - fn __getattr__(&self, name: &str) -> PyResult<()> { - Err(refused(&format!("connection.{name}"))) +impl SqliteCursor { + pub(crate) fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + pub(crate) fn __next__(&self) -> Option { + self.next_row() + } + + pub(crate) fn fetchone(&self) -> Option { + self.next_row() + } + + pub(crate) fn fetchall(&self) -> Vec { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain(..) + .collect() + } + + #[getter] + pub(crate) fn lastrowid(&self) -> i64 { + self.last_row_id } } @@ -121,18 +298,19 @@ impl SyncedCopyConnection { module = "sqlbuild.adapter.contract", name = "SyncedCopyAdapter" )] -pub(crate) struct SyncedCopyAdapter { - adapter: Py, - copy: PreviewCopy, - store: SqliteConnection, - views: Mutex, String)>>, - unknowns: Mutex>, - windows: Mutex>, +#[derive(Debug)] +pub struct SyncedCopyAdapter { + pub(crate) adapter: Py, + pub(crate) copy: PreviewCopy, + pub(crate) store: SqliteConnection, + pub(crate) views: Mutex, String)>>, + pub(crate) unknowns: Mutex>, + pub(crate) windows: Mutex>, } impl SyncedCopyAdapter { /// The facade over `adapter` answering from the copy written at `path`. - pub(crate) fn load(py: Python<'_>, adapter: Bound<'_, PyAny>, path: &str) -> PyResult { + pub fn load(py: Python<'_>, adapter: Bound<'_, PyAny>, path: &str) -> PyResult { let text: String = std::fs::read_to_string(path)?; let copy: PreviewCopy = serde_json::from_str(&text) .map_err(|error| PyRuntimeError::new_err(format!("invalid synced copy: {error}")))?; @@ -188,14 +366,14 @@ impl SyncedCopyAdapter { } /// When the copy was synced, as ISO 8601 text; `None` before every required table synced. - pub(crate) fn synced_at(&self) -> Option { + pub fn synced_at(&self) -> Option { self.copy .synced_at .map(|at| at.to_rfc3339_opts(SecondsFormat::AutoSi, true)) } /// What the preview could not know, in a stable order. - pub(crate) fn unknowns(&self) -> Vec { + pub fn unknowns(&self) -> Vec { let mut unknowns: Vec = PREVIEW_UNKNOWNS .iter() .map(|item| (*item).to_owned()) @@ -211,7 +389,7 @@ impl SyncedCopyAdapter { } /// Note one item the preview reports as unknown until the run plans live. - pub(crate) fn mark_unknown(&self, item: String) { + pub fn mark_unknown(&self, item: String) { self.unknowns .lock() .unwrap_or_else(PoisonError::into_inner) @@ -219,7 +397,7 @@ impl SyncedCopyAdapter { } /// Note that `model` reads from `start` up to the latest data when the run starts. - pub(crate) fn mark_window(&self, model: &str, start: &str) { + pub fn mark_window(&self, model: &str, start: &str) { self.windows .lock() .unwrap_or_else(PoisonError::into_inner) @@ -227,7 +405,7 @@ impl SyncedCopyAdapter { } /// Each model's window start, read up to the latest data when the run starts. - pub(crate) fn windows(&self) -> BTreeMap { + pub fn windows(&self) -> BTreeMap { self.windows .lock() .unwrap_or_else(PoisonError::into_inner) @@ -235,7 +413,7 @@ impl SyncedCopyAdapter { } /// The model's cursor `MAX` text, only when observed by its newest successful build. - pub(crate) fn target_max(&self, model: &str) -> Option<&str> { + pub fn target_max(&self, model: &str) -> Option<&str> { for run in &self.copy.latest_runs { if run.node_type != MODEL_NODE_TYPE || run.node_name != model { continue; @@ -255,10 +433,7 @@ impl SyncedCopyAdapter { } /// Every copied consumption row of the table in `(database, schema)`. - pub(crate) fn consumption( - &self, - (database, schema): (Option<&str>, &str), - ) -> Vec { + pub fn consumption(&self, (database, schema): (Option<&str>, &str)) -> Vec { let mut rows: Vec = Vec::new(); for record in &self.copy.consumption { if !same(Some(&record.state_schema), Some(schema)) @@ -298,11 +473,7 @@ impl SyncedCopyAdapter { } /// The source's columns as last described, only when it described exactly `sql`. - pub(crate) fn described_columns( - &self, - source: &str, - sql: &str, - ) -> PyResult>> { + pub fn described_columns(&self, source: &str, sql: &str) -> PyResult>> { let hash: String = sha256_hex(sql.as_bytes()); for observation in &self.copy.observations { if observation.kind != SOURCE_COLUMNS_KIND || observation.node != source { @@ -322,7 +493,7 @@ impl SyncedCopyAdapter { } /// The relations a listing call named by its `database`, `schemas` and optional `names`. - fn listed_by<'py>( + pub(crate) fn listed_by<'py>( &self, py: Python<'py>, function: &str, @@ -340,7 +511,7 @@ impl SyncedCopyAdapter { Ok(self.listed(database.as_deref(), schemas.as_deref(), names.as_deref())) } - fn listed( + pub(crate) fn listed( &self, database: Option<&str>, schemas: Option<&[String]>, @@ -363,7 +534,7 @@ impl SyncedCopyAdapter { listed } - fn relation_info<'py>( + pub(crate) fn relation_info<'py>( &self, py: Python<'py>, record: &WarehouseCatalogRecord, @@ -389,7 +560,7 @@ impl SyncedCopyAdapter { ) } - fn columns<'py>( + pub(crate) fn columns<'py>( &self, py: Python<'py>, record: &WarehouseCatalogRecord, @@ -412,18 +583,21 @@ impl SyncedCopyAdapter { #[pymethods] impl SyncedCopyAdapter { #[pyo3(signature = (connection_config = None))] - fn connect(&self, connection_config: Option>) -> SyncedCopyConnection { + pub(crate) fn connect( + &self, + connection_config: Option>, + ) -> SyncedCopyConnection { drop(connection_config); SyncedCopyConnection } #[pyo3(signature = (connection = None))] - fn close(&self, connection: Option>) { + pub(crate) fn close(&self, connection: Option>) { drop(connection); } #[pyo3(name = "list_relations", signature = (**options))] - fn relation_listing<'py>( + pub(crate) fn relation_listing<'py>( &self, py: Python<'py>, options: Option>, @@ -437,7 +611,7 @@ impl SyncedCopyAdapter { } #[pyo3(name = "get_all_columns", signature = (**options))] - fn all_columns<'py>( + pub(crate) fn all_columns<'py>( &self, py: Python<'py>, options: Option>, @@ -450,7 +624,7 @@ impl SyncedCopyAdapter { } #[pyo3(name = "get_columns_for_relations", signature = (*, connection, relations))] - fn columns_by_relation<'py>( + pub(crate) fn columns_by_relation<'py>( &self, py: Python<'py>, connection: Bound<'py, PyAny>, @@ -477,7 +651,7 @@ impl SyncedCopyAdapter { /// The latest-row read over this location's copied fingerprints, through a local view. #[pyo3(signature = (*, database, schema))] - fn render_read_latest_fingerprints_sql( + pub(crate) fn render_read_latest_fingerprints_sql( &self, py: Python<'_>, database: Option, @@ -515,7 +689,7 @@ impl SyncedCopyAdapter { /// Only reads of the copied fingerprints run; any other statement would reach the warehouse. #[pyo3(signature = (*, connection, sql))] - fn execute( + pub(crate) fn execute( &self, py: Python<'_>, connection: Bound<'_, PyAny>, @@ -529,7 +703,11 @@ impl SyncedCopyAdapter { } #[pyo3(signature = (*, connection, sql))] - fn query_column_names(&self, connection: Bound<'_, PyAny>, sql: &str) -> PyResult<()> { + pub(crate) fn query_column_names( + &self, + connection: Bound<'_, PyAny>, + sql: &str, + ) -> PyResult<()> { drop(connection); Err(refused(&format!( "DESCRIBE {}", @@ -538,7 +716,11 @@ impl SyncedCopyAdapter { } /// Values and dialect methods come from the real adapter; any other method is refused. - fn __getattr__<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + pub(crate) fn __getattr__<'py>( + &self, + py: Python<'py>, + name: &str, + ) -> PyResult> { let value: Bound<'py, PyAny> = native_field(self.adapter.bind(py), name)?; if !value.is_callable() || PASSTHROUGH_NAMES.contains(&name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 04ebc46920..256fbc8e10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::analysis_session::session::{ start_model_analysis_session, }; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; -use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; @@ -29,6 +28,8 @@ use crate::bindings::_helpers::compiled_views::model_input_view::{ use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::cache::main::content_digest::content_digest; +use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -235,7 +236,7 @@ fn run_session<'py>( .finished_session() .and_then(|finished| finished.relation_tables()) { - crate::bindings::_helpers::compile_context::delta::capture(|capture| { + sqlbuild_python_base::compile_context::main::delta::capture::capture(|capture| { capture.relations = Some(tables.clone()); }); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 9943002f46..af5650a37b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -38,10 +38,11 @@ use std::path::PathBuf; use crate::bindings::_helpers::analysis_session::nullability_rules::{ RuleFailure, nullability_callback, raised, }; -use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; use crate::bindings::models::Positional; use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::cache::main::open_store::open_store; +use sqlbuild_python_base::cache::main::save_store::save_store; use sqlbuild_python_base::types::CompilerDetach; /// `(name, type, nullability)`. @@ -370,9 +371,9 @@ pub(crate) fn start_model_analysis_session( ) -> PyResult { let py: Python<'_> = catalog.py(); let mut request: SessionRequest = - crate::bindings::_helpers::compile_context::delta::inject_relations(session_request( - request, - ))?; + sqlbuild_python_base::compile_context::main::delta::inject_relations::inject_relations( + session_request(request), + )?; let rule_failure: RuleFailure = RuleFailure::default(); let dialect: String = request.dialect.clone(); request.nullability_callback = adapter_rules @@ -381,8 +382,9 @@ pub(crate) fn start_model_analysis_session( let mut session: AnalysisSession = py .compiler_detach(|| start_analysis_session(request, catalog)) .map_err(|reason| internal_error(&reason))?; - let cache: Option<(PathBuf, String)> = - cache.filter(|_| crate::bindings::_helpers::compile_context::delta::injection().is_none()); + let cache: Option<(PathBuf, String)> = cache.filter(|_| { + sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() + }); let mut cache_path: Option = None; let mut cache_stats: Option = None; if let Some((path, environment)) = cache { @@ -487,7 +489,8 @@ fn session_request(request: RequestRow) -> SessionRequest { dynamic_families_by_table: families_by_table(dynamic_families_by_table), models: models.into_iter().map(model_request).collect(), forced_order: None, - retain_relations: crate::bindings::_helpers::compile_context::delta::capturing(), + retain_relations: sqlbuild_python_base::compile_context::main::delta::capturing::capturing( + ), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 393504c715..ef9222d133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -19,12 +19,12 @@ use crate::bindings::_helpers::analysis_session::session::{ ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, }; -use crate::bindings::_helpers::compile_driver::stage_work::{ - EXPRESSION_SHAPE_BATCHES, EXPRESSION_SHAPES_INFERRED, count_stage, -}; use crate::bindings::_helpers::model_config::header_families::family_specs; use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; +use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPES_INFERRED; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 8f5126599a..84e2457856 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -17,7 +17,6 @@ use sqlbuild_host::models::{AuditExecutionResult, AuditOutcome}; use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressReporter; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; @@ -27,7 +26,6 @@ use crate::bindings::_helpers::compile_driver::static_context::compile_static_co use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; use crate::bindings::_helpers::result_output::documents::{ audit_execution_json, audit_projection, write_execution_json_output, @@ -38,7 +36,9 @@ use crate::bindings::_helpers::result_output::resource_ids::audit_identity; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs index 2ca1179c20..e15b03c0c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs @@ -7,17 +7,14 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::auditing::main::evaluate_measurement::evaluate_measurement as evaluate; use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds}; use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; fn measurement_error(py: Python<'_>, message: &str) -> PyErr { - match error_class(py, ErrorId::MeasurementAudit).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::MeasurementAudit, message) } /// The canonical outcome for one finite measured value. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index 163924d25f..3d2fafc277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -1,13 +1,13 @@ //! Project variable values rendered as the text SQL and config interpolation inserts. use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; const JSON_MODULE: &str = "json"; /// Characters of a structured value's JSON preview before it is cut short. @@ -16,10 +16,7 @@ const PREVIEW_ELLIPSIS: &str = "..."; /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: String) -> PyErr { - match host_entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, FacadeHost::SharedInputError, &message) } /// Compact, key-sorted JSON of `value`, cut to the preview limit. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs index 8f3a9cb9f1..aa668a4b4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -12,6 +12,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; use sqlbuild_model_config::header_metadata::main::json_meta_problem::json_meta_problem; +use sqlbuild_python_base::native_host::main::host_error::host_error; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use sqlbuild_python_base::boundary::main::imported::imported; @@ -37,10 +38,7 @@ const HOOK_READS_INVALID: &str = /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: &str) -> PyErr { - match host_entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, FacadeHost::SharedInputError, message) } /// `value < bound` under Python's comparison. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 2de90ace5e..1ecd6227f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -18,10 +18,8 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; -use crate::bindings::_helpers::build_command::support::{frozenset, tuple}; use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; -use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::discovery::provider_secrets::{ consumer_uses, provider_uses, require_used_secrets, sink_uses, }; @@ -32,6 +30,9 @@ use crate::bindings::_helpers::observations::source_capture::{ use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_command::main::frozenset::frozenset; +use sqlbuild_python_base::build_command::main::tuple::tuple; +use sqlbuild_python_base::diagnostics::models::BuildPhaseTimingTracker; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 2bfa001e94..05ee0cc822 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -16,9 +16,6 @@ use crate::bindings::_helpers::build_command::execution::{ Outcome, Preparation, write_run_context, }; use crate::bindings::_helpers::build_command::invocation::BuildContext; -use crate::bindings::_helpers::build_command::support::{ - boolean, emit, joined, none, text, utc_now, -}; use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::check::check_results::check_result_text; @@ -37,6 +34,12 @@ use sqlbuild_core::build_progress::models::PythonNodeText; use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::joined::joined; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::build_command::main::utc_now::utc_now; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 4cebbd294d..294c8c5da0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -15,9 +15,6 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::lifecycle::{ DirectPythonLifecycle, LifecycleInputs, prepare_direct_python_lifecycle, }; -use crate::bindings::_helpers::build_command::support::{ - boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, -}; use crate::bindings::_helpers::build_progress::build_callbacks::{ BuildProgressCallbacks, batch_complete_callback, check_callback, }; @@ -39,7 +36,6 @@ use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timest use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; @@ -47,7 +43,18 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::display_text::display_text; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::frozenset::frozenset; +use sqlbuild_python_base::build_command::main::joined::joined; +use sqlbuild_python_base::build_command::main::keyword_relay::keyword_relay; +use sqlbuild_python_base::build_command::main::line_writer::line_writer; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index cff18feb3c..5556b457d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -10,10 +10,8 @@ use sqlbuild_core::build_command::main::duration_limit_error::duration_limit_err use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; -use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; @@ -23,9 +21,13 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::or_none::or_none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index c0b39e6e81..8ab3d82fbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -12,9 +12,6 @@ use sqlbuild_core::build_command::models::PythonNodeRow; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_command::support::{ - boolean, emit, frozenset, or_none, text, tuple, -}; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; @@ -23,6 +20,12 @@ use crate::bindings::_helpers::planner_scope::loaders::build_intermediate_source use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::frozenset::frozenset; +use sqlbuild_python_base::build_command::main::or_none::or_none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs index 643665def4..38f57e0e67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs @@ -7,4 +7,3 @@ pub(crate) mod lifecycle; pub(crate) mod planning; pub(crate) mod policies; pub(crate) mod registration; -pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 1b9feaae1f..b230110d6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -11,9 +11,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, model_limit}; -use crate::bindings::_helpers::build_command::support::{ - boolean, emit, joined, keyword_relay, none, -}; use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; @@ -21,6 +18,11 @@ use crate::bindings::_helpers::manifest::builder::build_manifest; use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::joined::joined; +use sqlbuild_python_base::build_command::main::keyword_relay::keyword_relay; +use sqlbuild_python_base::build_command::main::none::none; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs index f093ac4b7e..4494babe1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs @@ -17,12 +17,19 @@ use sqlbuild_core::nested_progress::main::render_group_header::render_group_head use sqlbuild_core::nested_progress::models::AuditAggregate; use crate::bindings::_helpers::build_progress::result_text::{audit_text, optional, text}; -use crate::bindings::_helpers::build_progress::terminal::{ - CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, claim, consume_terminal, flush, projector, release, - shared_lines, tty_flag, write, -}; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; +use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; +use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; +use sqlbuild_python_base::build_progress::main::terminal::claim::claim; +use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; +use sqlbuild_python_base::build_progress::main::terminal::flush::flush; +use sqlbuild_python_base::build_progress::main::terminal::projector::projector; +use sqlbuild_python_base::build_progress::main::terminal::release::release; +use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; +use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; +use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The identity an audit plan entry or result reports to the progress projector. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index 15f456c54c..2fda521d2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -25,16 +25,27 @@ use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, buil use crate::bindings::_helpers::build_progress::result_text::{ function_text, load_text, model_text, optional, seed_text, test_text, text, }; -use crate::bindings::_helpers::build_progress::terminal::{ - CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, - consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, - terminal_columns, tty_flag, write, -}; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; +use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; +use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; +use sqlbuild_python_base::build_progress::constants::SPINNER_FRAMES; +use sqlbuild_python_base::build_progress::constants::SPINNER_TICK_SECONDS; +use sqlbuild_python_base::build_progress::main::terminal::claim::claim; +use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; +use sqlbuild_python_base::build_progress::main::terminal::flush::flush; +use sqlbuild_python_base::build_progress::main::terminal::join_spinner::join_spinner; +use sqlbuild_python_base::build_progress::main::terminal::projector::projector; +use sqlbuild_python_base::build_progress::main::terminal::release::release; +use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; +use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; +use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; +use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; +use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs index 40ac4d0284..cc0e980ca6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs @@ -4,7 +4,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::build_progress::build_callbacks::BuildProgressCallbacks; -use crate::bindings::_helpers::build_progress::transient_lines::TransientLineCoordinator; +use sqlbuild_python_base::build_progress::models::TransientLineCoordinator; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index be876d6c6e..dc06fff1c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -4,11 +4,7 @@ pub(crate) mod footer; pub(crate) mod getters; pub(crate) mod nested_progress; pub(crate) mod plan_facts; -pub(crate) mod progress; pub(crate) mod progress_reporters; pub(crate) mod progress_text; pub(crate) mod registration; pub(crate) mod result_text; -pub(crate) mod terminal; -pub(crate) mod transient_lines; -pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs index 3bd9549f24..1220615519 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs @@ -13,13 +13,24 @@ use sqlbuild_core::nested_progress::main::render_nested_spinner_line::render_nes use sqlbuild_core::nested_progress::models::{NestedChildRow, NestedItem, NestedLayout}; use crate::bindings::_helpers::build_progress::result_text::text; -use crate::bindings::_helpers::build_progress::terminal::{ - CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, - consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, - terminal_columns, tty_flag, write, -}; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; +use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; +use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; +use sqlbuild_python_base::build_progress::constants::SPINNER_FRAMES; +use sqlbuild_python_base::build_progress::constants::SPINNER_TICK_SECONDS; +use sqlbuild_python_base::build_progress::main::terminal::claim::claim; +use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; +use sqlbuild_python_base::build_progress::main::terminal::flush::flush; +use sqlbuild_python_base::build_progress::main::terminal::join_spinner::join_spinner; +use sqlbuild_python_base::build_progress::main::terminal::projector::projector; +use sqlbuild_python_base::build_progress::main::terminal::release::release; +use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; +use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; +use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; +use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; +use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MIN_NAME_WIDTH: usize = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs index 196f6b43f5..91b3157fda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs @@ -13,11 +13,12 @@ use sqlbuild_core::progress::main::connection_start_message::connection_start_me use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; use sqlbuild_core::progress::main::is_planning_failure_message::is_planning_failure_message; -use crate::bindings::_helpers::build_progress::progress::{ - NativeProgressProjector, current_projector, projector_operation_active, write_persistent, -}; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::build_progress::main::progress::projector_operation_active::projector_operation_active; +use sqlbuild_python_base::build_progress::main::progress::write_persistent::write_persistent; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::build_progress::models::NativeProgressProjector; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["adapter_name", "stream"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs index 31f51f91c0..15422b3ba7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs @@ -16,9 +16,9 @@ use sqlbuild_core::progress::models::{ExpectationStep, InspectionRecord}; use sqlbuild_core::progress::types::DifferenceSample; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::build_progress::progress::write_persistent; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::progress::write_persistent::write_persistent; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 4330892110..36128e91a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -3,16 +3,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, terminal, transient_lines, - transient_status, + audit_progress, build_callbacks, footer, nested_progress, }; +use sqlbuild_python_base::build_progress::main::terminal::register_terminal::register_terminal; +use sqlbuild_python_base::build_progress::main::transient_lines::register_transient_lines::register_transient_lines; +use sqlbuild_python_base::build_progress::main::transient_status::register_transient_status::register_transient_status; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audit_progress::register(module)?; build_callbacks::register(module)?; footer::register(module)?; nested_progress::register(module)?; - terminal::register(module)?; - transient_lines::register(module)?; - transient_status::register(module) + register_terminal(module)?; + register_transient_lines(module)?; + register_transient_status(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs index 80649ee353..6009f9bdc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -25,10 +25,10 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; -use crate::bindings::_helpers::result_output::integration_values::json_value; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::result_output::main::json_value::json_value; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs deleted file mode 100644 index f0d7ca70dd..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs +++ /dev/null @@ -1,141 +0,0 @@ -//! The coordinator of one live transient terminal line and persistent writes. - -use std::sync::{Mutex, MutexGuard, PoisonError}; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; - -use crate::bindings::_helpers::build_progress::terminal::tty_flag; -use sqlbuild_python_base::boundary::main::imported::imported; - -/// The stream a transient line is drawn on and the renderer that owns it. -type ActiveLine = (Py, Py); - -static SHARED_COORDINATOR: PyOnceLock> = PyOnceLock::new(); - -/// Keep persistent terminal lines from landing on a live spinner or status line. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TransientLineCoordinator { - /// The re-entrant lock every writer holds while it draws. - #[pyo3(get)] - pub(crate) lock: Py, - active: Mutex>, -} - -impl TransientLineCoordinator { - fn active(&self) -> MutexGuard<'_, Option> { - self.active.lock().unwrap_or_else(PoisonError::into_inner) - } - - /// Run `body` while holding the coordinator lock. - fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { - let lock: &Bound<'_, PyAny> = self.lock.bind(py); - lock.call_method0("acquire")?; - let result: PyResult = body(); - lock.call_method0("release")?; - result - } - - /// The owner whose line a write to `stream` would land on. - fn interrupted_owner<'py>( - &self, - stream: &Bound<'py, PyAny>, - ) -> PyResult>> { - let py: Python<'py> = stream.py(); - let Some((owner_stream, owner)) = self - .active() - .as_ref() - .map(|(owner_stream, owner)| (owner_stream.bind(py).clone(), owner.bind(py).clone())) - else { - return Ok(None); - }; - if owner_stream.is(stream) || (tty_flag(&owner_stream)? && tty_flag(stream)?) { - return Ok(Some(owner)); - } - Ok(None) - } -} - -#[pymethods] -impl TransientLineCoordinator { - #[new] - fn new(py: Python<'_>) -> PyResult { - Ok(Self { - lock: imported(py, "threading", "RLock")?.call0()?.unbind(), - active: Mutex::new(None), - }) - } - - /// Record the owner of the transient line currently drawn on a terminal stream. - #[pyo3(signature = (*, stream, owner))] - fn claim(&self, stream: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { - self.locked(stream.py(), || { - *self.active() = Some((stream.clone().unbind(), owner.clone().unbind())); - Ok(()) - }) - } - - /// Forget a transient owner once its line is no longer drawn. - #[pyo3(signature = (*, owner))] - fn release(&self, owner: &Bound<'_, PyAny>) -> PyResult<()> { - self.locked(owner.py(), || { - let mut active = self.active(); - if active - .as_ref() - .is_some_and(|(_, current)| current.is(owner)) - { - *active = None; - } - Ok(()) - }) - } - - /// Erase and forget any live transient line, so nothing redraws it afterwards. - fn clear_active(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || { - let taken: Option = self.active().take(); - if let Some((_, owner)) = taken { - owner.bind(py).call_method0("clear_transient_line")?; - } - Ok(()) - }) - } - - /// Write lasting text, clearing and then redrawing any live transient line. - #[pyo3(signature = (*, stream, text))] - fn write_persistent(&self, stream: &Bound<'_, PyAny>, text: &Bound<'_, PyAny>) -> PyResult<()> { - self.locked(stream.py(), || { - let owner: Option> = self.interrupted_owner(stream)?; - if let Some(owner) = &owner { - owner.call_method0("clear_transient_line")?; - } - stream.call_method1("write", (text,))?; - stream.call_method0("flush")?; - if let Some(owner) = &owner { - owner.call_method0("redraw_transient_line")?; - } - Ok(()) - }) - } -} - -/// The coordinator shared by every terminal writer in this process. -pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { - SHARED_COORDINATOR - .get_or_try_init(py, || Py::new(py, TransientLineCoordinator::new(py)?)) - .map(|coordinator| coordinator.bind(py).clone()) -} - -/// Return the coordinator shared by every terminal writer in this process. -#[pyfunction] -pub(crate) fn shared_transient_line_coordinator( - py: Python<'_>, -) -> PyResult> { - shared_coordinator(py) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(shared_transient_line_coordinator, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs deleted file mode 100644 index 06e35b6b16..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ /dev/null @@ -1,384 +0,0 @@ -//! One transient status line with a spinner on a terminal and plain lines elsewhere. - -use std::sync::mpsc::{self, RecvTimeoutError}; -use std::sync::{Mutex, MutexGuard, PoisonError}; -use std::time::Duration; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::progress::main::format_status_spinner_line::format_status_spinner_line; - -use crate::bindings::_helpers::build_progress::terminal::{ - CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, flush, - join_spinner, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, -}; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const NO_PROGRESS_VARIABLE: &str = "SQLBUILD_NO_PROGRESS"; -const PROGRESS_DISABLED_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; -const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; - -/// Mutable reporter state; only held briefly and never across Python calls. -#[derive(Default)] -struct StatusState { - message: Option, - active: bool, - drawn: bool, - frame: usize, - stop_event: Option>, - threads: Vec>, - exit_hook_registered: bool, -} - -/// Render one transient status line with optional spinner updates. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TransientStatusReporter { - stream: Py, - lines: Py, - lock: Py, - style: CliStyle, - enabled: bool, - state: Mutex, -} - -impl TransientStatusReporter { - fn state(&self) -> MutexGuard<'_, StatusState> { - self.state.lock().unwrap_or_else(PoisonError::into_inner) - } - - fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { - within(self.lock.bind(py), |_| body()) - } - - pub(crate) fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("stream", self.stream.bind(py))?; - options.set_item("text", text)?; - self.lines - .bind(py) - .call_method("write_persistent", (), Some(&options))?; - Ok(()) - } - - fn write_message(&self, py: Python<'_>, message: &str, dim_output: bool) -> PyResult<()> { - let formatted: String = if dim_output { - self.style.muted(message) - } else { - message.to_owned() - }; - self.write_persistent(py, &format!("{formatted}\n")) - } - - fn draw(&self, py: Python<'_>) -> PyResult<()> { - let (message, frame) = { - let mut state = self.state(); - let Some(message) = state.message.clone().filter(|_| state.drawn) else { - return Ok(()); - }; - let frame: usize = state.frame; - state.frame = (frame + 1) % SPINNER_FRAMES.len(); - (message, frame) - }; - let line: String = format_status_spinner_line( - SPINNER_FRAMES[frame], - &message, - terminal_columns(py)?, - self.style, - ); - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, &format!("{CLEAR_LINE}{line}"))?; - flush(stream) - } - - fn enter_status(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - let register_hook: bool = - !std::mem::replace(&mut reporter.state().exit_hook_registered, true); - if register_hook { - imported(py, "atexit", "register")?.call1((native_field(slf, "close")?,))?; - } - let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); - write(stream, HIDE_CURSOR)?; - reporter.state().drawn = true; - reporter.draw(py)?; - let (event, thread) = start_spinner(slf.as_any())?; - let mut state = reporter.state(); - state.stop_event = Some(event.unbind()); - state.threads.push(thread.unbind()); - Ok(()) - } - - fn close_status(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - reporter.locked(py, || { - release(reporter.lines.bind(py), slf.as_any())?; - reporter.exit_status(py)?; - let mut state = reporter.state(); - state.message = None; - state.active = false; - Ok(()) - }) - } - - fn exit_status(&self, py: Python<'_>) -> PyResult<()> { - let (drawn, event) = { - let mut state = self.state(); - (std::mem::take(&mut state.drawn), state.stop_event.take()) - }; - if let Some(event) = event { - event.bind(py).call_method0("set")?; - } - if drawn { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; - flush(stream)?; - } - Ok(()) - } -} - -#[pymethods] -impl TransientStatusReporter { - #[new] - #[pyo3(signature = (*, stream, use_color = false, enabled = true))] - pub(crate) fn new( - py: Python<'_>, - stream: &Bound<'_, PyAny>, - use_color: bool, - enabled: bool, - ) -> PyResult { - let lines: Bound<'_, PyAny> = shared_lines(py)?; - Ok(Self { - enabled: enabled && !progress_spinners_disabled() && tty_flag(stream)?, - stream: stream.clone().unbind(), - lock: native_field(&lines, "lock")?.unbind(), - lines: lines.unbind(), - style: CliStyle { use_color }, - state: Mutex::new(StatusState::default()), - }) - } - - pub(crate) fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - if reporter.enabled { - return reporter.locked(py, || { - Self::close_status(slf)?; - reporter.state().message = Some(message); - Self::enter_status(slf)?; - claim( - reporter.lines.bind(py), - reporter.stream.bind(py), - slf.as_any(), - )?; - reporter.state().active = true; - Ok(()) - }); - } - reporter.state().active = true; - reporter.write_message(py, &message, true) - } - - pub(crate) fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { - if self.enabled { - let updated: bool = self.locked(py, || { - let drawn: bool = { - let mut state = self.state(); - if state.message.is_none() { - return Ok(false); - } - state.message = Some(message.clone()); - state.drawn - }; - if drawn { - self.draw(py)?; - } - Ok(true) - })?; - if updated { - return Ok(()); - } - } - self.write_message(py, &message, true) - } - - #[pyo3(signature = (*, message, blank_line_after = false))] - pub(crate) fn complete( - slf: &Bound<'_, Self>, - message: &str, - blank_line_after: bool, - ) -> PyResult<()> { - let py: Python<'_> = slf.py(); - Self::close(slf)?; - slf.get().write_message(py, message, true)?; - if blank_line_after { - slf.get().write_blank_line(py)?; - } - Ok(()) - } - - /// Complete with a pre-styled message written verbatim. - #[pyo3(signature = (*, message, blank_line_after = false))] - pub(crate) fn complete_styled( - slf: &Bound<'_, Self>, - message: &str, - blank_line_after: bool, - ) -> PyResult<()> { - Self::close(slf)?; - let blank: &str = if blank_line_after { "\n" } else { "" }; - slf.get() - .write_persistent(slf.py(), &format!("{message}\n{blank}")) - } - - pub(crate) fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { - Self::close(slf)?; - slf.get().write_message(slf.py(), message, false) - } - - fn close(slf: &Bound<'_, Self>) -> PyResult<()> { - Self::close_status(slf)?; - let threads: Vec> = std::mem::take(&mut slf.get().state().threads); - for thread in threads { - join_spinner(thread.bind(slf.py()))?; - } - Ok(()) - } - - /// Erase the live status so a persistent line can be written in its place; the spinner idles. - fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { - if std::mem::take(&mut self.state().drawn) { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; - flush(stream)?; - } - Ok(()) - } - - /// Draw the live status again below persistent output, reusing its running spinner. - fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { - let reporter: &Self = slf.get(); - let (redraw, spinning) = { - let state = reporter.state(); - ( - state.message.is_some() && !state.drawn, - state.stop_event.is_some(), - ) - }; - if !redraw { - return Ok(()); - } - if !spinning { - return Self::enter_status(slf); - } - write(reporter.stream.bind(slf.py()), HIDE_CURSOR)?; - reporter.state().drawn = true; - reporter.draw(slf.py()) - } - - /// Render one preflight progress update. - fn report_preflight_progress(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { - if message.starts_with(PREFLIGHT_COMPLETION_PREFIX) { - return Self::complete(slf, &message, true); - } - if !slf.get().state().active { - return Self::start(slf, message); - } - slf.get().update(slf.py(), message) - } - - fn write_blank_line(&self, py: Python<'_>) -> PyResult<()> { - self.write_persistent(py, "\n") - } - - fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { - while !stop_event - .call_method1("wait", (SPINNER_TICK_SECONDS,))? - .is_truthy()? - { - self.locked(py, || { - if stop_event.call_method0("is_set")?.is_truthy()? { - return Ok(()); - } - self.draw(py) - })?; - } - Ok(()) - } -} - -/// Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. -pub(crate) fn progress_spinners_disabled() -> bool { - std::env::var(NO_PROGRESS_VARIABLE) - .is_ok_and(|value| PROGRESS_DISABLED_VALUES.contains(&value.trim().to_lowercase().as_str())) -} - -/// A spinner reporter on stderr, or `None` when stderr is not a terminal or progress is off. -fn stderr_reporter(py: Python<'_>) -> PyResult>> { - let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; - if progress_spinners_disabled() || !tty_flag(&stderr)? { - return Ok(None); - } - Py::new(py, TransientStatusReporter::new(py, &stderr, false, true)?).map(Some) -} - -/// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. -pub(crate) fn maybe_status<'py, T>( - py: Python<'py>, - (message, enabled): (&str, bool), - body: impl FnOnce() -> PyResult, -) -> PyResult { - let Some(reporter) = enabled.then(|| stderr_reporter(py)).transpose()?.flatten() else { - return body(); - }; - let reporter: Bound<'py, TransientStatusReporter> = reporter.into_bound(py); - TransientStatusReporter::start(&reporter, message.to_owned())?; - let result: PyResult = body(); - TransientStatusReporter::close(&reporter)?; - result -} - -/// Run `body`, drawing a stderr spinner named `message` only once it has run for `delay`. -pub(crate) fn delayed_status<'py, T>( - py: Python<'py>, - (message, delay): (&str, Duration), - body: impl FnOnce() -> PyResult, -) -> PyResult { - let Some(reporter) = stderr_reporter(py)? else { - return body(); - }; - let (finished, waiting) = mpsc::channel::<()>(); - let pending: Py = reporter.clone_ref(py); - let message: String = message.to_owned(); - let timer = std::thread::spawn(move || -> PyResult<()> { - if waiting.recv_timeout(delay) != Err(RecvTimeoutError::Timeout) { - return Ok(()); - } - Python::attach(|py| TransientStatusReporter::start(pending.bind(py), message)) - }); - let result: PyResult = body(); - let _ = finished.send(()); - let started: PyResult<()> = py.detach(|| timer.join()).unwrap_or_else(|_| Ok(())); - TransientStatusReporter::close(reporter.bind(py))?; - started?; - result -} - -/// A native status reporter writing to `stream`. -pub(crate) fn status_reporter<'py>( - stream: &Bound<'py, PyAny>, - use_color: bool, -) -> PyResult> { - let reporter: TransientStatusReporter = - TransientStatusReporter::new(stream.py(), stream, use_color, true)?; - Ok(Bound::new(stream.py(), reporter)?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs index 82afaadbda..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs @@ -1,4 +1 @@ -pub(crate) mod native_parse_store; -pub(crate) mod native_store; pub(crate) mod registration; -pub(crate) mod sql_test_scan_store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_parse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_parse_store.rs deleted file mode 100644 index b7deab8dae..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_parse_store.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! Statements a dialect's native parser read whole, kept so a warm compile skips parsing them again. - -use std::path::{Path, PathBuf}; - -use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::assembly::project::models::SyntaxCheck; -use sqlbuild_cache::digest::main::content_digest::content_digest; -use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use sqlbuild_cache::digest::types::ContentDigest; -use sqlbuild_cache::store::models::NativeStore; - -use crate::bindings::_helpers::cache::native_store::{ - current_code_identity, open_store, save_store, -}; -use crate::bindings::_helpers::cache::sql_test_scan_store::debug; - -/// The native store kind holding read-whole verdicts. -const NATIVE_PARSE_STORE_KIND: &str = "native-parses"; -/// The store file below the compile cache folder. -const NATIVE_PARSE_STORE_FILE_NAME: &str = "native-parses.bin"; -/// Bumped whenever what a key covers changes; older stores open empty. -const NATIVE_PARSE_STORE_VERSION: &str = "native-parse-store-v1"; - -/// The statements this build's native parsers read whole, by dialect, SQL and placeholders. -pub(crate) struct NativeParseStore { - store: NativeStore, - path: PathBuf, -} - -impl NativeParseStore { - /// The compile cache folder's store; none when it cannot open, which is logged for debugging. - pub(crate) fn open_in(py: Python<'_>, cache_dir: &Path) -> PyResult> { - let path: PathBuf = cache_dir.join(NATIVE_PARSE_STORE_FILE_NAME); - let identity: String = current_code_identity(py)?; - let environment: String = - hex_digest(&content_digest(&[NATIVE_PARSE_STORE_VERSION, &identity])); - match open_store(py, &path, NATIVE_PARSE_STORE_KIND, &environment) { - Ok(store) => Ok(Some(Self { store, path })), - Err(error) => { - debug(py, "native parse store not loaded: %s", &error.to_string())?; - Ok(None) - } - } - } - - /// Whether the parser read the statement under `key` whole in an earlier compile. - pub(crate) fn read_whole(&mut self, key: &ContentDigest) -> bool { - self.store.get(key).is_some() - } - - /// Record that the parser read the statement under `key` whole. - pub(crate) fn record(&mut self, key: ContentDigest) { - self.store.put(key, Vec::new()); - } - - /// Atomically save verdicts this compile recorded; a failed save is logged for debugging. - pub(crate) fn save(&self, py: Python<'_>) -> PyResult<()> { - if let Err(error) = save_store(py, &self.store, &self.path, &[]) { - debug(py, "native parse store not saved: %s", &error.to_string())?; - } - Ok(()) - } -} - -/// The key `dialect`'s verdict on `check` is stored under. -pub(crate) fn parse_key(dialect: &str, check: &SyntaxCheck) -> ContentDigest { - let mut parts: Vec<&str> = vec![dialect, &check.sql]; - for (name, value) in &check.placeholders { - parts.push(name); - parts.push(value); - } - content_digest(&parts) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs index c9b123dbc1..a671da646e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cache::{native_store, sql_test_scan_store}; +use sqlbuild_python_base::cache::main::register_native_store::register_native_store; +use sqlbuild_python_base::cache::main::register_sql_test_scan_store::register_sql_test_scan_store; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - native_store::register(module)?; - sql_test_scan_store::register(module)?; + register_native_store(module)?; + register_sql_test_scan_store(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 2b25114035..bf71b84c81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -26,8 +26,9 @@ use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; use crate::bindings::_helpers::change_selection::environment_nodes::with_base_dependencies; use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; use crate::bindings::_helpers::discovery::project_config::configured_git_base_ref; -use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, sorted_dumps}; use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; +use sqlbuild_python_base::planner_identity::constants::COMPACT; +use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; /// Where base compiles work when the project's cache directory cannot be created. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 7fa6cb52db..b843473116 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; @@ -15,7 +14,6 @@ use crate::bindings::_helpers::discovery::provider_secrets::{ use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; @@ -33,6 +31,8 @@ use sqlbuild_core::check_command::main::validate_check_selection::validate_check use sqlbuild_core::check_command::models::CheckResultText; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs index 5b3f6c256d..aef2aa2779 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -14,11 +14,11 @@ use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, Check use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; -use crate::bindings::_helpers::result_output::integration_values::json_value; use crate::bindings::_helpers::result_output::terminal_index::current_terminal_event_index; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::result_output::main::json_value::json_value; const WARN_SEVERITY: &str = "warn"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 7f5cf1f54e..18af7aaf5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -5,7 +5,6 @@ use std::path::PathBuf; use crate::bindings::_helpers::audit_command::command::run_audit_command; use crate::bindings::_helpers::build_command::command::run_build_command; use crate::bindings::_helpers::check::check_command::run_check_command; -use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; @@ -55,6 +54,7 @@ use sqlbuild_cli::dispatch::models::{ ClassRef, CommandFacts, DiffSide, Dispatch, DispatchFailure, Field, Route, Source, }; use sqlbuild_core::format_command::main::format_path_selectors::format_path_selectors; +use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index ab3c9aeea8..b37d0aca7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -19,11 +19,11 @@ use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPol use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; use sqlbuild_orchestration::runs::models::RunOutcome; -use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; +use sqlbuild_python_base::build_progress::main::transient_lines::shared_transient_line_coordinator::shared_transient_line_coordinator; use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; -use crate::bindings::_helpers::cli::launch_state::offer_launch_record; +use sqlbuild_python_base::cli::main::offer_launch_record::offer_launch_record; use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, @@ -36,9 +36,10 @@ use crate::bindings::_helpers::compile_command::constants::COMPILE_COMMAND_NAME; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; -use crate::bindings::_helpers::native_host::interrupt::{InterruptHandler, interrupted}; +use sqlbuild_python_base::native_host::models::InterruptHandler; +use sqlbuild_python_base::native_host::main::interrupted::interrupted; use crate::bindings::_helpers::output_capture::records::invocation_context; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; use crate::bindings::_helpers::recall::events::escaped_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index c26744d81a..7a3ebc8d8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; pub(crate) mod failure_cause; -pub(crate) mod launch_state; pub(crate) mod machine_output; pub(crate) mod namespace; pub(crate) mod observability; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 39b81187b0..ec77c89452 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,13 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::build_progress::progress::NativeProgressProjector; -use crate::bindings::_helpers::build_progress::progress::machine_output_mode; -use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; -use crate::bindings::_helpers::event_exporting::scope::EventExporterCommandScope; -use crate::bindings::_helpers::event_exporting::scope::event_exporter_command_scope; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex; use crate::bindings::_helpers::result_output::terminal_index::terminal_event_index_scope; @@ -29,8 +23,14 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_progress::main::progress::machine_output_mode::machine_output_mode; +use sqlbuild_python_base::build_progress::models::NativeProgressProjector; +use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_record; +use sqlbuild_python_base::event_exporting::main::event_exporter_command_scope::event_exporter_command_scope; +use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; const OBSERVABILITY_LOGGER: &str = "sqlbuild.cli.observability"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 0ab9b57a92..662970cd50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -2,10 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_progress::{progress, progress_reporters, progress_text}; -use crate::bindings::_helpers::cli::{ - argv, entry, launch_state, namespace, observability, termination, -}; +use crate::bindings::_helpers::build_progress::{progress_reporters, progress_text}; +use crate::bindings::_helpers::cli::{argv, entry, namespace, observability, termination}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, @@ -14,15 +12,17 @@ use crate::bindings::_helpers::{ presentation_text, query_command, recall, result_output, review_command, scenario_command, seed_command, sql_test_command, workspace_commands, }; +use sqlbuild_python_base::build_progress::main::progress::register_progress::register_progress; +use sqlbuild_python_base::cli::main::register_launch_state::register_launch_state; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; entry::register(module)?; - launch_state::register(module)?; + register_launch_state(module)?; namespace::register(module)?; observability::register(module)?; termination::register(module)?; - progress::register(module)?; + register_progress(module)?; progress_reporters::register(module)?; progress_text::register(module)?; janitor::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index e5edeab95e..a9b9a09989 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -25,24 +25,24 @@ use crate::bindings::_helpers::clone::clone_execution::{ }; use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, timed_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 637d093ff8..2fdb7a55d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -1,5 +1,6 @@ //! Connection config resolution for CLI commands. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::path::PathBuf; use pyo3::prelude::{ @@ -26,7 +27,6 @@ use crate::bindings::_helpers::dbt_interop::profile_connection::resolve_raw_dbt_ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; const ATTACH_KEY: &str = "attach"; @@ -68,12 +68,11 @@ fn warn(py: Python<'_>, message: &str) -> PyResult<()> { } fn dbt_profile_error(py: Python<'_>) -> PyErr { - match host_entry(py, ThirdPartyHost::DbtProfileError) - .and_then(|class| class.call1((DBT_PROFILE_REQUIRES_PROJECT,))) - { - Ok(instance) => PyErr::from_value(instance), - Err(error) => error, - } + host_error( + py, + ThirdPartyHost::DbtProfileError, + DBT_PROFILE_REQUIRES_PROJECT, + ) } /// The config with a referenced dbt profile resolved and user overrides kept on top. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index ed3befb78f..9120cbd84a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -5,15 +5,15 @@ use pyo3::types::PyBool; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index fa6ea5ff50..9e2bda19bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod adapters; pub(crate) mod concurrency; pub(crate) mod connection; pub(crate) mod invocation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs index c6d17197ec..7069192b6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::command_runtime::{ - adapters, connection, preview_target, warehouse_scope, warehouses, + connection, preview_target, warehouse_scope, warehouses, }; +use sqlbuild_python_base::command_runtime::main::register_adapters::register_adapters; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - adapters::register(module)?; + register_adapters(module)?; warehouse_scope::register(module)?; warehouses::register(module)?; connection::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index ff80436dde..f0c9a59597 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -1,11 +1,12 @@ //! The active command's warehouse group, `--warehouse` override and CLI vars. +use sqlbuild_python_base::boundary::main::context_variable::context_variable as shared_context_variable; use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::COMMAND_WAREHOUSE_GROUPS; @@ -21,19 +22,7 @@ const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; static CONTEXT: PyOnceLock> = PyOnceLock::new(); fn context_variable(py: Python<'_>) -> PyResult> { - Ok(CONTEXT - .get_or_try_init(py, || { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("default", py.None())?; - Ok::<_, pyo3::PyErr>( - py.import("contextvars")? - .getattr("ContextVar")? - .call((CONTEXT_NAME,), Some(&keywords))? - .unbind(), - ) - })? - .bind(py) - .clone()) + shared_context_variable(py, &CONTEXT, CONTEXT_NAME) } /// Installs one command's warehouse scope for the duration of a `with` block. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index 83a3d47d5b..89247cc49e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -4,8 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyTuple; -use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanCache; -use crate::bindings::_helpers::compile_driver::stage_work::{DECLARATION_SCOPES, count_stage}; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; @@ -15,6 +13,9 @@ use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODEL use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::cache::models::SqlTestScanCache; +use sqlbuild_python_base::compile_driver::constants::DECLARATION_SCOPES; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 8520002428..b49c068239 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -1,5 +1,6 @@ //! Every discovered model file attached as a model input by the native model loop. +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic_function::report_compile_diagnostic_function; use std::path::PathBuf; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -237,11 +238,15 @@ pub(crate) fn build_model_inputs<'py>( vec![ ( "report_diagnostic", - pyo3::wrap_pyfunction!(crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic, py)?.into_any(), + report_compile_diagnostic_function(py)?.into_any(), ), ( "record_template_reads", - pyo3::wrap_pyfunction!(crate::bindings::_helpers::model_config::template_data::record_template_reads, py)?.into_any(), + pyo3::wrap_pyfunction!( + crate::bindings::_helpers::model_config::template_data::record_template_reads, + py + )? + .into_any(), ), ( "visible_declarations", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs index d652433b13..d77574ef3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs @@ -14,14 +14,13 @@ use crate::bindings::_helpers::compile_command::target_writer::{ use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, publish_staged_target, staging_changed, write_static_target, }; -use crate::bindings::_helpers::compile_driver::stage_work::{ - BACKGROUND_SQL_TEST_PLANS, count_stage, -}; use crate::bindings::_helpers::compile_outputs::artifacts::staged_artifact_files; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_driver::constants::BACKGROUND_SQL_TEST_PLANS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MIN_PREPARED_ARTIFACT_MODELS: usize = 128; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index fd2579e6de..e9a710d0b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -14,14 +14,13 @@ use sqlbuild_core::cli_text::models::{ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; -use crate::bindings::_helpers::compile_outputs::json_report::{ - emit_json_report, emit_orjson_report, -}; use crate::bindings::_helpers::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::emit_json_report::emit_json_report; +use sqlbuild_python_base::compile_outputs::main::emit_orjson_report::emit_orjson_report; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -170,7 +169,7 @@ impl<'py> ReportInputs<'py> { return Ok(PyDict::new(py)); } let reasons: Bound<'py, PyDict> = semantic_coverage(&self.project)?; - if crate::bindings::_helpers::compile_context::delta::injection().is_some() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some() { return Ok(reasons); } let Some(names) = self.selected_names(MODEL_TYPE)? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 483c32c6fc..86e566f8af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -16,10 +16,6 @@ use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; -use crate::bindings::_helpers::build_progress::progress::{ - NativeProgressProjector, current_projector, -}; -use crate::bindings::_helpers::cli::launch_state::launch_record_path; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, @@ -36,25 +32,29 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::runtime_identity::{ invocation_identity, runtime_identity, }; -use crate::bindings::_helpers::compile_context::delta; use crate::bindings::_helpers::compile_delta::driver::{ self, DELTA_VERIFY_ENV_VAR, DeltaOutcome, delta_enabled, env_enabled, }; use crate::bindings::_helpers::compile_delta::verify; use crate::bindings::_helpers::compile_driver::command::run_compile_command; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; -use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::compile_outputs::reuse::{ ReuseCheckPy, ReuseRulesPy, check_compile_reuse, }; use crate::bindings::_helpers::discovery::project_config::warn_deprecated_settings; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::macro_bridge::first_compile::keep_delta_claim; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::build_progress::models::NativeProgressProjector; +use sqlbuild_python_base::cli::main::launch_record_path::launch_record_path; +use sqlbuild_python_base::compile_context::main::delta::reset::reset; +use sqlbuild_python_base::compile_context::main::delta::start_capture::start_capture; +use sqlbuild_python_base::compile_outputs::main::plain_json::plain_json; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; @@ -480,7 +480,7 @@ pub(crate) fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult( None => Ok(()), } }; - delta::reset(); + reset(); let project_dir: Bound<'py, PyAny> = native_field(attempt, "project_dir")?; let cache: Bound<'py, PyAny> = cache_directory(&project_dir)?; let compiled_root: PathBuf = std::path::absolute( @@ -553,7 +553,7 @@ fn compiled_with_delta<'py>( &format!("delta compile fell back: {reason}").into_bound_py_any(py)?, )?; } - delta::start_capture(); + start_capture(); return run_compile_command(py, request.clone(), Some(attempt.clone())); } DeltaOutcome::Finished(finished) => verify::hold_expected(verify::Expected { @@ -573,7 +573,7 @@ fn compiled_with_delta<'py>( ..verify::Expected::default() }), } - delta::start_capture(); + start_capture(); let full: PyResult = run_compile_command(py, request.clone(), Some(attempt.clone())); let raised: Option = full.as_ref().err().map(ToString::to_string); let differences: Vec = verify::differences(raised); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index ffe3dd9a4f..126919ed30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -308,7 +308,7 @@ pub(crate) fn print_to<'py>( impl RecordedCompileOutput { /// Print one stderr result line, such as a note, and record it; a delta only records it. pub(crate) fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { - if crate::bindings::_helpers::compile_context::delta::injection().is_none() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() { print_to(line.py(), "stderr", &line)?; } self.locked().0.push(line.unbind()); @@ -514,7 +514,9 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { json_output: facts.json_output.extract()?, rules: reuse_rules(py)?, }; - if let Some(mut capture) = crate::bindings::_helpers::compile_context::delta::take_capture() { + if let Some(mut capture) = + sqlbuild_python_base::compile_context::main::delta::take_capture::take_capture() + { capture.environment_names = native_field(&facts.input_reads, "environment_names")?.extract()?; let json_output: bool = facts.json_output.is_truthy()?; @@ -527,7 +529,7 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { if let Ok(sections) = image.encode() { pending = Some(sections); } - crate::bindings::_helpers::compile_context::delta::set_pending_image(pending); + sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(pending); } Err(reason) => { crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 6101b073c1..27c9fe1a28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -27,9 +27,6 @@ use crate::bindings::_helpers::compile_command::test_artifact_cache::{ IdentityContext, artifact_identity_payload, artifact_record_key_payload, identity_context, payload_digests, read_cache, write_cache, yield_interpreter, }; -use crate::bindings::_helpers::compile_driver::stage_work::{ - SQL_TEST_ARTIFACT_PLANS, SQL_TEST_MODEL_CHAINS, count_stage, -}; use crate::bindings::_helpers::compile_outputs::artifacts::{ publish_staged_artifacts, remove_stale_artifacts, }; @@ -46,6 +43,9 @@ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_driver::constants::SQL_TEST_ARTIFACT_PLANS; +use sqlbuild_python_base::compile_driver::constants::SQL_TEST_MODEL_CHAINS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::log_debug_message::log_debug_message; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -677,7 +677,7 @@ fn write_static_tests<'py>( } } let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; - if crate::bindings::_helpers::compile_context::delta::injection().is_some() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some() { let mut kept: HashMap = read_cache(&cache_dir)?; for (key, record) in records.drain(..) { kept.insert(key, record); @@ -698,7 +698,7 @@ pub(crate) fn write_static_target<'py>( ) -> PyResult> { let py: Python<'py> = target_dir.py(); let remove_stale_files: bool = prepared_target(target_dir)? - && crate::bindings::_helpers::compile_context::delta::injection().is_none(); + && sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none(); let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); let mut managed: Vec> = Vec::new(); let mut managed_models: Vec = Vec::new(); @@ -866,7 +866,9 @@ pub(crate) fn write_rule_failed_models<'py>( }; models.push((relative_path, query_sql)); } - if let Some(injection) = crate::bindings::_helpers::compile_context::delta::injection() { + if let Some(injection) = + sqlbuild_python_base::compile_context::main::delta::injection::injection() + { models.extend(models_not_rerun( &target, &models, @@ -910,7 +912,7 @@ fn models_not_rerun( let path: PathBuf = native_model_output_path(&compiled_dir, relative_path); let Ok(text) = std::fs::read_to_string(&path) else { return Err( - crate::bindings::_helpers::compile_context::delta::delta_unsupported( + sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported( "the stored compile's SQL for a model it did not rerun is unreadable", ), ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs index 4822e196b5..e365ca9343 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs @@ -12,8 +12,9 @@ use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::compile_outputs::json_report::{orjson_sorted, orjson_value}; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::compile_outputs::main::orjson_sorted::orjson_sorted; +use sqlbuild_python_base::compile_outputs::main::orjson_value::orjson_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs deleted file mode 100644 index 7c951bea17..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! The process clock compile CPU timings read: Python's `time.process_time_ns`. - -use pyo3::PyErr; -use pyo3::prelude::{PyAnyMethods, PyResult, Python}; -use sqlbuild_host::compile_context::types::ProcessClock; - -use sqlbuild_python_base::boundary::main::imported::imported; - -/// The process CPU clock of one interpreter, including its worker threads. -#[derive(Clone, Copy)] -pub(crate) struct PythonProcessClock<'py>(pub(crate) Python<'py>); - -impl ProcessClock for PythonProcessClock<'_> { - type Error = PyErr; - - fn process_time_ns(&self) -> PyResult { - imported(self.0, "time", "process_time_ns")? - .call0()? - .extract() - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs deleted file mode 100644 index f04188f3da..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs +++ /dev/null @@ -1,449 +0,0 @@ -//! Process-wide slots for what a full compile captures and what a delta compile injects. - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; - -use pyo3::prelude::PyResult; -use sqlbuild_analysis::assembly::analysis_session::models::{ - DynamicFamily, RelationTables, SessionRequest, -}; -use sqlbuild_cache::project_reuse::types::ImageSections; -use sqlbuild_discovery::header_changes::types::HeaderKeys; -use sqlbuild_rules::engine::main::retained_request::retained_request; -use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; - -/// Whether edits that report or raise errors compile as deltas; until then they fall back. -pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; - -/// What one model contributes to the partial-check reasons, apart from its dependencies. -#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub(crate) struct ModelCoverage { - pub(crate) name: String, - /// The model opts out of SQL analysis. - pub(crate) disabled: bool, - /// The model has a binding error, which makes every model reading it partial. - pub(crate) blocked: bool, - /// Its own reasons, sorted; those from invalid dependencies are derived. - pub(crate) partial: Vec, -} - -impl ModelCoverage { - /// Whether the model adds anything to the reasons. - pub(crate) fn notable(&self) -> bool { - self.disabled || self.blocked || !self.partial.is_empty() - } -} - -/// A stored compile's built-in rules request, as decoded for a delta compile. -#[derive(Clone, Debug)] -pub(crate) struct BaseRules { - pub(crate) request: EvaluateRequest, - /// Each model's payload digest, which keys its cached findings, in model order. - pub(crate) digests: Vec, -} - -impl BaseRules { - /// The rows a compile retained, decoded. - pub(crate) fn decoded(retained: &RetainedRows) -> Option { - Some(Self { - request: retained_request(retained)?, - digests: retained.digests.clone(), - }) - } -} - -/// A stored rules request, decoding on its own thread or decoded already. -#[derive(Debug)] -pub(crate) enum PendingRules { - Decoding(std::thread::JoinHandle>), - Ready(Box), -} - -impl PendingRules { - /// The decoded request, or `None` when it could not be decoded. - pub(crate) fn resolved(self) -> Option { - match self { - Self::Decoding(handle) => { - let Ok(rules) = handle.join() else { - return None; - }; - rules - } - Self::Ready(rules) => Some(*rules), - } - } -} - -/// The facts a full compile leaves for its project image beyond its analysis record. -#[derive(Debug, Default)] -pub(crate) struct ImageCapture { - /// The sources functions and audits read, which a delta compile always keeps. - pub(crate) pinned_sources: BTreeSet, - /// The sources each SQL test file reads, which a delta compile keeps when it reruns it. - pub(crate) test_sources: BTreeMap>, - /// The relation facts the model analysis session ended with. - pub(crate) relations: Option, - /// The built-in rules request and its raw findings, before suppressions. - pub(crate) rules: Option, - /// Each custom-rule fact digest the custom rule cache compared, by fact key. - pub(crate) fact_digests: BTreeMap, String>, - /// Whether the JSON report fell back to the standard library's serializer. - pub(crate) stdlib_json: bool, - /// Each model's scope usage digest, by model file path. - pub(crate) usages: BTreeMap, - /// Each SQL test's compiled-artifact chain, by test key text. - pub(crate) test_chains: Vec, - /// Each project model's `(path, name, materialized)`, when custom rules ran. - pub(crate) model_subjects: Option)>>, - /// Each compiled model's `(name, relative path, upstream (kind, name) pairs)`. - pub(crate) models: Vec, - /// Each report diagnostic as its JSON item text, with what it names. - pub(crate) diagnostics: Vec, - /// The text report's model lines `(name, failed, column count)`, when it rendered one. - pub(crate) text_models: Option>, - /// The template environment names the compile read. - pub(crate) environment_names: Vec, - /// Each discovered model's header keys, by model file path. - pub(crate) header_keys: BTreeMap, - /// The audit factories each model file's header attaches, for files attaching any. - pub(crate) audit_references: BTreeMap>, - /// Each model file's SQL lint cache key, in project model order. - pub(crate) lint_model_keys: Option>, - /// Each other SQL file's lint cache key, in walk order. - pub(crate) lint_file_keys: Option)>>, - /// The closed shape each model published for semantic checks, in publication order. - pub(crate) model_shapes: Option>, - /// The notable semantic coverage facts of every model, when semantic checks reported them. - pub(crate) coverage: Option>, - /// Why this compile cannot be the base of a delta compile, when something ruled it out. - pub(crate) unsupported: Option, -} - -/// The built-in rules request a compile evaluated and its raw findings. -#[derive(Debug)] -pub(crate) struct CapturedRules { - pub(crate) retained: RetainedRows, -} - -/// One model's published `(column, type)` shape. -pub(crate) type ModelShape = (String, Vec<(String, String)>); - -/// One compiled model's identity and references. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub(crate) struct CapturedModel { - pub(crate) name: String, - pub(crate) relative_path: String, - pub(crate) upstream: Vec<(String, String)>, -} - -/// One report diagnostic: its JSON text and the resource and file it names. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub(crate) struct CapturedDiagnostic { - pub(crate) json: String, - pub(crate) resource_name: Option, - pub(crate) paths: Vec, - /// Whether the diagnostic is an error. - #[serde(default)] - pub(crate) error: bool, -} - -/// One SQL test's file and the models its compiled artifact is keyed by. -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct CapturedTest { - pub(crate) key: String, - pub(crate) relative_path: String, - pub(crate) chain: Vec, -} - -/// What a delta compile hands the stages it reruns for the changed models. -#[derive(Debug, Default)] -pub(crate) struct DeltaInjection { - /// Model files to discover, as project-relative POSIX paths. - pub(crate) model_paths: BTreeSet, - /// SQL test files to discover, as project-relative POSIX paths. - pub(crate) test_paths: BTreeSet, - /// The paths and names of every model the delta compile reruns. - pub(crate) rerun_names: BTreeSet, - /// The project directory the stored compile and the delta compile both read. - pub(crate) project_dir: std::path::PathBuf, - /// Every model and SQL test file of the whole project, as project-relative POSIX paths. - pub(crate) source_paths: BTreeSet, - /// Every model file of the whole project, which path defaults are matched against. - pub(crate) project_model_paths: Vec, - /// Every model name of the whole project, which references resolve against. - pub(crate) model_names: Vec, - /// The whole project's relation facts, injected for relations the delta does not analyse. - pub(crate) relations: RelationTables, - /// The whole project's built-in rules request, taken by the round that evaluates it. - pub(crate) rules: Mutex>, - /// The whole project's custom-rule fact digests. - pub(crate) fact_digests: BTreeMap, String>, - /// Each project model's `(path, name, materialized)`, the custom-rule subjects. - pub(crate) model_subjects: Option)>>, - /// The files that changed since the stored compile, as project-relative POSIX paths. - pub(crate) changed_paths: BTreeSet, - /// Where loaded YAML documents are kept for later delta compiles. - pub(crate) yaml_store: Option, - /// The only sources discovery keeps, those the rerun models read; `None` keeps every one. - pub(crate) source_names: Option>, - /// The whole project's SQL lint cache keys, as the image recorded them. - pub(crate) lint_model_keys: Option>, - pub(crate) lint_file_keys: Option)>>, - /// The closed shapes the whole project's models published. - pub(crate) model_shapes: Vec, - /// The whole project's text report model lines, with the rerun models' lines to replace. - pub(crate) text_models: Vec<(String, bool, i64)>, - /// The whole project's notable semantic coverage facts, as the image recorded them. - pub(crate) coverage: Option>, - /// Every project model in project order, with the models it reads. - pub(crate) coverage_order: Vec<(String, Vec)>, - /// The audit factories each model file's header attaches, for files attaching any. - pub(crate) audit_references: BTreeMap>, -} - -static CAPTURE: Mutex> = Mutex::new(None); -static INJECTION: Mutex>> = Mutex::new(None); - -/// Why the running delta compile cannot continue, once something ruled it out. -static UNSUPPORTED: Mutex> = Mutex::new(None); -static PENDING_IMAGE: Mutex> = Mutex::new(None); -static RECORD_MERGE: Mutex> = Mutex::new(None); - -fn locked(slot: &Mutex) -> MutexGuard<'_, T> { - slot.lock().unwrap_or_else(PoisonError::into_inner) -} - -/// Start capturing image facts for the compile about to run. -pub(crate) fn start_capture() { - *locked(&CAPTURE) = Some(ImageCapture::default()); -} - -/// Stop capturing without keeping anything, for a project that turns delta compiles off. -pub(crate) fn stop_capture() { - *locked(&CAPTURE) = None; -} - -/// Stop capturing and take what was captured. -pub(crate) fn take_capture() -> Option { - locked(&CAPTURE).take() -} - -/// Whether the running compile captures image facts. -pub(crate) fn capturing() -> bool { - locked(&CAPTURE).is_some() -} - -/// Record into the capture, when one is running. -pub(crate) fn capture(record: impl FnOnce(&mut ImageCapture)) { - if let Some(capture) = locked(&CAPTURE).as_mut() { - record(capture); - } -} - -/// Record sources a resource other than a model reads, when a capture is running. -pub(crate) fn record_source_reads<'a>(names: impl IntoIterator) { - capture(|capture| { - capture - .pinned_sources - .extend(names.into_iter().map(str::to_owned)); - }); -} - -/// Record sources an SQL test file reads, when a capture is running. -pub(crate) fn record_test_source_reads<'a>( - relative_path: &str, - names: impl IntoIterator, -) { - capture(|capture| { - capture - .test_sources - .entry(relative_path.replace('\\', "/")) - .or_default() - .extend(names.into_iter().map(str::to_owned)); - }); -} - -/// Mark the running capture as unusable as a delta base. -pub(crate) fn capture_unsupported(reason: &str) { - capture(|capture| { - if capture.unsupported.is_none() { - capture.unsupported = Some(reason.to_owned()); - } - }); -} - -/// Run `body` with `injection` visible to the rerun stages; returns it unless still held. -pub(crate) fn with_injection( - injection: DeltaInjection, - body: impl FnOnce() -> T, -) -> (T, Option) { - *locked(&INJECTION) = Some(Arc::new(injection)); - let result: T = body(); - let injection: Option> = locked(&INJECTION).take(); - ( - result, - injection.and_then(|injection| { - let Ok(injection) = Arc::try_unwrap(injection) else { - return None; - }; - Some(injection) - }), - ) -} - -/// The running delta compile's injection, if any. -pub(crate) fn injection() -> Option> { - locked(&INJECTION).clone() -} - -/// Rule out the running delta compile: returns the error that unwinds to its fallback. -pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { - let mut slot = locked(&UNSUPPORTED); - if slot.is_none() { - *slot = Some(reason.to_owned()); - } - pyo3::exceptions::PyRuntimeError::new_err(format!("delta compile unsupported: {reason}")) -} - -/// In a delta compile that leaves errors to full compiles, stop once `failed` reports one. -pub(crate) fn stop_on_errors(failed: bool) -> pyo3::PyResult<()> { - let errors_on_delta_path: bool = ERRORS_ON_DELTA_PATH; - if failed && !errors_on_delta_path && injection().is_some() { - return Err(delta_unsupported("the edit reports errors")); - } - Ok(()) -} - -/// Take the reason the delta compile was ruled out, if any. -pub(crate) fn take_delta_unsupported() -> Option { - locked(&UNSUPPORTED).take() -} - -/// The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. -pub(crate) fn injected_fact_digest(key: &[String]) -> Option { - let injection: Arc = injection()?; - if key - .iter() - .skip(1) - .any(|part| injection.rerun_names.contains(part)) - { - return None; - } - injection.fact_digests.get(key).cloned() -} - -/// Give a delta's analysis session the whole project's relation facts and analysis order. -pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { - let Some(injection) = injection() else { - return Ok(request); - }; - let relations: &RelationTables = &injection.relations; - request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); - request.retain_relations = true; - let analysed: BTreeSet<&str> = request - .models - .iter() - .map(|model| model.name.as_str()) - .collect(); - let rerun: BTreeSet<&str> = analysed - .iter() - .copied() - .chain(injection.rerun_names.iter().map(String::as_str)) - .collect(); - request.dynamic_families_by_table = injected_families( - &request.dynamic_families_by_table, - &relations.families, - &rerun, - )?; - for (table, injected) in [ - (&mut request.column_types, &relations.types), - (&mut request.column_nullability, &relations.nullability), - (&mut request.complete_schemas, &relations.complete), - ] { - let present: BTreeSet = table.iter().map(|(name, _)| name.clone()).collect(); - table.extend( - injected - .iter() - .filter(|(name, _)| !analysed.contains(name.as_str()) && !present.contains(name)) - .cloned(), - ); - } - Ok(request) -} - -/// The stored compile's dynamic column families, when no rerun model changed its own. -fn injected_families( - current: &[(String, Vec)], - stored: &[(String, Vec)], - rerun: &BTreeSet<&str>, -) -> pyo3::PyResult)>> { - let families_of = |table: &[(String, Vec)], name: &str| { - table - .iter() - .find(|(table, _)| table == name) - .map(|(_, families)| families.clone()) - }; - let changed: bool = current - .iter() - .any(|(name, _)| !rerun.contains(name.as_str())) - || rerun - .iter() - .any(|name| families_of(current, name) != families_of(stored, name)); - if changed { - return Err(delta_unsupported( - "a model's dynamic column families changed", - )); - } - Ok(stored.to_vec()) -} - -/// What a delta compile's reuse record keeps of the stored compile it patched. -#[derive(Debug, Default)] -pub(crate) struct RecordMerge { - /// Each stored artifact's stamp, verified by the reuse check, kept unless the delta wrote it. - pub(crate) kept: Vec, - /// The template environment names the stored compile read. - pub(crate) environment_names: Vec, - /// The module files the stored compile depended on, such as those behind stored macro calls. - pub(crate) module_paths: Vec, -} - -/// Hand the next reuse record the image to keep beside it. -pub(crate) fn set_pending_image(image: Option) { - *locked(&PENDING_IMAGE) = image; -} - -/// The image the next reuse record keeps, taken. -pub(crate) fn take_pending_image() -> Option { - locked(&PENDING_IMAGE).take() -} - -/// Hand the next reuse record what it keeps of the patched compile. -pub(crate) fn set_record_merge(merge: Option) { - *locked(&RECORD_MERGE) = merge; -} - -/// What the next reuse record keeps of the patched compile, taken. -pub(crate) fn take_record_merge() -> Option { - locked(&RECORD_MERGE).take() -} - -/// Forget every capture, injection and pending record fact, after a compile ends either way. -pub(crate) fn reset() { - let _ = take_capture(); - *locked(&INJECTION) = None; - let _ = take_delta_unsupported(); - set_pending_image(None); - set_record_merge(None); -} - -/// Run a capture hook; a failure only rules the compile out as a delta base. -pub(crate) fn guarded(hook: impl FnOnce() -> PyResult<()>) -> PyResult<()> { - if !capturing() { - return Ok(()); - } - if let Err(error) = hook() { - capture_unsupported(&format!("an image fact could not be recorded: {error}")); - } - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_codec.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_codec.rs deleted file mode 100644 index 3b9378d1c6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/image_codec.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! Image sections as length-prefixed records over a table of each distinct string. - -use rustc_hash::FxHashMap; -use sqlbuild_cache::store::errors::StoreDecodeError; -use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; - -/// The most items a list reserves room for before reading them, against corrupt counts. -const MAX_RESERVED_ITEMS: usize = 1 << 16; - -/// A stored value, or `None` when the section is damaged. -fn read(value: Result) -> Option { - let Ok(value) = value else { - return None; - }; - Some(value) -} - -/// A stored count or index, or `None` when it does not fit this platform. -fn length(value: u64) -> Option { - let Ok(length) = usize::try_from(value) else { - return None; - }; - Some(length) -} - -/// Writes one section: its values refer to a table of the distinct strings they hold. -#[derive(Default)] -pub(crate) struct Encoder<'a> { - ids: FxHashMap<&'a str, u64>, - strings: Vec<&'a str>, - body: RecordWriter, -} - -impl<'a> Encoder<'a> { - pub(crate) fn string(&mut self, value: &'a str) { - let next: u64 = self.strings.len() as u64; - let id: u64 = *self.ids.entry(value).or_insert(next); - if id == next { - self.strings.push(value); - } - self.body.put_u64(id); - } - - pub(crate) fn optional_string(&mut self, value: Option<&'a str>) { - self.flag(value.is_some()); - if let Some(value) = value { - self.string(value); - } - } - - pub(crate) fn count(&mut self, count: usize) { - self.body.put_u64(count as u64); - } - - pub(crate) fn flag(&mut self, value: bool) { - self.body.put_u64(u64::from(value)); - } - - pub(crate) fn bytes(&mut self, value: &[u8]) { - self.body.put_bytes(value); - } - - pub(crate) fn strings(&mut self, values: impl ExactSizeIterator) { - self.count(values.len()); - for value in values { - self.string(value); - } - } - - pub(crate) fn pairs(&mut self, pairs: &'a [(String, String)]) { - self.count(pairs.len()); - for (left, right) in pairs { - self.string(left); - self.string(right); - } - } - - /// The section: the string table, then the values. - pub(crate) fn finish(self) -> Vec { - let mut section: RecordWriter = RecordWriter::default(); - section.put_u64(self.strings.len() as u64); - for value in &self.strings { - section.put_str(value); - } - section.put_bytes(&self.body.into_bytes()); - section.into_bytes() - } -} - -/// Reads a section an [`Encoder`] wrote; any malformed value fails the whole read. -pub(crate) struct Decoder<'a> { - strings: Vec, - body: RecordReader<'a>, -} - -impl<'a> Decoder<'a> { - pub(crate) fn new(section: &'a [u8]) -> Option { - let mut reader: RecordReader<'a> = RecordReader::new(section); - let count: usize = length(read(reader.u64())?)?; - let mut strings: Vec = Vec::with_capacity(count.min(section.len())); - for _ in 0..count { - strings.push(read(reader.string())?); - } - let body: &'a [u8] = read(reader.bytes())?; - read(reader.finish())?; - Some(Self { - strings, - body: RecordReader::new(body), - }) - } - - pub(crate) fn string(&mut self) -> Option { - let id: usize = length(read(self.body.u64())?)?; - self.strings.get(id).cloned() - } - - pub(crate) fn optional_string(&mut self) -> Option> { - Some(if self.flag()? { - Some(self.string()?) - } else { - None - }) - } - - pub(crate) fn count(&mut self) -> Option { - length(read(self.body.u64())?) - } - - pub(crate) fn flag(&mut self) -> Option { - match read(self.body.u64())? { - 0 => Some(false), - 1 => Some(true), - _ => None, - } - } - - pub(crate) fn bytes(&mut self) -> Option<&'a [u8]> { - read(self.body.bytes()) - } - - pub(crate) fn list( - &mut self, - mut item: impl FnMut(&mut Self) -> Option, - ) -> Option> { - let count: usize = self.count()?; - let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); - for _ in 0..count { - items.push(item(self)?); - } - Some(items) - } - - pub(crate) fn strings(&mut self) -> Option> { - self.list(Self::string) - } - - pub(crate) fn pairs(&mut self) -> Option> { - self.list(|decoder| Some((decoder.string()?, decoder.string()?))) - } - - /// Whether every value was read. - pub(crate) fn finish(self) -> Option<()> { - read(self.body.finish()) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 7c86168120..2d54546bf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -12,21 +12,11 @@ use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; use sqlbuild_cache::project_reuse::types::ImageSections; -use crate::bindings::_helpers::build_progress::progress::{ - NativeProgressProjector, current_projector, -}; use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; use crate::bindings::_helpers::compile_command::reuse_recording::{ RecordedCompileOutput, print_to, }; -use crate::bindings::_helpers::compile_context::delta::BaseRules; -use crate::bindings::_helpers::compile_context::delta::ERRORS_ON_DELTA_PATH; -use crate::bindings::_helpers::compile_context::delta::PendingRules; -use crate::bindings::_helpers::compile_context::delta::{ - CapturedDiagnostic, CapturedModel, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, - start_capture, take_capture, take_delta_unsupported, with_injection, -}; use crate::bindings::_helpers::compile_delta::image::{ ImageGate, ImageRelations, ImageRules, ProjectImage, }; @@ -39,6 +29,21 @@ use crate::bindings::_helpers::rules::custom_evaluation::stored_custom_rule_refu use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; use sqlbuild_discovery::header_changes::main::header_change::header_change; use sqlbuild_discovery::header_changes::models::HeaderChange; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::build_progress::models::NativeProgressProjector; +use sqlbuild_python_base::compile_context::constants::ERRORS_ON_DELTA_PATH; +use sqlbuild_python_base::compile_context::main::delta::start_capture::start_capture; +use sqlbuild_python_base::compile_context::main::delta::take_capture::take_capture; +use sqlbuild_python_base::compile_context::main::delta::take_delta_unsupported::take_delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::with_injection::with_injection; +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_context::models::DeltaInjection; +use sqlbuild_python_base::compile_context::models::ImageCapture; +use sqlbuild_python_base::compile_context::models::PendingRules; +use sqlbuild_python_base::compile_context::models::RecordMerge; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Set to `1` to always compile the whole project. @@ -294,7 +299,7 @@ fn lent_injection( yaml_store: store_path .parent() .and_then(std::path::Path::parent) - .map(crate::bindings::_helpers::compile_context::yaml_cache::yaml_store_path), + .map(sqlbuild_python_base::compile_context::main::yaml_cache::yaml_store_path::yaml_store_path), project_dir, source_paths: image .models @@ -1247,8 +1252,10 @@ pub(crate) fn commit( for line in &stderr_lines { print_to(py, STDERR, line.as_str())?; } - crate::bindings::_helpers::compile_context::delta::set_pending_image(image); - crate::bindings::_helpers::compile_context::delta::set_record_merge(Some(merge)); + sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(image); + sqlbuild_python_base::compile_context::main::delta::set_record_merge::set_record_merge(Some( + merge, + )); crate::bindings::_helpers::compile_driver::command::store_and_print( (&request, attempt), &output, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index af898eccf8..a0b7f70019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -7,13 +7,17 @@ use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; -use crate::bindings::_helpers::compile_context::delta::{BaseRules, PendingRules}; -use crate::bindings::_helpers::compile_context::delta::{ - CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, -}; -use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; use sqlbuild_discovery::header_changes::types::HeaderKeys; +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_context::models::Decoder; +use sqlbuild_python_base::compile_context::models::Encoder; +use sqlbuild_python_base::compile_context::models::ImageCapture; +use sqlbuild_python_base::compile_context::models::ModelCoverage; +use sqlbuild_python_base::compile_context::models::PendingRules; +use sqlbuild_python_base::compile_context::types::ModelShape; /// Changes whenever the image's meaning changes, so an older image is never read. pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v9"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs index 3eabed2a52..da4c6ceec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs @@ -5,10 +5,11 @@ use std::path::{Path, PathBuf}; use pyo3::PyResult; -use crate::bindings::_helpers::compile_context::delta::{ - capture, capturing, delta_unsupported, injection, -}; use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; use crate::bindings::_helpers::rules::lint_pipeline::LintFile; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index 9440835206..125c22a667 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -7,12 +7,15 @@ use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethod use sqlbuild_core::cli_text::models::ModelLine; use crate::bindings::_helpers::compile_command::report::diagnostic_item; -use crate::bindings::_helpers::compile_context::delta::guarded; -use crate::bindings::_helpers::compile_context::delta::{ - CapturedDiagnostic, CapturedModel, CapturedTest, capture, capturing, injection, -}; -use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_outputs::main::plain_json::plain_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The graph key kind of a model. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index a22e5f2103..c69dff8f27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -5,9 +5,10 @@ use std::collections::HashMap; use pyo3::PyResult; use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; -use crate::bindings::_helpers::compile_context::delta::BaseRules; -use crate::bindings::_helpers::compile_context::delta::PendingRules; -use crate::bindings::_helpers::compile_context::delta::{delta_unsupported, injection}; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::PendingRules; /// In a delta, the whole project's rules request with the rerun rows taken from `request`. pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 70764a8b68..883111a1b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -8,12 +8,8 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; -use crate::bindings::_helpers::compile_driver::stage_work::{ - COLUMN_LINEAGE, MODEL_CONTRACTS, count_stage, -}; use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; @@ -30,6 +26,10 @@ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::Status; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::compile_driver::constants::COLUMN_LINEAGE; +use sqlbuild_python_base::compile_driver::constants::MODEL_CONTRACTS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -159,7 +159,7 @@ pub(crate) fn analyze<'py>( let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?) - && crate::bindings::_helpers::compile_context::delta::injection().is_none(); + && sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none(); let sql_lint_enabled: bool = enabled && !options.no_sql_validation.is_truthy()?; let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(sql_lint_enabled, options.no_cache()?))?; @@ -222,7 +222,9 @@ fn analyzed<'py>( driver.graph((&discovered, &adapter), &lint_dialect)?; let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; - crate::bindings::_helpers::compile_context::delta::stop_on_errors(graph_failed)?; + sqlbuild_python_base::compile_context::main::delta::stop_on_errors::stop_on_errors( + graph_failed, + )?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { if let Some(early_rules) = early_rules { @@ -232,10 +234,9 @@ fn analyzed<'py>( } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, (&dialect, &adapter))?; - crate::bindings::_helpers::compile_context::delta::stop_on_errors(any_error(&native_field( - &contracts.result, - "diagnostics", - )?)?)?; + sqlbuild_python_base::compile_context::main::delta::stop_on_errors::stop_on_errors(any_error( + &native_field(&contracts.result, "diagnostics")?, + )?)?; let selected_keys: Bound<'py, PyAny> = select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 0be5dd3b1e..5d054f0092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -9,9 +9,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_host::compile_context::main::record_compile_metric::record_compile_metric; -use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, unicode_sorted_dumps}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::planner_identity::constants::COMPACT; +use sqlbuild_python_base::planner_identity::main::unicode_sorted_dumps::unicode_sorted_dumps; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index c228a6c72a..621adc4833 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -8,7 +8,7 @@ use pyo3::types::{PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_context::diagnostic_collector::collect_compile_diagnostics; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; use crate::bindings::_helpers::compiled_views::records::project_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index ed83399659..d00ffb9d14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -23,12 +23,8 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ ADAPTER_RULE_ID, nullability_rules, text_pairs, }; use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; -use crate::bindings::_helpers::compile_context::compile_timings::PythonProcessClock; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; -use crate::bindings::_helpers::compile_driver::stage_work::{ - EXPRESSION_SHAPES, MODEL_ANALYSIS, count_stage, -}; use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::compiled_views::source_views::read_source_view; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -39,6 +35,10 @@ use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, f use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::compile_context::models::PythonProcessClock; +use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPES; +use sqlbuild_python_base::compile_driver::constants::MODEL_ANALYSIS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::log_warning_message::log_warning_message; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index ab4a457d79..cbb08a55c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -9,15 +9,15 @@ use crate::bindings::_helpers::analysis_session::session::{ FamilyRow, NativeModelAnalysisSession, PivotRequestRow, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; -use crate::bindings::_helpers::compile_driver::stage_work::{ - PIVOT_PROOFS, SESSION_PIVOT_PROOFS, count_stage, -}; use crate::bindings::_helpers::model_config::header_families::HeaderFamilies; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_rows; use pyo3::IntoPyObjectExt; +use sqlbuild_python_base::compile_driver::constants::PIVOT_PROOFS; +use sqlbuild_python_base::compile_driver::constants::SESSION_PIVOT_PROOFS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 4bc62d5e0b..d966251b77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -24,7 +24,6 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::compile_driver::stage_work::{SEMANTIC_COMPLETION, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_effective_adapter_name, resolve_effective_scenario_config, @@ -37,6 +36,8 @@ use crate::bindings::_helpers::semantic_checks::stage::{ }; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::compile_driver::constants::SEMANTIC_COMPLETION; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -115,7 +116,7 @@ pub(crate) fn assembled_project<'py>( let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native_field(&native, "resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; - if crate::bindings::_helpers::compile_context::delta::injection().is_none() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() { report_scope_index_errors(&scope_index)?; } let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 7af2f464c7..fa8b6fc2d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyList, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; -use crate::bindings::_helpers::compile_driver::stage_work::{SQL_TEST_ASSEMBLY, count_stage}; use crate::bindings::_helpers::sql_test_glue::compiled_tests::assemble_native_sql_tests; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; +use sqlbuild_python_base::compile_driver::constants::SQL_TEST_ASSEMBLY; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index bd7b0cb71f..088bac0238 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -15,7 +15,6 @@ use sqlbuild_rules::custom_host_pool::constants::{ use std::path::PathBuf; use std::rc::Rc; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ format_compile_json_with_notice, format_compile_text, @@ -27,13 +26,11 @@ use crate::bindings::_helpers::compile_command::reuse_recording::{ use crate::bindings::_helpers::compile_command::target_writer::{ remove_failed_compile, write_rule_failed_models, }; -use crate::bindings::_helpers::compile_context::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::phases::{ elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, target_dir, write_artifacts, write_dag_artifact, }; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; use crate::bindings::_helpers::rules::rules_config::rules_config_of; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -41,6 +38,9 @@ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::Status; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::compile_context::main::input_reads::record_compile_input_reads::record_compile_input_reads; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -277,8 +277,8 @@ pub(crate) fn compile_phases<'py>( let skip_write: bool = native_field(request, "profile_flags")? .getattr("skip_write")? .is_truthy()?; - let staging: bool = - !skip_write && crate::bindings::_helpers::compile_context::delta::injection().is_none(); + let staging: bool = !skip_write + && sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none(); let (outcome, input_reads, artifact_writes): ( CommandOutcome<'py>, Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 11f3c28341..1bf2372e9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; use sqlbuild_model_config::templates::models::DeferredSegment; @@ -12,7 +13,6 @@ use sqlbuild_model_config::templates::models::DeferredSegment; use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -176,10 +176,7 @@ fn target_field<'py>( /// A `PlannerInputError` with `message` and the default code. fn planner_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::PlannerInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::PlannerInput, &message) } /// Validate and resolve the deferred target config. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index d5796dda00..ae41fdc99d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::compile_driver::config_validation::{ ProjectConfiguration, reject_invalid_project_configuration, }; use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; -use crate::bindings::_helpers::compile_driver::stage_work::{PROJECT_DISCOVERIES, count_stage}; use crate::bindings::_helpers::discovery::constants::{ ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, }; @@ -43,13 +42,15 @@ use crate::bindings::_helpers::discovery::sql_test_records::{ use crate::bindings::_helpers::discovery::yaml_records::{ discover_schema_files, discover_source_files, }; -use crate::bindings::_helpers::event_exporting::scope::configure_discovered_event_exporters; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produced}; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_driver::constants::PROJECT_DISCOVERIES; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::event_exporting::main::configure_discovered_event_exporters::configure_discovered_event_exporters; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -286,7 +287,7 @@ fn assembled_inputs<'py>( )? .is_truthy()? { - crate::bindings::_helpers::compile_context::delta::stop_capture(); + sqlbuild_python_base::compile_context::main::delta::stop_capture::stop_capture(); } let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 69869f096b..f7248baa18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_attachment::effective_config::{ build_effective_connection, build_effective_settings, build_effective_vars, }; use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; -use crate::bindings::_helpers::compile_context::diagnostic_collector::collect_compile_diagnostics; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::records::{compile_inputs_view, inputs_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index cf1caa44ca..e0b34f994a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -24,7 +24,6 @@ pub(crate) mod python_plan_entries; pub(crate) mod registration; pub(crate) mod relation_targets; pub(crate) mod selection_phases; -pub(crate) mod stage_work; pub(crate) mod static_context; pub(crate) mod target_defaults; pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index d3e189a300..c1f3620e1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -8,9 +8,9 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::diagnostics::trackers::current_tracker; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::diagnostics::main::current_tracker::current_tracker; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 3d8bf5f3f3..6ab72cb494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -25,8 +25,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; -use crate::bindings::_helpers::native_host::connections::connection_failure; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 0461c5e25b..9e397246e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -29,9 +29,6 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; -use crate::bindings::_helpers::compile_driver::stage_work::{ - PROJECT_GRAPHS, PROMOTION_CONFLICTS, count_stage, -}; use crate::bindings::_helpers::compiled_views::records::project_with; use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -40,6 +37,9 @@ use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::compile_driver::constants::PROJECT_GRAPHS; +use sqlbuild_python_base::compile_driver::constants::PROMOTION_CONFLICTS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 52a5cce413..9fa5fc9389 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -1,14 +1,11 @@ //! Register the native compile command driver. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_context::{diagnostic_collector, input_reads}; -use crate::bindings::_helpers::compile_driver::{ - analysis, analysis_cache, assembly, command, deferral, diff, discovery, inputs, pipeline, - pipeline_entries, plan_pipeline, plan_result, project, python_plan_entries, relation_targets, - selection_phases, stage_work, static_context, targets, -}; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::register_diagnostic_collector::register_diagnostic_collector; +use sqlbuild_python_base::compile_context::main::input_reads::register_input_reads::register_input_reads; +use crate::bindings::_helpers::compile_driver::{analysis, analysis_cache, assembly, command, deferral, diff, discovery, inputs, pipeline, pipeline_entries, plan_pipeline, plan_result, project, python_plan_entries, relation_targets, selection_phases, static_context, targets}; +use sqlbuild_python_base::compile_driver::main::register_stage_work::register_stage_work; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; @@ -17,22 +14,19 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { command::register(module)?; project::register(module)?; inputs::register(module)?; - input_reads::register(module)?; + register_input_reads(module)?; discovery::register(module)?; pipeline::register(module)?; plan_pipeline::register(module)?; pipeline_entries::register(module)?; plan_result::register(module)?; deferral::register(module)?; - diagnostic_collector::register(module)?; + register_diagnostic_collector(module)?; diff::register(module)?; python_plan_entries::register(module)?; relation_targets::register(module)?; selection_phases::register(module)?; static_context::register(module)?; - module.add_function(wrap_pyfunction!( - stage_work::compile_stage_work_info, - module - )?)?; + register_stage_work(module)?; targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs deleted file mode 100644 index 12cac440d6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! How often each compile stage ran in this process, for tests that assert stage work. - -use std::collections::BTreeMap; - -use pyo3::prelude::{Py, PyRefMut, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::{pyclass, pyfunction}; - -pub(crate) const EXPRESSION_SHAPES: &str = "infer_native_expression_source_shapes"; -pub(crate) const EXPRESSION_SHAPE_BATCHES: &str = "expression_shape_batches"; -pub(crate) const EXPRESSION_SHAPES_INFERRED: &str = "expression_shapes_inferred"; -pub(crate) const COLUMN_LINEAGE: &str = "build_project_column_lineage"; -pub(crate) const MODEL_CONTRACTS: &str = "evaluate_model_contracts"; -pub(crate) const PROJECT_DISCOVERIES: &str = "discover_project_inputs"; -pub(crate) const MODEL_ANALYSIS: &str = "analyze_native_model_sql"; -pub(crate) const SEMANTIC_COMPLETION: &str = "complete_semantic_project"; -pub(crate) const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; -pub(crate) const PROMOTION_CONFLICTS: &str = "promotion_conflict_diagnostics"; -pub(crate) const PIVOT_PROOFS: &str = "pivot_proofs"; -pub(crate) const SESSION_PIVOT_PROOFS: &str = "session_pivot_proofs"; -pub(crate) const DECLARATION_SCOPES: &str = "build_declaration_scope"; -pub(crate) const LINEAGE_GRAPH_BUILDS: &str = "lineage_graph_builds"; -pub(crate) const PROJECT_GRAPHS: &str = "build_project_graph"; -pub(crate) const FORMAT_LINT_BATCHES: &str = "format_lint_batches"; -pub(crate) const FORMAT_LINT_FILES: &str = "format_lint_files"; -pub(crate) const FORMAT_CACHE_HITS: &str = "format_cache_hits"; -pub(crate) const FORMAT_CACHE_MISSES: &str = "format_cache_misses"; -pub(crate) const SQL_TEST_ARTIFACT_PLANS: &str = "plan_compiled_sql_test_artifacts"; -pub(crate) const SQL_TEST_MODEL_CHAINS: &str = "sql_test_model_chain_names_by_key"; -pub(crate) const BACKGROUND_SQL_TEST_PLANS: &str = "background_sql_test_plans"; -/// Models whose header columns are reused from the native header parse. -pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; -/// Models whose header columns are parsed again from their config values. -pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; -/// Config values converted into typed nodes for a key a reader asked for. -pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads"; -/// Reads of a model or project that is not a native compiled view, read by attribute instead. -pub(crate) const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks"; - -/// Compile stage runs since the last reset, by stage. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct StageWork { - runs: BTreeMap<&'static str, u64>, -} - -static STAGE_WORK: PyOnceLock> = PyOnceLock::new(); - -fn stage_work(py: Python<'_>) -> PyResult> { - Ok(STAGE_WORK - .get_or_try_init(py, || Py::new(py, StageWork::default()))? - .bind(py) - .borrow_mut()) -} - -/// Add `count` runs of `stage`. -pub(crate) fn count_stage(py: Python<'_>, stage: &'static str, count: u64) -> PyResult<()> { - *stage_work(py)?.runs.entry(stage).or_default() += count; - Ok(()) -} - -/// Report the runs of each compile stage so far, optionally resetting them. -#[pyfunction] -#[pyo3(signature = (*, reset = false))] -pub(crate) fn compile_stage_work_info( - py: Python<'_>, - reset: bool, -) -> PyResult> { - let mut work = stage_work(py)?; - let info: BTreeMap = work - .runs - .iter() - .map(|(stage, count)| ((*stage).to_owned(), *count)) - .collect(); - if reset { - *work = StageWork::default(); - } - Ok(info) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs index 6e37e35303..6ebbc96c6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod artifacts; -pub(crate) mod json_report; pub(crate) mod registration; pub(crate) mod reuse; pub(crate) mod runtime_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs index 26fea3bc26..5fe2825d3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_outputs::{json_report, reuse, runtime_target}; +use crate::bindings::_helpers::compile_outputs::{reuse, runtime_target}; +use sqlbuild_python_base::compile_outputs::main::register_json_report::register_json_report; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - json_report::register(module)?; + register_json_report(module)?; reuse::register(module)?; runtime_target::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 57b9a64b2e..79a17820b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -230,11 +230,11 @@ impl NativeReuseAttempt { artifacts_written: record.artifacts_written, dag_path: record.dag_path, json_output: record.json_output, - image: crate::bindings::_helpers::compile_context::delta::take_pending_image(), + image: sqlbuild_python_base::compile_context::main::delta::take_pending_image::take_pending_image(), vouched_target_stamps: None, }; if let Some(merge) = - crate::bindings::_helpers::compile_context::delta::take_record_merge() + sqlbuild_python_base::compile_context::main::delta::take_record_merge::take_record_merge() { for stamp in &merge.kept { compile diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 3fc0c8c420..250d1058f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -19,8 +19,8 @@ use sqlbuild_host::models::{ }; use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; -use crate::bindings::_helpers::native_host::results::ReadResult; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; const RUN_DIR: &str = "run"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs index 2e79147382..4f548c9a9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs @@ -9,11 +9,11 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::types::{PyDict, PyString}; -use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; use crate::bindings::_helpers::compiled_views::function_views::NativeFunctionInput; use crate::bindings::_helpers::compiled_views::source_views::NativeSourceInput; +use sqlbuild_python_base::cache::models::SqlTestScanStore; const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 93935aecfe..f0d6e873c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -21,12 +21,14 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - ImportedDeclarations, imported_declarations, module_declarations, run_controls, run_resources, -}; use crate::bindings::_helpers::native_host::orchestration_triggers::{ CalendarRequest, compile_calendars, compile_sensors, }; +use sqlbuild_python_base::orchestration_reads::main::imported_declarations::imported_declarations; +use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; +use sqlbuild_python_base::orchestration_reads::main::run_controls::run_controls; +use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; +use sqlbuild_python_base::orchestration_reads::models::ImportedDeclarations; use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs index bf6e302515..38f2b1c0e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs @@ -3,8 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind an audit object; any other object is a Python-built record, counted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs index bcfe2464e1..a560c7998d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs @@ -3,8 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a function object; any other object is a Python-built record, counted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index d293161009..93e1daa526 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ DeclaredRow, LineageInput, NativeCompiledModel, }; use sqlbuild_project::types::InferredRow; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One lineage output with its transform kind and confidence values, and its sources. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs index 11edcdbe5a..b3f0994076 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MODELS_FIELD: &str = "models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs index f654a7e1ff..841770af33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs @@ -9,7 +9,6 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{pyclass, pymethods}; -use crate::bindings::_helpers::compile_driver::stage_work::{LINEAGE_GRAPH_BUILDS, count_stage}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ NativeCompiledModel, model_graph_row, @@ -23,6 +22,8 @@ use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSourc use crate::bindings::_helpers::graph::project_graph::{ NativeProjectGraph, ResourceRow, project_graph, }; +use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MODELS_FIELD: &str = "models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs index 25a6ff3bbc..95ebe5b0d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs @@ -3,7 +3,6 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ NativeCompiledModel, model_graph_row, }; @@ -12,6 +11,8 @@ use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A resource's graph facts without constructing its public keys or dependency tuple. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs index ab65b2226a..1912c33f0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs @@ -3,8 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a seed object; any other object is a Python-built record, counted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs index 3da441bceb..46dc34fa56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs @@ -3,8 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The native view behind a source object; any other object is a Python-built record, counted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs index 06821468aa..a06822539d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs @@ -9,6 +9,7 @@ use sqlbuild_cli::compute_logs::models::{ CaptureCompletion, CaptureDocument, CaptureInstant, InventoryEntry, }; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -35,10 +36,7 @@ fn exception_class(kind: ComputeLogErrorKind) -> ExecutorHost { /// The Python exception for a storage rejection of `kind`. pub(crate) fn rejection(py: Python<'_>, kind: ComputeLogErrorKind, message: &str) -> PyErr { - match host_entry(py, exception_class(kind)).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, exception_class(kind), message) } /// The Python exception a native storage failure raises. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 0826477f7c..7abdec05e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -19,20 +19,20 @@ use sqlbuild_core::contract_command::models::{ }; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::refactor::contract_compare::compare_contracts; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 9a0cd62366..3565cb6f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -3,6 +3,4 @@ pub(crate) mod effective_config; pub(crate) mod exception_types; pub(crate) mod model_contracts; pub(crate) mod registration; -pub(crate) mod resource_identities; pub(crate) mod spec_resolution; -pub(crate) mod state_literals; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 3d7599cdd2..e3f1ba85ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -17,8 +17,8 @@ use sqlbuild_analysis::sql_macros::main::untyped_call_outputs::untyped_call_outp use sqlbuild_analysis::type_system::errors::TypeNormalizationError; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::type_system::normalization::normalization_error; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::type_system::main::normalization_error::normalization_error; use sqlbuild_python_base::types::CompilerDetach; /// One contract diagnostic: its code, severity, message, column, location, related output and help. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index db24e3be65..30f503d77e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -4,18 +4,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::authored_values::sql_values; use crate::bindings::_helpers::contracts::{ - contract_diagnostics, effective_config, exception_types, resource_identities, spec_resolution, - state_literals, + contract_diagnostics, effective_config, exception_types, spec_resolution, }; use crate::bindings::_helpers::discovery::project_config; +use sqlbuild_python_base::contracts::main::register_resource_identities::register_resource_identities; +use sqlbuild_python_base::contracts::main::register_state_literals::register_state_literals; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { contract_diagnostics::register(module)?; effective_config::register(module)?; exception_types::register(module)?; project_config::register(module)?; - resource_identities::register(module)?; + register_resource_identities(module)?; spec_resolution::register(module)?; sql_values::register(module)?; - state_literals::register(module) + register_state_literals(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs index cadad17d1e..3d44849997 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs @@ -17,8 +17,8 @@ use crate::bindings::_helpers::cost_command::bounds::{parse_bound, parse_since}; use crate::bindings::_helpers::cost_command::refresh::refresh_pending_cost_run; use crate::bindings::_helpers::cost_command::refusal::cost_error; use crate::bindings::_helpers::cost_report::store; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field; /// Preserve Python's datetime parsing at the public request edge, then leave Python records behind. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index e7e6deddbf..56ba668b30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -10,7 +10,6 @@ use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refres use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_target_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -22,6 +21,7 @@ use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 8c3830bfb5..366ecd2dd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -1,5 +1,6 @@ //! One call's cursor values read from and built as the native cursor value records. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::cell::RefCell; use pyo3::IntoPyObject; @@ -20,7 +21,6 @@ use crate::bindings::_helpers::cursor::values::{ }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -39,10 +39,7 @@ pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ /// `CursorAlgebraError(message)`. pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - match host_entry(py, FacadeHost::CursorAlgebraError).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, FacadeHost::CursorAlgebraError, message) } /// The cursor values one native call reads, so unchanged inputs come back as the same objects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs index a12d3c2eab..4a80d9bb8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs @@ -13,7 +13,7 @@ use sqlbuild_analysis::dag_artifact::types::DagValue; use sqlbuild_analysis::dag_artifact::types::DagColumnLineage; -use crate::bindings::_helpers::python_values::values::python_value; +use sqlbuild_python_base::python_values::main::python_value::python_value; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs index 0a008b0a8c..a9ed9eb49c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -1,5 +1,6 @@ //! DAG artifact loading and SQLBuild selectors resolved over a DAG artifact mapping. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::collections::BTreeSet; use pyo3::exceptions::PyOSError; @@ -18,7 +19,6 @@ use sqlbuild_host::entries::models::ThirdPartyHost; use crate::bindings::_helpers::planner_selection::carrier::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; @@ -100,9 +100,9 @@ fn selection_error(py: Python<'_>, error: DagSelectionError, raw: &str) -> PyErr DagSelectionError::UnitTest(selector) => { carrier.error(unit_test_selector_rejection(&selector)) } - DagSelectionError::Input(message) => host_entry(py, ThirdPartyHost::DagsterDagInputError) - .and_then(|class| class.call1((message,))) - .map_or_else(|failure| failure, PyErr::from_value), + DagSelectionError::Input(message) => { + host_error(py, ThirdPartyHost::DagsterDagInputError, &message) + } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs index f83971c093..5a6f00f6a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs @@ -1,5 +1,6 @@ //! Build or validate a SQLBuild project from a dbt project, its profile and `dbt debug`. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::io; use pyo3::exceptions::PyRuntimeError; @@ -349,11 +350,7 @@ fn is_value_error(py: Python<'_>, error: &PyErr) -> bool { } fn profile_error(py: Python<'_>, message: String) -> PyErr { - match host_entry(py, ThirdPartyHost::DbtProfileError).and_then(|class| class.call1((message,))) - { - Ok(instance) => PyErr::from_value(instance), - Err(error) => error, - } + host_error(py, ThirdPartyHost::DbtProfileError, &message) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs index 86dc4fe197..c14a530f2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -3,9 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; /// `sys.stdout` or `sys.stderr`, looked up when written. pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs index e52ed1e6d5..f130400958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs @@ -22,7 +22,6 @@ use sqlbuild_core::dbt_process::main::spawn_dbt_json_process::spawn_dbt_json_pro use sqlbuild_core::dbt_process::main::stop_dbt_process::stop_dbt_process; use sqlbuild_core::dbt_process::models::DbtJsonProcess; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::dbt_events::stream::DbtEventStream; use crate::bindings::_helpers::dbt_interop::runner::{ dbt_command_lifecycle, launch_failure, launch_os_error, report_exit, @@ -30,6 +29,7 @@ use crate::bindings::_helpers::dbt_interop::runner::{ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index 6830352885..fec7e7abd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -8,7 +8,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; use std::time::Instant; -use crate::bindings::_helpers::build_command::support::{boolean, none, text}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; @@ -23,6 +22,9 @@ use crate::bindings::_helpers::dbt_interop::runtime_config::config_overrides; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 65348244ed..4711e6a15a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::build_command::support::{boolean, emit, none, text, tuple}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; @@ -17,6 +16,11 @@ use crate::bindings::_helpers::dbt_interop::runtime_execution::execute_dbt_comma use crate::bindings::_helpers::dbt_interop::sqlbuild_work::execute_dbt_sqlbuild_work; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index bddddeb635..138bd0bf77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, none}; use crate::bindings::_helpers::command_runtime::connection::{ anchor_local_database_path, with_dbt_profile, }; @@ -18,6 +17,9 @@ use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::keyword_relay::keyword_relay; +use sqlbuild_python_base::build_command::main::none::none; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs index c932430278..42f842b790 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs @@ -12,9 +12,9 @@ use sqlbuild_config::dbt_profile::models::{ ConnectionKey, ConnectionSource, DbtInitConfig, NormalizedProfile, ProfileEntry, ProfileValue, }; use sqlbuild_host::entries::models::ThirdPartyHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; @@ -35,11 +35,7 @@ const INIT_TOML_OPTIONS: (&[&str], &[&str]) = ( ); pub(crate) fn raised(py: Python<'_>, error: DbtProfileError) -> PyErr { - match host_entry(py, ThirdPartyHost::DbtProfileError).and_then(|class| class.call1((error.0,))) - { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } + host_error(py, ThirdPartyHost::DbtProfileError, &error.0) } fn profile_value(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs index 7bb9db8da4..02bdf28c81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_host::entries::models::ThirdPartyHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use crate::bindings::_helpers::dbt_interop::cli::resolved_path; use crate::bindings::_helpers::dbt_interop::profile::normalize_dbt_profile_connection; @@ -31,11 +32,7 @@ const PROFILE_CONNECTION_OPTIONS: (&[&str], &[&str]) = ( const NON_EMPTY_MESSAGE: &str = "dbt_profile connection values must be non-empty strings"; pub(crate) fn profile_error(py: Python<'_>, message: String) -> PyErr { - match host_entry(py, ThirdPartyHost::DbtProfileError).and_then(|class| class.call1((message,))) - { - Ok(instance) => PyErr::from_value(instance), - Err(error) => error, - } + host_error(py, ThirdPartyHost::DbtProfileError, &message) } /// A raw connection value as a stripped non-empty string, or `None` when absent. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs index 73afb01850..85e0f56b1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -1,18 +1,18 @@ //! Execute dbt through the subprocess boundary and finalize results in artifact order. -use crate::bindings::_helpers::build_progress::transient_status::TransientStatusReporter; use crate::bindings::_helpers::dbt_events::stream::render_dbt_node_result; use crate::bindings::_helpers::dbt_interop::cli::parse_dbt_ls_json_lines; use crate::bindings::_helpers::dbt_interop::event_stream::execute_dbt_json_event_stream; use crate::bindings::_helpers::dbt_interop::execution::dbt_execution_ls_argv; use crate::bindings::_helpers::dbt_interop::runtime_results::{append_missing, result_ids}; -use crate::bindings::_helpers::presentation_text::style::PythonCliStyle; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyInt, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::models::TransientStatusReporter; +use sqlbuild_python_base::presentation_text::models::PythonCliStyle; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index e075b271a7..ab6dd33019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -11,9 +11,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{BuildExecutionResult, BuildStatus}; use crate::bindings::_helpers::build_command::policies::enforce_plan_safety_policies_binding; -use crate::bindings::_helpers::build_command::support::{ - boolean, emit, joined, keyword_relay, none, text, tuple, -}; use crate::bindings::_helpers::build_progress::build_callbacks::{ BuildProgressCallbacks, batch_complete_callback, }; @@ -22,12 +19,19 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_text::format_execution_header; use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::compile_outputs::runtime_target::write_runtime_target; -use crate::bindings::_helpers::native_host::results::ReadResult; -use crate::bindings::_helpers::presentation_text::style::PythonCliStyle; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::emit::emit; +use sqlbuild_python_base::build_command::main::joined::joined; +use sqlbuild_python_base::build_command::main::keyword_relay::keyword_relay; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; +use sqlbuild_python_base::presentation_text::models::PythonCliStyle; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 79ae70884e..3c21e86d6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -23,7 +23,6 @@ use sqlbuild_core::debug_report::models::{ }; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::{ local_database_files, project_connection_config, }; @@ -39,16 +38,16 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; use crate::bindings::_helpers::discovery::configuration::discover_configured_providers; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; -use crate::bindings::_helpers::native_host::provider_settings::{ - missing_secrets, secret_variables, -}; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; +use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_secrets; +use sqlbuild_python_base::orchestration_reads::main::secret_variables::secret_variables; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index fb03912c6b..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1,6 +1 @@ -pub(crate) mod console_records; -pub(crate) mod getters; -pub(crate) mod log_records; -pub(crate) mod redaction; pub(crate) mod registration; -pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs index f3a44ab4cd..ebe06cf0fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::diagnostics::{console_records, log_records, redaction, trackers}; +use sqlbuild_python_base::diagnostics::main::register_console_records::register_console_records; +use sqlbuild_python_base::diagnostics::main::register_log_records::register_log_records; +use sqlbuild_python_base::diagnostics::main::register_redaction::register_redaction; +use sqlbuild_python_base::diagnostics::main::register_trackers::register_trackers; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - console_records::register(module)?; - log_records::register(module)?; - redaction::register(module)?; - trackers::register(module) + register_console_records(module)?; + register_log_records(module)?; + register_redaction(module)?; + register_trackers(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 1eb7899162..1389abde3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -34,11 +34,11 @@ use crate::bindings::_helpers::diff_report::support::{ request_facts, user_error_parts, }; use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; -use crate::bindings::_helpers::frontier::encoder::emit_error; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs index 535af1767a..01981aceeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs @@ -10,11 +10,11 @@ use sqlbuild_cli::diff_report::models::{ }; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; -use crate::bindings::_helpers::frontier::encoder::emit_error; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; const DEFAULT_MAX_VALUE_LENGTH: usize = 160; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 2b15fd84c2..90f7869f5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -15,7 +15,6 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; @@ -29,6 +28,7 @@ use sqlbuild_analysis::diff_selection::models::RawQueryShape; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs index 9fdbdf97be..c787bcdc5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_keys.rs @@ -4,7 +4,8 @@ use sqlbuild_discovery::header_changes::main::header_key_digests::header_key_dig use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::compile_context::delta::{capture, capturing}; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; /// Record one discovered model's header keys for the image. pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 026499a9be..434dbf3a20 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -3,16 +3,16 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::discovery::locations::project_relative_path; use crate::bindings::_helpers::discovery::python_module_files::discover_python_module_files; -use crate::bindings::_helpers::event_exporting::scope::current_scope; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::event_exporting::main::current_scope::current_scope; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -113,9 +113,7 @@ pub(crate) fn discovered<'py>( /// `sqlbuild._native.(message)`. pub(crate) fn discovery_error(py: Python<'_>, class_name: ErrorId, message: String) -> PyErr { - error_class(py, class_name) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value) + new_plain_error(py, class_name, &message) } /// `discovery_error` raised from `cause`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index 01f30aed94..baa821e746 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -9,19 +9,20 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_context::input_reads::record_settings_class_read; -use crate::bindings::_helpers::contracts::resource_identities::validate_resource_identity; use crate::bindings::_helpers::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; use crate::bindings::_helpers::discovery::imported_modules::{ ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, provider_by_name, provider_usages, shim, tuple_of, }; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; -use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::compile_context::main::input_reads::record_settings_class_read::record_settings_class_read; +use sqlbuild_python_base::contracts::main::validate_resource_identity::validate_resource_identity; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::orchestration_reads::main::provider_state::provider_state; +use sqlbuild_python_base::orchestration_reads::models::ProviderState; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index c61363a29e..9c8b60bc56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -115,7 +115,7 @@ pub(crate) fn discovered_model_files<'py>( let py: Python<'py> = project_dir.py(); let options: ModelFileOptions = model_file_options( extraction, - crate::bindings::_helpers::compile_context::delta::injection() + sqlbuild_python_base::compile_context::main::delta::injection::injection() .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), ); let items: Vec> = model_items(project_dir, &options)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs index f4f0288eb7..fbc3692ebc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs @@ -15,7 +15,7 @@ use sqlbuild_discovery::orchestration_files::models::{ use sqlbuild_orchestration::scheduler::main::parse_cron::parse_cron; use sqlbuild_orchestration::store::errors::ControlError; -use crate::bindings::_helpers::native_host::orchestration_declarations::run_resources; +use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; /// A `D017` at `position` of `path`, worded as discovery words its own. type ValueError = (FilePosition, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index c1678788e3..a2dca5cebf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -1,6 +1,5 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::bindings::_helpers::compile_context::delta::injection; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; @@ -12,6 +11,7 @@ use sqlbuild_discovery::project_validation::models::{ NodeFact, NodeKind, ProjectFacts, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, SeedFileFact, SourceFact, SourceFileFact, TestBlockFact, TestFileFact, }; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs index dd10e35080..e80a1f7e5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs @@ -5,7 +5,7 @@ use std::cell::Cell; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::native_host::provider_settings::missing_secrets; +use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_secrets; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs index 75b1488c2c..82b5c7185e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::models::ProjectRoot; use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_python_base::boundary::main::context_variable::context_variable; use std::path::PathBuf; use crate::bindings::_helpers::discovery::raised::discovery_error; @@ -28,19 +29,8 @@ const SURROGATE_PASS: &str = "surrogatepass"; static ACTIVE_SNAPSHOT: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the active snapshot, created once per interpreter. -fn active_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - ACTIVE_SNAPSHOT - .get_or_try_init(py, || { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("default", py.None())?; - Ok::<_, pyo3::PyErr>( - py.import("contextvars")? - .getattr("ContextVar")? - .call((SNAPSHOT_VARIABLE,), Some(&keywords))? - .unbind(), - ) - }) - .map(|variable| variable.bind(py)) +fn active_variable(py: Python<'_>) -> PyResult> { + context_variable(py, &ACTIVE_SNAPSHOT, SNAPSHOT_VARIABLE) } /// The active snapshot when it belongs to `project_dir`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 7798a14185..5ce345c80d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -55,7 +55,7 @@ type RecordBuilder<'py, T> = fn options() -> PyResult { Ok(sql_test_file_options( - crate::bindings::_helpers::compile_context::delta::injection() + sqlbuild_python_base::compile_context::main::delta::injection::injection() .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 72dc055cae..e04bf5ca27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -349,9 +349,9 @@ pub(crate) fn parse_sources_document<'py>( parse_error(py, error, SOURCE_PARSE_ERROR).unwrap_or_else(|error| error) })?; let referenced: Option> = - crate::bindings::_helpers::compile_context::delta::injection() + sqlbuild_python_base::compile_context::main::delta::injection::injection() .and_then(|injection| injection.source_names.clone()); - crate::bindings::_helpers::compile_context::delta::record_source_reads( + sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads( loaded_sources(&sources).iter().map(String::as_str), ); let mut built: Vec> = Vec::with_capacity(sources.len()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index f4a843c00d..5db87ecd59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -14,6 +14,7 @@ use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; +use sqlbuild_python_base::compile_context::main::yaml_cache::retain_read_sources::retain_read_sources; use std::path::PathBuf; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; @@ -155,13 +156,14 @@ fn load_files<'py>( let py: Python<'py> = project_dir.py(); let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; let root: ProjectRoot = project_root(project_dir)?; - let mut documents: Option< - crate::bindings::_helpers::compile_context::yaml_cache::DocumentStore, - > = if matches!(kind, YamlKind::Source) { - crate::bindings::_helpers::compile_context::yaml_cache::open_documents(py) - } else { - None - }; + let mut documents: Option = + if matches!(kind, YamlKind::Source) { + sqlbuild_python_base::compile_context::main::yaml_cache::open_documents::open_documents( + py, + ) + } else { + None + }; for batch in batches(&tree.get().inner, paths, YAML_BATCH_BYTES) { let named: Vec = batch .iter() @@ -185,9 +187,7 @@ fn load_files<'py>( let mut outcomes: Vec = outcomes.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; if matches!(kind, YamlKind::Source) { - crate::bindings::_helpers::compile_context::yaml_cache::retain_read_sources( - &mut outcomes, - ); + outcomes = retain_read_sources(outcomes); } let mut outcomes = outcomes.into_iter(); for path in batch { @@ -212,7 +212,9 @@ fn load_files<'py>( } } if let Some(documents) = &documents { - crate::bindings::_helpers::compile_context::yaml_cache::save_documents(py, documents); + sqlbuild_python_base::compile_context::main::yaml_cache::save_documents::save_documents( + py, documents, + ); } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs deleted file mode 100644 index 40a01f27cc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Python modules, thread names and wait limits of native lifecycle-event export. - -pub(crate) const SCOPE_CONTEXT_VARIABLE: &str = "sqlbuild_event_exporter_command_scope"; -pub(crate) const DISPATCHER_THREAD_NAME: &str = "sqlbuild-event-exporter-dispatcher"; -pub(crate) const NOTIFIER_THREAD_NAME: &str = "sqlbuild-event-exporter-notifier"; -pub(crate) const HEALTH_THREAD_NAME: &str = "sqlbuild-event-exporter-health"; -pub(crate) const OUTPUT_SINK_THREAD_NAME: &str = "sqlbuild-command-output-sink"; -/// Prefix of each exporter's long-lived delivery thread name. -pub(crate) const INVOCATION_THREAD_PREFIX: &str = "sqlbuild-event-exporter-"; -/// Attribute naming the worker of a long-lived dispatcher thread, for the exit hook. -pub(crate) const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; -/// The longest single wait of a polling delivery loop, as the Python dispatcher polled. -pub(crate) const POLL_SECONDS: f64 = 0.01; -/// The most one dispatch round charges a sink call, so the dispatcher's GIL waits are not counted. -pub(crate) const CHARGE_LIMIT_SECONDS: f64 = 2.0 * POLL_SECONDS; -/// Extra time `shutdown` waits for the dispatcher thread beyond the drain deadline. -pub(crate) const JOIN_GRACE_SECONDS: f64 = 0.05; -/// How long shutdown and interpreter exit wait for each stopped dispatcher thread. -pub(crate) const EXIT_JOIN_SECONDS: f64 = 1.0; -/// The most attempts one exporter's delivery thread holds, at most the queue capacity; more stay queued. -pub(crate) const LANE_WINDOW: usize = 64; -/// The longest a finalizer waits for sink calls still running after shutdown before it runs anyway. -pub(crate) const IDLE_WAIT_SECONDS: f64 = 1.0; -/// No time left before a deadline. -pub(crate) const NO_SECONDS: f64 = 0.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index feb703528c..34ee4b45f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -26,26 +26,31 @@ use sqlbuild_core::event_export::models::{ }; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::constants::{ - DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, IDLE_WAIT_SECONDS, - JOIN_GRACE_SECONDS, NO_SECONDS, NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, - WORKER_ATTRIBUTE, -}; -use crate::bindings::_helpers::event_exporting::models::{ - BoundEventExporter, EventExportSummary, EventExporterFailure, -}; -use crate::bindings::_helpers::event_exporting::values::{ - export_policy, input_error, severity_from, state_error, -}; use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, Lane, WorkerRole}; -use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; -use crate::bindings::_helpers::native_threads::exit_signal::{ - interpreter_exiting, signal_interpreter_exit, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; +use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::IDLE_WAIT_SECONDS; +use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::OUTPUT_SINK_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; +use sqlbuild_python_base::event_exporting::constants::WORKER_ATTRIBUTE; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::main::input_error::input_error; +use sqlbuild_python_base::event_exporting::main::severity_from::severity_from; +use sqlbuild_python_base::event_exporting::main::state_error::state_error; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; +use sqlbuild_python_base::native_threads::main::interpreter_exiting::interpreter_exiting; +use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One bound lifecycle-event sink. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs index afdb63c544..ee7ad4da9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs @@ -4,11 +4,12 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; -use crate::bindings::_helpers::event_exporting::models::{ - BoundEventExporter, EventExportSummary, EventExporterAccounting, EventExporterCounts, - EventExporterFailure, -}; -use crate::bindings::_helpers::event_exporting::scope::EventExporterCommandScope; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; +use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; +use sqlbuild_python_base::event_exporting::models::EventExporterCounts; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs index 2425304a06..16eba7bfb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs @@ -1,9 +1,5 @@ -pub(crate) mod constants; pub(crate) mod dispatcher; pub(crate) mod getters; -pub(crate) mod models; pub(crate) mod policy; pub(crate) mod registration; -pub(crate) mod scope; -pub(crate) mod values; pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs deleted file mode 100644 index 9170225d62..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs +++ /dev/null @@ -1,320 +0,0 @@ -//! Immutable event exporter accounting, failure and binding values Python reads. - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; -use sqlbuild_core::event_export::constants::EVENT_EXPORT_KINDS; -use sqlbuild_core::event_export::models::{ - ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, -}; - -use crate::bindings::_helpers::event_exporting::values::{ - accounting_repr, bound_exporter_arguments, counts_repr, python_repr, severity_from, - severity_object, text_repr, -}; - -/// Frozen attempt counters for one exporter or their aggregate. -#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(crate) struct EventExporterCounts { - pub(crate) counts: ExporterCounts, -} - -/// Frozen accounting snapshot for one named exporter. -#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(crate) struct EventExporterAccounting { - exporter_name: String, - counts: ExporterCounts, -} - -/// Final or point-in-time best-effort delivery accounting. -#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(crate) struct EventExportSummary { - pub(crate) snapshot: ExportSnapshot, -} - -/// Sanitized exporter failure information safe for diagnostics. -#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(crate) struct EventExporterFailure { - pub(crate) exporter_name: String, - pub(crate) error_type: String, - pub(crate) event_kind: String, - pub(crate) event_severity: ExportSeverity, -} - -/// One lifecycle-event sink bound to command-scoped provider instances. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct BoundEventExporter { - pub(crate) name: String, - pub(crate) function: Py, - pub(crate) provider_arguments: Py, - pub(crate) filter: ExporterFilter, -} - -#[pymethods] -impl EventExporterCounts { - #[getter] - pub(crate) fn accepted(&self) -> u64 { - self.counts.accepted - } - - #[getter] - pub(crate) fn filtered(&self) -> u64 { - self.counts.filtered - } - - #[getter] - pub(crate) fn delivered(&self) -> u64 { - self.counts.delivered - } - - #[getter] - pub(crate) fn failed(&self) -> u64 { - self.counts.failed - } - - #[getter] - pub(crate) fn dropped(&self) -> u64 { - self.counts.dropped - } - - fn __repr__(&self) -> String { - counts_repr(&self.counts) - } -} - -#[pymethods] -impl EventExporterAccounting { - #[getter] - pub(crate) fn exporter_name(&self) -> &str { - &self.exporter_name - } - - #[getter] - pub(crate) fn counts(&self) -> EventExporterCounts { - EventExporterCounts { - counts: self.counts, - } - } - - fn __repr__(&self, py: Python<'_>) -> PyResult { - accounting_repr(py, (&self.exporter_name, &self.counts)) - } -} - -#[pymethods] -impl EventExportSummary { - #[getter] - pub(crate) fn aggregate(&self) -> EventExporterCounts { - EventExporterCounts { - counts: self.snapshot.aggregate, - } - } - - #[getter] - pub(crate) fn per_exporter<'py>(&self, py: Python<'py>) -> PyResult> { - let items: Vec = self - .snapshot - .per_exporter - .iter() - .map(|(name, counts)| EventExporterAccounting { - exporter_name: name.clone(), - counts: *counts, - }) - .collect(); - PyTuple::new(py, items) - } - - #[getter] - pub(crate) fn queue_depth(&self) -> usize { - self.snapshot.queue_depth - } - - #[getter] - pub(crate) fn queue_capacity(&self) -> usize { - self.snapshot.queue_capacity - } - - #[getter] - pub(crate) fn flush_complete(&self) -> bool { - self.snapshot.flush_complete - } - - #[getter] - pub(crate) fn accepted(&self) -> u64 { - self.snapshot.aggregate.accepted - } - - #[getter] - pub(crate) fn filtered(&self) -> u64 { - self.snapshot.aggregate.filtered - } - - #[getter] - pub(crate) fn delivered(&self) -> u64 { - self.snapshot.aggregate.delivered - } - - #[getter] - pub(crate) fn failed(&self) -> u64 { - self.snapshot.aggregate.failed - } - - #[getter] - pub(crate) fn dropped(&self) -> u64 { - self.snapshot.aggregate.dropped - } - - fn __repr__(&self, py: Python<'_>) -> PyResult { - let per_exporter: Vec = self - .snapshot - .per_exporter - .iter() - .map(|(name, counts)| accounting_repr(py, (name, counts))) - .collect::>>()?; - let per_exporter: String = match per_exporter.as_slice() { - [single] => format!("({single},)"), - items => format!("({})", items.join(", ")), - }; - Ok(format!( - "EventExportSummary(aggregate={}, per_exporter={per_exporter}, queue_depth={}, \ - queue_capacity={}, flush_complete={})", - counts_repr(&self.snapshot.aggregate), - self.snapshot.queue_depth, - self.snapshot.queue_capacity, - if self.snapshot.flush_complete { - "True" - } else { - "False" - } - )) - } -} - -#[pymethods] -impl EventExporterFailure { - #[getter] - pub(crate) fn exporter_name(&self) -> &str { - &self.exporter_name - } - - #[getter] - pub(crate) fn error_type(&self) -> &str { - &self.error_type - } - - #[getter] - pub(crate) fn event_kind(&self) -> &str { - &self.event_kind - } - - #[getter] - pub(crate) fn event_severity<'py>(&self, py: Python<'py>) -> PyResult> { - severity_object(py, self.event_severity) - } - - fn __repr__(&self, py: Python<'_>) -> PyResult { - Ok(format!( - "EventExporterFailure(exporter_name={}, error_type={}, event_kind={}, event_severity={})", - text_repr(py, &self.exporter_name)?, - text_repr(py, &self.error_type)?, - text_repr(py, &self.event_kind)?, - python_repr(&severity_object(py, self.event_severity)?)? - )) - } -} - -#[pymethods] -impl BoundEventExporter { - #[new] - #[pyo3(signature = (*arguments, **options))] - fn new(arguments: &Bound<'_, PyTuple>, options: Option<&Bound<'_, PyDict>>) -> PyResult { - let [ - name, - function, - provider_arguments, - event_kinds, - min_severity, - ] = bound_exporter_arguments(arguments, options)?; - let (Some(name), Some(function), Some(provider_arguments)) = - (name, function, provider_arguments) - else { - return Err(pyo3::exceptions::PyTypeError::new_err( - "BoundEventExporter() missing required arguments", - )); - }; - Self::create( - name.extract()?, - (function.unbind(), provider_arguments.unbind()), - (event_kinds.as_ref(), min_severity.as_ref()), - ) - } - - #[getter] - pub(crate) fn name(&self) -> &str { - &self.name - } - - #[getter] - pub(crate) fn function(&self, py: Python<'_>) -> Py { - self.function.clone_ref(py) - } - - #[getter] - pub(crate) fn provider_arguments(&self, py: Python<'_>) -> Py { - self.provider_arguments.clone_ref(py) - } - - #[getter] - pub(crate) fn event_kinds<'py>(&self, py: Python<'py>) -> PyResult> { - PyFrozenSet::new(py, self.filter.event_kinds.iter()) - } - - #[getter] - pub(crate) fn min_severity<'py>(&self, py: Python<'py>) -> PyResult> { - severity_object(py, self.filter.min_severity) - } -} - -impl BoundEventExporter { - /// A sink bound to `provider_arguments`, filtered by `event_kinds` and `min_severity` when given. - pub(crate) fn create( - name: String, - (function, provider_arguments): (Py, Py), - (event_kinds, min_severity): (Option<&Bound<'_, PyAny>>, Option<&Bound<'_, PyAny>>), - ) -> PyResult { - let event_kinds = match event_kinds { - Some(kinds) => kinds - .try_iter()? - .map(|kind| kind.and_then(|kind| kind.extract::())) - .collect::>()?, - None => EVENT_EXPORT_KINDS - .iter() - .map(|kind| (*kind).to_owned()) - .collect(), - }; - let min_severity: ExportSeverity = match min_severity { - Some(severity) => severity_from(severity)?, - None => ExportSeverity::Debug, - }; - Ok(Self { - name, - function, - provider_arguments, - filter: ExporterFilter { - event_kinds, - min_severity, - }, - }) - } -} - -impl EventExportSummary { - /// A Python summary of `snapshot`. - pub(crate) fn object(py: Python<'_>, snapshot: ExportSnapshot) -> PyResult> { - Py::new(py, Self { snapshot }) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs index 8641e01e07..e5b4153519 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs @@ -10,9 +10,9 @@ use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity}; use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION; use sqlbuild_core::lifecycle_events::main::catalog::catalog; -use crate::bindings::_helpers::event_exporting::values::{ - export_policy, severity_from, severity_object, -}; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::main::severity_from::severity_from; +use sqlbuild_python_base::event_exporting::main::severity_object::severity_object; /// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs index f073cfd678..6335096ee9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -5,13 +5,15 @@ use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, wr use crate::bindings::_helpers::event_exporting::dispatcher::{ EventExporterDispatcher, stop_event_exporter_threads, }; -use crate::bindings::_helpers::event_exporting::models::{ - BoundEventExporter, EventExportSummary, EventExporterAccounting, EventExporterCounts, - EventExporterFailure, -}; +use crate::bindings::_helpers::event_exporting::policy; use crate::bindings::_helpers::event_exporting::workers::DispatcherWorker; -use crate::bindings::_helpers::event_exporting::{policy, scope}; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::event_exporting::main::register_scope::register_scope; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; +use sqlbuild_python_base::event_exporting::models::EventExporterCounts; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; @@ -25,5 +27,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(hook.clone())?; imported(module.py(), "atexit", "register")?.call1((hook,))?; policy::register(module)?; - scope::register(module) + register_scope(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs deleted file mode 100644 index 3b7774b5f6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs +++ /dev/null @@ -1,521 +0,0 @@ -//! `EventExporterCommandScope`: typed sink binding, its provider session and the active command scope. - -use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard}; - -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyclass, - pyfunction, pymethods, wrap_pyfunction, -}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; - -use crate::bindings::_helpers::event_exporting::constants::SCOPE_CONTEXT_VARIABLE; -use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; -use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::context_variable::context_variable; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::native_host::main::write_console::write_console; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; - -static CURRENT_SCOPE: PyOnceLock> = PyOnceLock::new(); - -/// Discovered extensions and the provider session one command bound. -struct ScopeState { - configured: bool, - provider_close_registered: bool, - provider_session: Option>, - project_dir: Option>, - providers: Option>, - event_exporters: Option>, - command_output_sinks: Option>, -} - -/// Bind discovered typed sinks and own their command-scoped provider session. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct EventExporterCommandScope { - dispatcher: Py, - state: Mutex, -} - -/// Releases and force-closes a provider session on the closing thread once sink delivery is idle. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ProviderSessionCloser { - session: Py, -} - -/// Installs one command-owned exporter scope while entered. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct EventExporterScopeContext { - scope: Py, - token: Mutex>>, -} - -fn empty_tuple(py: Python<'_>) -> Py { - PyTuple::empty(py).into_any().unbind() -} - -fn optional(py: Python<'_>, value: Option<&Py>) -> Py { - value.map_or_else(|| py.None(), |value| value.clone_ref(py)) -} - -/// `{usage.parameter_name: session.get(usage.provider_name)}` for one declaration. -fn provider_arguments<'py>( - session: &Bound<'py, PyAny>, - declaration: &Bound<'py, PyAny>, -) -> PyResult> { - let arguments: Bound<'py, PyDict> = PyDict::new(session.py()); - for usage in native_field(declaration, "provider_usages")?.try_iter()? { - let usage: Bound<'py, PyAny> = usage?; - arguments.set_item( - native_field(&usage, "parameter_name")?, - mapping_get(session, native_field(&usage, "provider_name")?)?, - )?; - } - Ok(arguments) -} - -/// Each discovered provider whose required secret settings are unset, with their variables. -fn providers_missing_secrets( - providers: &Bound<'_, PyAny>, -) -> PyResult>> { - let mut missing: HashMap> = HashMap::new(); - for provider in providers.try_iter()? { - let provider: Bound<'_, PyAny> = provider?; - if let ProviderState::MissingSecrets(variables) = - provider_state(&native_field(&provider, "provider_class")?)? - { - missing.insert(native_field(&provider, "name")?.extract()?, variables); - } - } - Ok(missing) -} - -/// Whether `declaration` can bind; when a provider it takes is missing secrets, a notice says it is left unbound. -fn bindable( - declaration: &Bound<'_, PyAny>, - missing: &HashMap>, -) -> PyResult { - for usage in native_field(declaration, "provider_usages")?.try_iter()? { - let provider: String = native_field(&usage?, "provider_name")?.extract()?; - if let Some(variables) = missing.get(&provider) { - write_console( - declaration.py(), - "stderr", - &format!( - "sink '{}' is not bound: provider '{provider}' is missing {}\n", - native_field(declaration, "name")?.str()?, - variables.join(", ") - ), - )?; - return Ok(false); - } - } - Ok(true) -} - -/// Bound lifecycle exporters and command-output sinks over `session`, except those using `missing` providers. -fn bindings<'py>( - session: &Bound<'py, PyAny>, - (event_exporters, command_output_sinks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - missing: &HashMap>, -) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyTuple>)> { - let py: Python<'py> = session.py(); - let mut exporters: Vec> = Vec::new(); - for exporter in event_exporters.try_iter()? { - let exporter: Bound<'py, PyAny> = exporter?; - if !bindable(&exporter, missing)? { - continue; - } - let arguments: Bound<'py, PyDict> = provider_arguments(session, &exporter)?; - exporters.push( - Bound::new( - py, - BoundEventExporter::create( - native_field(&exporter, "name")?.extract()?, - ( - native_field(&exporter, "function")?.unbind(), - arguments.into_any().unbind(), - ), - ( - Some(&native_field(&exporter, "event_kinds")?), - Some(&native_field(&exporter, "min_severity")?), - ), - )?, - )? - .into_any(), - ); - } - let sink_class: Bound<'py, PyAny> = host_entry(py, ExecutorHost::BoundCommandOutputSink)?; - let mut sinks: Vec> = Vec::new(); - for sink in command_output_sinks.try_iter()? { - let sink: Bound<'py, PyAny> = sink?; - if !bindable(&sink, missing)? { - continue; - } - let arguments: Bound<'py, PyDict> = provider_arguments(session, &sink)?; - sinks.push(call_with( - &sink_class, - &[ - ("name", native_field(&sink, "name")?), - ("function", native_field(&sink, "function")?), - ("provider_arguments", arguments.into_any()), - ("streams", native_field(&sink, "streams")?), - ], - )?); - } - Ok((PyTuple::new(py, exporters)?, PyTuple::new(py, sinks)?)) -} - -impl EventExporterCommandScope { - fn locked(&self) -> PyResult> { - self.state - .lock() - .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned")) - } - - /// Bind discovered sinks over one command-scoped provider session, once. - fn bind_extensions<'py>( - &self, - py: Python<'py>, - project_dir: Option>, - (providers, event_exporters, command_output_sinks): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ), - ) -> PyResult<()> { - { - let mut state = self.locked()?; - if state.configured { - return Ok(()); - } - state.configured = true; - } - let project_dir: Option> = project_dir - .map(|project_dir| project_dir.call_method0("resolve")) - .transpose()?; - { - let mut state = self.locked()?; - state.project_dir = project_dir.map(Bound::unbind); - state.providers = Some(providers.clone().unbind()); - state.event_exporters = Some(event_exporters.clone().unbind()); - state.command_output_sinks = Some(command_output_sinks.clone().unbind()); - } - if !event_exporters.is_truthy()? && !command_output_sinks.is_truthy()? { - let empty = PyTuple::empty(py).into_any(); - return self.bind_dispatcher(py, (empty.clone(), empty)); - } - let missing: HashMap> = providers_missing_secrets(&providers)?; - let session: Bound<'_, PyAny> = call_with( - &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[ - ("discovered_providers", providers), - ("allow_shared", PyBool::new(py, false).to_owned().into_any()), - ], - )?; - session.call_method0("hold_open")?; - let (exporters, sinks) = match bindings( - &session, - (&event_exporters, &command_output_sinks), - &missing, - ) { - Ok(bound) => bound, - Err(error) => { - session.call_method0("release")?; - call_with( - &native_field(&session, "close")?, - &[("force", PyBool::new(py, true).to_owned().into_any())], - )?; - return Err(error); - } - }; - self.locked()?.provider_session = Some(session.unbind()); - self.bind_dispatcher(py, (exporters.into_any(), sinks.into_any())) - } - - fn bind_dispatcher( - &self, - py: Python<'_>, - (exporters, sinks): (Bound<'_, PyAny>, Bound<'_, PyAny>), - ) -> PyResult<()> { - let dispatcher: &Bound<'_, PyAny> = self.dispatcher.bind(py); - dispatcher.call_method1("bind", (exporters,))?; - dispatcher.call_method1("bind_command_output_sinks", (sinks,))?; - Ok(()) - } -} - -#[pymethods] -impl EventExporterCommandScope { - #[new] - #[pyo3(signature = (*, dispatcher))] - pub(crate) fn new(dispatcher: Py) -> Self { - Self { - dispatcher, - state: Mutex::new(ScopeState { - configured: false, - provider_close_registered: false, - provider_session: None, - project_dir: None, - providers: None, - event_exporters: None, - command_output_sinks: None, - }), - } - } - - #[getter] - pub(crate) fn dispatcher(&self, py: Python<'_>) -> Py { - self.dispatcher.clone_ref(py) - } - - #[getter] - pub(crate) fn provider_session(&self, py: Python<'_>) -> PyResult> { - Ok(optional(py, self.locked()?.provider_session.as_ref())) - } - - #[getter] - pub(crate) fn project_dir(&self, py: Python<'_>) -> PyResult> { - Ok(optional(py, self.locked()?.project_dir.as_ref())) - } - - #[getter] - pub(crate) fn providers(&self, py: Python<'_>) -> PyResult> { - let state = self.locked()?; - Ok(state - .providers - .as_ref() - .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) - } - - #[getter] - pub(crate) fn event_exporters(&self, py: Python<'_>) -> PyResult> { - let state = self.locked()?; - Ok(state - .event_exporters - .as_ref() - .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) - } - - #[getter] - pub(crate) fn command_output_sinks(&self, py: Python<'_>) -> PyResult> { - let state = self.locked()?; - Ok(state - .command_output_sinks - .as_ref() - .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) - } - - /// Bind once after normal project discovery has validated declarations. - fn configure(&self, py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>) -> PyResult<()> { - self.bind_extensions( - py, - None, - ( - native_field(discovered_inputs, "providers")?, - native_field(discovered_inputs, "event_exporters")?, - native_field(discovered_inputs, "command_output_sinks")?, - ), - ) - } - - /// Bind declarations discovered at the outer command startup boundary. - #[pyo3(signature = (**options))] - fn configure_extensions( - &self, - py: Python<'_>, - options: Option>, - ) -> PyResult<()> { - let options: KeywordArguments<'_> = KeywordArguments::read( - py, - "configure_extensions", - options, - ( - &["providers", "event_exporters"], - &["project_dir", "command_output_sinks"], - ), - )?; - let project_dir: Bound<'_, PyAny> = options.value("project_dir")?; - let command_output_sinks: Bound<'_, PyAny> = options.value("command_output_sinks")?; - let command_output_sinks: Bound<'_, PyAny> = if command_output_sinks.is_none() { - PyTuple::empty(py).into_any() - } else { - command_output_sinks - }; - self.bind_extensions( - py, - (!project_dir.is_none()).then_some(project_dir), - ( - options.value("providers")?, - options.value("event_exporters")?, - command_output_sinks, - ), - ) - } - - /// Return startup declarations only for the same project root. - #[pyo3(signature = (*, project_dir))] - fn cached_extensions( - &self, - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, - ) -> PyResult> { - let (own, providers, exporters, sinks) = { - let state = self.locked()?; - let Some(own) = state.project_dir.as_ref() else { - return Ok(py.None()); - }; - ( - own.clone_ref(py), - optional(py, state.providers.as_ref()), - optional(py, state.event_exporters.as_ref()), - optional(py, state.command_output_sinks.as_ref()), - ) - }; - if project_dir.call_method0("resolve")?.ne(own.bind(py))? { - return Ok(py.None()); - } - Ok(PyTuple::new(py, [providers, exporters, sinks])? - .into_any() - .unbind()) - } - - /// Flush sink delivery before reverse-order provider teardown. - fn close(&self, py: Python<'_>) -> PyResult> { - let summary: Py = self.dispatcher.bind(py).call_method0("shutdown")?.unbind(); - let session: Py = { - let mut state = self.locked()?; - let Some(session) = state - .provider_session - .as_ref() - .map(|session| session.clone_ref(py)) - else { - return Ok(summary); - }; - if state.provider_close_registered { - return Ok(summary); - } - state.provider_close_registered = true; - session - }; - let closer: Py = Py::new(py, ProviderSessionCloser { session })?; - self.dispatcher - .bind(py) - .call_method1("finalize_when_idle", (closer,))?; - Ok(summary) - } - - /// Reuse bound sink functions and provider instances for output records. - fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { - self.dispatcher - .bind(py) - .call_method1("export_output", (records,))?; - Ok(()) - } -} - -#[pymethods] -impl ProviderSessionCloser { - fn __call__(&self, py: Python<'_>) -> PyResult<()> { - let session: &Bound<'_, PyAny> = self.session.bind(py); - let released = session.call_method0("release"); - call_with( - &native_field(session, "close")?, - &[("force", PyBool::new(py, true).to_owned().into_any())], - )?; - released?; - Ok(()) - } -} - -#[pymethods] -impl EventExporterScopeContext { - fn __enter__(&self, py: Python<'_>) -> PyResult> { - let token = context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? - .call_method1("set", (self.scope.bind(py),))?; - *self - .token - .lock() - .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? = - Some(token.unbind()); - Ok(self.scope.clone_ref(py)) - } - - #[pyo3(signature = (*_exc_info))] - fn __exit__(&self, py: Python<'_>, _exc_info: &Bound<'_, PyTuple>) -> PyResult { - let token = self - .token - .lock() - .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? - .take(); - if let Some(token) = token { - context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? - .call_method1("reset", (token,))?; - } - Ok(false) - } -} - -/// Install one command-owned exporter scope. -#[pyfunction] -pub(crate) fn event_exporter_command_scope(scope: Py) -> EventExporterScopeContext { - EventExporterScopeContext { - scope, - token: Mutex::new(None), - } -} - -/// The active command-owned exporter scope, or `None`. -pub(crate) fn current_scope(py: Python<'_>) -> PyResult> { - context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)?.call_method0("get") -} - -/// Return the active command-owned exporter scope, if any. -#[pyfunction] -fn current_event_exporter_command_scope(py: Python<'_>) -> PyResult> { - current_scope(py) -} - -/// Configure the active exporter scope from validated discovery inputs. -pub(crate) fn configure_discovered_event_exporters(discovered: &Bound<'_, PyAny>) -> PyResult<()> { - let scope: Bound<'_, PyAny> = current_scope(discovered.py())?; - if !scope.is_none() { - scope.call_method1("configure", (discovered,))?; - } - Ok(()) -} - -/// Return startup-discovered extensions for the active project root. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -fn cached_event_exporter_extensions<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let scope: Bound<'py, PyAny> = current_scope(py)?; - if scope.is_none() { - return Ok(scope); - } - call_with( - &native_field(&scope, "cached_extensions")?, - &[("project_dir", project_dir.clone())], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(event_exporter_command_scope, module)?)?; - module.add_function(wrap_pyfunction!( - current_event_exporter_command_scope, - module - )?)?; - module.add_function(wrap_pyfunction!(cached_event_exporter_extensions, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs index 2a58946ad3..5b515d2e05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -13,17 +13,18 @@ use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_even use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::constants::{ - CHARGE_LIMIT_SECONDS, INVOCATION_THREAD_PREFIX, LANE_WINDOW, NO_SECONDS, POLL_SECONDS, -}; use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; -use crate::bindings::_helpers::event_exporting::models::{ - EventExportSummary, EventExporterFailure, -}; -use crate::bindings::_helpers::event_exporting::values::error_type_of; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::event_exporting::constants::CHARGE_LIMIT_SECONDS; +use sqlbuild_python_base::event_exporting::constants::INVOCATION_THREAD_PREFIX; +use sqlbuild_python_base::event_exporting::constants::LANE_WINDOW; +use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; +use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; +use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// Which long-lived dispatcher thread a worker runs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs index 8e461eb31b..cd72890a7f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs @@ -5,6 +5,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyInt, PyList, PyString, PyStringMethods, PyTuple}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_state::execution_history::constants::ENVELOPE_TEXT_FIELDS; use sqlbuild_state::execution_history::errors::{HistoryError, HistoryErrorKind}; use sqlbuild_state::execution_history::models::{ @@ -39,10 +40,7 @@ fn exception_class(kind: HistoryErrorKind) -> ExecutorHost { /// The execution history exception for a rejection of `kind`. pub(crate) fn rejection(py: Python<'_>, kind: HistoryErrorKind, message: &str) -> PyErr { - match host_entry(py, exception_class(kind)).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, exception_class(kind), message) } /// An `ExecutionHistoryStorageError`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs index fd2ff3e65d..128ca5f191 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs @@ -50,13 +50,13 @@ use crate::bindings::_helpers::execution_history::conversion::{ use crate::bindings::_helpers::execution_history::statements::{ column, execute, execute_plain, execute_query, fetch_one, tuple_of, }; -use crate::bindings::_helpers::native_host::sqlite_connection::{ - SqliteConnection, SqliteCursor, sqlite_params, -}; -use crate::bindings::_helpers::native_threads::reentrant_lock::ReentrantLock; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::models::ReentrantLock; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::warehouse_access::main::sqlite_params::sqlite_params; +use sqlbuild_python_base::warehouse_access::models::SqliteConnection; +use sqlbuild_python_base::warehouse_access::models::SqliteCursor; const SQLITE_MODULE: &str = "sqlite3"; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs index a595566ade..56220bb4fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs @@ -5,9 +5,10 @@ use pyo3::types::PyTuple; use sqlbuild_state::execution_history::models::{Query, SqlParam}; use crate::bindings::_helpers::execution_history::conversion::python_params; -use crate::bindings::_helpers::native_host::sqlite_connection::{ - SqliteConnection, SqliteCursor, SqliteRow, sqlite_params, -}; +use sqlbuild_python_base::warehouse_access::main::sqlite_params::sqlite_params; +use sqlbuild_python_base::warehouse_access::models::SqliteConnection; +use sqlbuild_python_base::warehouse_access::models::SqliteCursor; +use sqlbuild_python_base::warehouse_access::models::SqliteRow; /// `target.execute(sql, params)`. pub(crate) fn execute<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index 2792621e29..475ff3013d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -1,5 +1,6 @@ //! Latest fingerprint rows decoded into `Fingerprint` models and indexed into a `FingerprintSet`. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::ffi::CString; use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; @@ -13,7 +14,6 @@ use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -28,10 +28,7 @@ const TIMESTAMP_COLUMN: usize = 11; /// `FingerprintInputError(message)`. pub(crate) fn fingerprint_input_error(py: Python<'_>, message: &str) -> PyErr { - match error_class(py, ErrorId::FingerprintInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::FingerprintInput, message) } /// The `FingerprintSet` of fetched latest rows, a model winning a node-name collision. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 136d73efe6..6bf94189fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -23,11 +23,11 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::main::ensure_event_id_column::ensure_event_id_column; const TIME_MODULE: &str = "time"; const LOGGING_MODULE: &str = "logging"; const LOGGER_NAME: &str = "sqlbuild.fingerprints"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs index a4c6ae3c1f..b12b70c7a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs @@ -9,13 +9,13 @@ use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; -use crate::bindings::_helpers::cache::native_store::{ - current_code_identity, open_store, save_store, -}; use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; use crate::bindings::_helpers::frontier::constants::{ CACHE_DIRECTORY_NAME, COMPILER_CACHE_DIRECTORY_NAME, NATIVE_CACHE_NAMESPACE_SUFFIX, }; +use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; +use sqlbuild_python_base::cache::main::open_store::open_store; +use sqlbuild_python_base::cache::main::save_store::save_store; /// The native store kind holding clean format results. const FORMAT_RESULT_STORE_KIND: &str = "format-results"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index 1453ca2cbf..977b0f7f8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -16,7 +16,6 @@ use sqlbuild_core::format_command::main::render_lint_text::render_lint_text; use sqlbuild_core::format_command::models::LintReport; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::format_command::config::{FormatConfig, format_config, lint_config}; use crate::bindings::_helpers::format_command::files::FormatChange; @@ -24,13 +23,14 @@ use crate::bindings::_helpers::format_command::report::lint_report; use crate::bindings::_helpers::format_command::run::{FormatOutcome, FormatRequest, run_format}; use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_lint_inputs}; use crate::bindings::_helpers::format_command::stages::quote_fixable_file_count; -use crate::bindings::_helpers::frontier::encoder::emit_error; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 220260d31c..86eccff280 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -18,9 +18,6 @@ use sqlbuild_rules::lint_text::models::{ FormatFault, FormatFile, FormattedBodies, SuppressionFinding, }; -use crate::bindings::_helpers::compile_driver::stage_work::{ - FORMAT_CACHE_HITS, FORMAT_CACHE_MISSES, FORMAT_LINT_BATCHES, FORMAT_LINT_FILES, count_stage, -}; use crate::bindings::_helpers::discovery::description_inputs::externally_described_models; use crate::bindings::_helpers::format_command::cache::FormatCache; use crate::bindings::_helpers::format_command::files::{ @@ -38,6 +35,11 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ use crate::bindings::_helpers::rules::lint_text::{relative_parts, unsafe_fault}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::compile_driver::constants::FORMAT_CACHE_HITS; +use sqlbuild_python_base::compile_driver::constants::FORMAT_CACHE_MISSES; +use sqlbuild_python_base::compile_driver::constants::FORMAT_LINT_BATCHES; +use sqlbuild_python_base::compile_driver::constants::FORMAT_LINT_FILES; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index edc26a8d87..fa4dbbd098 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -13,7 +13,6 @@ use sqlbuild_core::format_command::main::partition_lint_selectors::partition_lin use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -31,6 +30,7 @@ use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_s use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index bee525dbd0..6a6e07528e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -14,7 +14,6 @@ use sqlbuild_core::freshness_text::models::{FreshnessSourceResult, FreshnessStat use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_orchestration::runs::models::NodeOutcome; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; @@ -23,13 +22,14 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_selection::selectors::{ expand_project_upstream_keys, select_project_graph, }; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 6b9393472d..eb97aa8dd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::freshness::observation::{ }; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::text; -use crate::bindings::_helpers::native_host::synced_copy::WarehouseAccessRefusedError; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -31,6 +30,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::main::is_warehouse_access_refused::is_warehouse_access_refused; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; @@ -39,7 +39,7 @@ pub(crate) const PREVIEW_MESSAGE: &str = "a plan preview does not observe source /// The message of an adapter, freshness observation or preview access error, if it is one. fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { - if error.is_instance_of::(py) { + if is_warehouse_access_refused(py, error) { return Ok(Some(text(py, PREVIEW_MESSAGE))); } let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 9593af16ac..0879f31789 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; use crate::bindings::_helpers::native_host::consumption_state::state_table_name; -use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -25,6 +24,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::main::ensure_event_id_column::ensure_event_id_column; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs index 45ef619c78..031f90557f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -1,7 +1,5 @@ pub(crate) mod cache_directory; pub(crate) mod compile_frontier; pub(crate) mod constants; -pub(crate) mod encoder; -pub(crate) mod python_types; pub(crate) mod registration; pub(crate) mod stage_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index 582bcbd0d3..4ac289aaa3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -8,11 +8,12 @@ use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; use sqlbuild_cache::stage_capture::main::mask_capture_noise::mask_capture_noise as masked; use sqlbuild_cache::stage_capture::main::next_capture_number::next_capture_number; -use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::frontier::models::CaptureEncoder; use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index aaf628d0c5..89d973a5a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -14,10 +14,10 @@ use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; use crate::bindings::_helpers::init_command::scaffold::scaffold_blank_project; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::workspace_commands::commands::update_sqlbuild_skills; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index 715e3d1173..5be7d8d2ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -22,7 +22,6 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compile_context::delta::record_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ AuditBlockRow, AuditFileRow, AuditSettings, }; @@ -45,6 +44,7 @@ use crate::bindings::_helpers::model_config::header_metadata::thresholds; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use sqlbuild_project::models::AttachedTarget; use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; const THRESHOLDS_KEY: &str = "thresholds"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 1798cc6884..1054b80b3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -30,7 +30,6 @@ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use crate::bindings::_helpers::analysis_session::query_columns::query_columns_error; use crate::bindings::_helpers::attachments::function_headers::parsed_header; -use crate::bindings::_helpers::compile_context::delta::record_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ FunctionRow, FunctionSettings, }; @@ -50,6 +49,7 @@ use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::PythonRuntime; use sqlbuild_project::models::ReferenceRow; use sqlbuild_project::types::NamedTypeRow; +use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 0c7bbcd33c..603ec3c3c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -31,7 +31,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::cache::native_parse_store::NativeParseStore; use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditSettings; @@ -111,6 +110,7 @@ use crate::bindings::_helpers::sqltext::static_sql::{ }; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index 1dd1e4538e..a14e4b5d5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -17,9 +17,6 @@ use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_model_config::model_validation::models::ColumnMigration; use sqlbuild_model_config::types::NodeKind; -use crate::bindings::_helpers::compile_driver::stage_work::{ - NATIVE_HEADER_COLUMNS, PARSED_HEADER_COLUMNS, count_stage, -}; use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{ @@ -42,6 +39,9 @@ use crate::bindings::_helpers::model_config::header_metadata::{ }; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, typed_values}; use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; +use sqlbuild_python_base::compile_driver::constants::NATIVE_HEADER_COLUMNS; +use sqlbuild_python_base::compile_driver::constants::PARSED_HEADER_COLUMNS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index cfa9abac64..7e6418483a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -22,9 +22,10 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::bindings::_helpers::cache::native_parse_store::{NativeParseStore, parse_key}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::input_loops::model_loop_prepass::spawned; +use sqlbuild_python_base::cache::main::parse_key::parse_key; +use sqlbuild_python_base::cache::models::NativeParseStore; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index e026b79ca1..0cfad4fc9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -22,9 +22,6 @@ use sqlbuild_pysource::definition_locations::main::decorator_location::decorator use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; -use crate::bindings::_helpers::compile_context::delta::{ - DeltaInjection, capture, capturing, injection, -}; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ DiagnosticSettings, HookDescriptionRow, LoaderRow, PythonResourceRow, ResourceDiagnostics, ScenarioDescriptionRow, @@ -33,6 +30,10 @@ use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MISSING_DESCRIPTION_CODE: &str = "P010"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs index 1d0eab751b..3745d12287 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs @@ -14,7 +14,6 @@ use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenari use sqlbuild_attachments::test_targets::models::ScenarioCteSources; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compile_context::delta::record_test_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; use crate::bindings::_helpers::input_loops::model_loop::compile_error; use crate::bindings::_helpers::input_loops::model_loop_calls::origin_references; @@ -24,6 +23,7 @@ use crate::bindings::_helpers::input_loops::source_loop::{ use crate::bindings::_helpers::input_loops::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; +use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 6c406bd892..7655836248 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -35,8 +35,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; -use crate::bindings::_helpers::compile_context::delta::record_test_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ SqlTestSettings, TestBlockRow, TestCaseRow, TestFileRow, }; @@ -51,6 +49,8 @@ use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, scoped_declarations, }; use crate::bindings::_helpers::render::sql_macro_tests::without_with_list_calls; +use sqlbuild_python_base::cache::models::SqlTestScanStore; +use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MODEL_MODE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index d4f89a4f55..0b9d9d3994 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -40,8 +40,8 @@ use crate::bindings::_helpers::janitor::janitor_phases::{ use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::scenario_command::setup::stream_isatty; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index bcc60b6984..a37b585ab6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -3,8 +3,8 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::native_host::connections::connection_failure; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use std::time::Instant; use pyo3::IntoPyObject; @@ -14,9 +14,6 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; -use crate::bindings::_helpers::build_progress::transient_status::{ - TransientStatusReporter, status_reporter, -}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; @@ -26,6 +23,8 @@ use crate::bindings::_helpers::janitor::janitor_planning::{ use crate::bindings::_helpers::janitor::janitor_usage::{UsageSettings, check_janitor_usage}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::build_progress::models::TransientStatusReporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs index 309eca58dd..7ed709a93b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs @@ -15,10 +15,9 @@ use sqlbuild_core::janitor_text::main::janitor_phase_start_text::janitor_phase_s use sqlbuild_core::janitor_text::main::render_janitor_item_line::render_janitor_item_line; use sqlbuild_core::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorPlanText}; -use crate::bindings::_helpers::build_progress::terminal::tty_flag; -use crate::bindings::_helpers::build_progress::transient_status::{ - TransientStatusReporter, progress_spinners_disabled, -}; +use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; +use sqlbuild_python_base::build_progress::main::transient_status::progress_spinners_disabled::progress_spinners_disabled; +use sqlbuild_python_base::build_progress::models::TransientStatusReporter; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The running phase's start text, size, finished count, column width and start time. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs index 2eaf116cc0..075ffb283c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -10,12 +10,12 @@ use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; -use crate::bindings::_helpers::build_progress::transient_status::TransientStatusReporter; use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_keys::{Folded, Key, record, tuple}; use crate::bindings::_helpers::janitor::janitor_planning::completed; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::build_progress::models::TransientStatusReporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs index d483657774..8cba387ca9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs @@ -4,19 +4,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; use sqlbuild_core::lifecycle_events::models::PayloadValue; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// `ObservabilityValidationError(message)`. pub(crate) fn validation_error(py: Python<'_>, message: &str) -> PyErr { - match host_entry(py, ExecutorHost::ObservabilityValidationError) - .and_then(|class| class.call1((message,))) - { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, ExecutorHost::ObservabilityValidationError, message) } /// Whether `value` is a `collections.abc.Mapping`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 26b9f1e86c..b23f41d303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -31,8 +31,6 @@ use sqlbuild_analysis::lineage_command::models::{ }; use sqlbuild_core::cli_text::models::CliStyle; -use crate::bindings::_helpers::build_progress::transient_status::maybe_status; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -48,7 +46,9 @@ use crate::bindings::_helpers::planner_selection::selectors::{ match_project_selector, parse_project_selector, }; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use sqlbuild_python_base::build_progress::main::transient_status::maybe_status::maybe_status; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 82e2f6e089..605c5b84a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -1,5 +1,6 @@ //! Project column lineage in FAST or RICH mode, and direct semantic uses, from a project view. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::HashMap; use pyo3::prelude::{ @@ -32,7 +33,6 @@ use crate::bindings::_helpers::lineage::project_schema::{ use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; @@ -335,12 +335,11 @@ fn output_columns(classes: &LineageClasses<'_>, facts: &Bound<'_, PyAny>) -> PyR } fn unknown_dialect_error(py: Python<'_>, dialect: &str) -> PyErr { - match error_class(py, ErrorId::UnknownLineageDialect) - .and_then(|class| class.call1((format!("Unknown dialect: {dialect}"),))) - { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error( + py, + ErrorId::UnknownLineageDialect, + &format!("Unknown dialect: {dialect}"), + ) } fn rich_lineage<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs index 487b727e29..37571cebf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -12,11 +12,11 @@ use sqlbuild_core::dlt_destination::models::{ }; use sqlbuild_core::loader_connection::models::LoaderConnection; use sqlbuild_host::entries::models::ThirdPartyHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::positional_record; use crate::bindings::_helpers::load::ingestr_command::connection; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; const DLT_ERROR: ThirdPartyHost = ThirdPartyHost::DltIntegrationError; @@ -34,9 +34,7 @@ positional_record! { /// A `DltIntegrationError`, or the failure to construct one. fn dlt_error(py: Python<'_>, message: String) -> PyErr { - host_entry(py, DLT_ERROR) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value) + host_error(py, DLT_ERROR, &message) } /// The text of a string value, or `None` for any other value. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs index 3ef564f133..acad1cfa58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs @@ -1,5 +1,6 @@ //! The ingestr command line for one declarative ingestr source, built from its runtime values. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::collections::BTreeMap; use pyo3::prelude::{ @@ -38,9 +39,7 @@ const LIST_FIELDS: [&str; 2] = ["primary_key", "extra_args"]; /// An `IngestrIntegrationError`, or the failure to construct one. fn ingestr_error(py: Python<'_>, message: String) -> PyErr { - host_entry(py, INGESTR_ERROR) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value) + host_error(py, INGESTR_ERROR, &message) } /// The source's ingestr config, raising when the source declares no valid one. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 22509087d9..69dc9d3b6a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -1,5 +1,6 @@ //! `sqb load`: run the selected source loaders and report each load. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::time::Instant; use pyo3::prelude::{ @@ -19,7 +20,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; @@ -36,18 +36,18 @@ use crate::bindings::_helpers::load::load_progress::{ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::result_output::documents::format_load_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -182,10 +182,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult, message: String) -> PyErr { - match error_class(py, ErrorId::CliUser).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::CliUser, &message) } fn target_name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs index bcd3bf3a47..935ca17110 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -12,9 +12,9 @@ use sqlbuild_core::load_command::main::render_load_spinner_line::render_load_spi use sqlbuild_core::load_command::models::{LoadResultText, LoadSpinnerRow, LoadStatus}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; -use crate::bindings::_helpers::build_progress::terminal::start_spinner; -use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; +use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; +use sqlbuild_python_base::build_progress::main::transient_lines::shared_transient_line_coordinator::shared_transient_line_coordinator; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 58b7abddbf..25abafa3f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -28,12 +28,12 @@ use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::planner_scope::values::project_graph; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; /// The Python node kind whose functions load sources. @@ -94,7 +94,7 @@ pub(crate) fn orchestration_snapshot<'py>( } } input.sqb_version = env!("CARGO_PKG_VERSION").into(); - input.environment_hash = environment_hash(py, &project_dir)?; + input.environment_hash = environment_hash(py, &project_dir, env!("CARGO_PKG_VERSION"))?; let owner = project.owner().bind(py).borrow(); let (nodes, columns) = model_nodes(py, &project, &owner, input.nodes)?; input.nodes = nodes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index cea5153f6d..4090cb4ff6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -10,9 +10,9 @@ use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_pat use crate::bindings::_helpers::location_command::constants::{ MODEL_NODE_KIND, SNAPSHOT_HIDDEN_SETTING, SOURCE_NODE_KIND, }; -use crate::bindings::_helpers::native_host::provider_settings::{ - missing_secrets, provider_class_settings, setting_rows, -}; +use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_secrets; +use sqlbuild_python_base::orchestration_reads::main::provider_class_settings::provider_class_settings; +use sqlbuild_python_base::orchestration_reads::main::setting_rows::setting_rows; use sqlbuild_python_base::records::main::fields::field::field; /// How a consumer's code runs inside other snapshot nodes, besides its own node. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs index d47ea31186..11e0d1c083 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs @@ -5,7 +5,7 @@ use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_context::delta::injection; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; /// Whether a compile claimed the first-compile slot, and whether a delta compile only borrowed it. #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs index c7b0461e0e..dd34d77fae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs @@ -10,10 +10,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::module_digests::main::module_digests_metadata::module_digests_metadata as metadata_text; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::cache::native_store::digest_files; -use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::macro_bridge::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::cache::main::digest_files::digest_files; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// The fields of one stored row: a path and its digest. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs index fb3ca6db19..21573a9fe5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs @@ -9,15 +9,16 @@ use pyo3::types::{ PyTuple, PyTypeMethods, }; use pyo3::{create_exception, pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::boundary::main::utf16_units::utf16_units; use sqlbuild_cache::macro_store_keys::main::canonical_sort_text::canonical_sort_text; use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; -use crate::bindings::_helpers::frontier::encoder::emit_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -49,20 +50,10 @@ fn type_name(kind: &Bound<'_, PyAny>) -> PyResult { /// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. pub(crate) fn text(value: &Bound<'_, PyString>) -> PyResult { - if let Ok(text) = value.to_str() { - return Ok(JsonValue::String(text.to_owned())); + match value.to_str() { + Ok(text) => Ok(JsonValue::String(text.to_owned())), + Err(_) => utf16_units(value).map(JsonValue::Utf16), } - let encoded: Bound<'_, PyBytes> = value - .call_method1("encode", ("utf-16-le", "surrogatepass"))? - .downcast_into::()?; - Ok(JsonValue::Utf16( - pyo3::types::PyBytesMethods::as_bytes(&encoded) - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) - .collect(), - )) } /// `[tag, value]`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs index 87a7d4a07d..2ed36afb06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs @@ -12,7 +12,7 @@ use sqlbuild_analysis::manifest::models::{ }; use sqlbuild_analysis::python_values::models::PythonValue; -use crate::bindings::_helpers::python_values::values::python_value; +use sqlbuild_python_base::python_values::main::python_value::python_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs index ef0e191e3d..ae49a505f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs @@ -1,5 +1,6 @@ //! Microbatch events, scopes and intervals read from their Python shapes. +use sqlbuild_python_base::native_host::main::host_error::host_error; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -11,17 +12,11 @@ use sqlbuild_state::microbatches::models::{BoundInterval, CursorBound}; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::as_scalar; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `MicrobatchStateError(message)`. pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { - match host_entry(py, ExecutorHost::MicrobatchStateError) - .and_then(|class| class.call1((message,))) - { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, ExecutorHost::MicrobatchStateError, message) } /// `str(value)` of a string enum member or string. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs index 326d82a58b..dee8a8be60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs @@ -14,11 +14,11 @@ use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; -use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::microbatches::conversion::{optional_text, state_error}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 7f063f6e72..9ad0e502dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -14,7 +14,6 @@ pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; -pub(crate) mod compile_context; pub(crate) mod compile_delta; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; @@ -58,7 +57,6 @@ pub(crate) mod manifest; pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_host; -pub(crate) mod native_threads; pub(crate) mod observations; pub(crate) mod output_capture; pub(crate) mod plan_command; @@ -77,7 +75,6 @@ pub(crate) mod planner_strategy; pub(crate) mod planner_warehouse; pub(crate) mod presentation_text; pub(crate) mod project_assembly; -pub(crate) mod python_values; pub(crate) mod query_command; pub(crate) mod recall; pub(crate) mod record_behaviours; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs index 11c7d60e58..ac61d18106 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs @@ -27,10 +27,10 @@ use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ ContractClasses, HeaderValue, audit_instance, thresholds, }; -use crate::bindings::_helpers::native_host::header_projection::AuditProjection; -use crate::bindings::_helpers::result_output::integration_values::json_value; use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; +use sqlbuild_python_base::native_host::models::AuditProjection; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::result_output::main::json_value::json_value; /// Native-built specs never become Python audit records just to be read by attachment. pub(crate) enum AuditInstance<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs index 2b0f272413..e3d1f1019f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use sqlbuild_model_config::header_metadata::models::DynamicFamily; -use crate::bindings::_helpers::native_host::header_projection::FamilyProjection; +use sqlbuild_python_base::native_host::models::FamilyProjection; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 48c1aed053..6690095c67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -73,7 +73,9 @@ fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { let mut models: HashSet = each(inputs, "model_files", model_stem)? .into_iter() .collect(); - if let Some(injection) = crate::bindings::_helpers::compile_context::delta::injection() { + if let Some(injection) = + sqlbuild_python_base::compile_context::main::delta::injection::injection() + { models.extend(injection.model_names.iter().cloned()); } let mut seeds: HashSet = HashSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index a5d88bc90d..48273a287f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -10,7 +10,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::compile_context::input_reads::{context_read, environment_read}; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_templates::{ TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, @@ -20,6 +19,8 @@ use crate::bindings::_helpers::model_config::unicode_environment::unicode_enviro use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::compile_context::main::input_reads::context_read::context_read; +use sqlbuild_python_base::compile_context::main::input_reads::environment_read::environment_read; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs index e8cc23abd6..190cb7abbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs @@ -15,11 +15,12 @@ use sqlbuild_model_config::config_values::main::checked_text::checked_text; use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::{ConfigNode, CursorBound, HostChildren}; -use crate::bindings::_helpers::compile_driver::stage_work::{TYPED_CONFIG_READS, count_stage}; use crate::bindings::_helpers::model_config::config_nodes::{ HostHandle, PyConfigEntries, PyConfigNode, config_node, config_object, is_exact_container, }; use crate::bindings::_helpers::model_config::config_values::config_value_type_error; +use sqlbuild_python_base::compile_driver::constants::TYPED_CONFIG_READS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::cached_value::cached_value; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs deleted file mode 100644 index 9de61aa2ec..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Authoring parameters reserved by the code-location host. - -/// The calendar's evaluation window is supplied by the host, not a provider. -pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; - -pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; -pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; - -/// The sensor context is supplied by the host, not a provider. -pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; - -pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; -/// Frames of these modules construct declarations on the author's behalf. -pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; - -/// The authoring classes publication imports calendar and sensor modules for. -pub(crate) const CALENDAR_CLASS: &str = "Calendar"; -pub(crate) const SENSOR_CLASS: &str = "Sensor"; - -/// Pages one bounded state-history SELECT streams before the scan continues from its cursor. -pub(crate) const HISTORY_STREAM_PAGES: usize = 50; - -/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. -pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; -pub(crate) const ALL_PREFIX_TARGET: &str = "all"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 2049bc3727..363921da4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -21,11 +21,11 @@ use sqlbuild_state::consumption::models::{ use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; /// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. pub(crate) const ROWS_PER_STATEMENT: usize = 500; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs deleted file mode 100644 index b40db7b080..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! Public header records and user-supplied constructors at the Python boundary. - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; - -/// Factory results, source locations and the cached public audit tuple are Python-owned edges. -pub(crate) struct AuditProjection { - pub(crate) custom: Option>>, - pub(crate) generated: Vec>, - pub(crate) location: Py, - pub(crate) records: PyOnceLock>, -} - -/// A supplied constructor and its public family records; native callers retain the family specs. -pub(crate) struct FamilyProjection { - class: Py, - pub(crate) custom: bool, - records: PyOnceLock>, -} - -impl FamilyProjection { - pub(crate) fn new(class: &Bound<'_, PyAny>, custom: bool) -> Self { - Self { - class: class.clone().unbind(), - custom, - records: PyOnceLock::new(), - } - } - - pub(crate) fn project<'py>( - &self, - py: Python<'py>, - specs: &[DynamicFamily], - ) -> PyResult> { - self.records - .get_or_try_init(py, || { - let mut records: Vec> = Vec::with_capacity(specs.len()); - for family in specs { - let fields = PyDict::new(py); - fields.set_item("name", &family.name)?; - fields.set_item("pivot_column", &family.pivot_column)?; - fields.set_item("value_column", &family.value_column)?; - fields.set_item("aggregate", &family.aggregate)?; - fields.set_item("type", &family.data_type)?; - fields.set_item("name_pattern", &family.name_pattern)?; - records.push(self.class.bind(py).call((), Some(&fields))?); - } - PyTuple::new(py, records).map(Bound::unbind) - }) - .map(|value| value.bind(py).clone()) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs deleted file mode 100644 index d10774e0fc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Python's default compilation Ctrl+C handler, retained across unraisable callbacks. - -use std::cell::Cell; - -use pyo3::exceptions::PyKeyboardInterrupt; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyTuple}; - -thread_local! { - static INTERRUPTED: Cell = const { Cell::new(false) }; -} -const SIGINT: i64 = 2; - -/// The previous Python handler and the command's replacement, restored on every exit path. -pub(crate) struct InterruptHandler { - previous: Py, - installed: Py, -} - -impl InterruptHandler { - /// Capture only Python's default main-thread handler; leave embedding applications' handlers alone. - pub(crate) fn install(py: Python<'_>) -> PyResult> { - let threading = py.import("threading")?; - if !threading - .call_method0("current_thread")? - .is(threading.call_method0("main_thread")?) - { - return Ok(None); - } - let signal = py.import("signal")?; - let previous = signal.call_method1("getsignal", (SIGINT,))?; - if !previous.is(signal.getattr("default_int_handler")?) { - return Ok(None); - } - let installed = PyCFunction::new_closure( - py, - None, - None, - |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { - INTERRUPTED.set(true); - Err(PyKeyboardInterrupt::new_err(())) - }, - )?; - INTERRUPTED.set(false); - signal.call_method1("signal", (SIGINT, &installed))?; - Ok(Some(Self { - previous: previous.unbind(), - installed: installed.into_any().unbind(), - })) - } -} - -impl Drop for InterruptHandler { - fn drop(&mut self) { - Python::attach(|py| { - let restored: PyResult<()> = (|| { - let signal = py.import("signal")?; - if signal - .call_method1("getsignal", (SIGINT,))? - .is(self.installed.bind(py)) - { - signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; - } - Ok(()) - })(); - drop(restored); - }); - INTERRUPTED.set(false); - } -} - -/// Whether the default handler observed Ctrl+C during the current command. -pub(crate) fn interrupted() -> bool { - INTERRUPTED.get() -} - -/// Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. -pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { - py.check_signals()?; - if interrupted() { - return Err(PyKeyboardInterrupt::new_err(())); - } - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 122a8152ec..794ea5865f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,24 +1,11 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod audit_results_state; -pub(crate) mod connections; -pub(crate) mod constants; pub(crate) mod consumption_state; -pub(crate) mod header_projection; -pub(crate) mod interrupt; pub(crate) mod model_lookup; pub(crate) mod observations_state; -pub(crate) mod orchestration_declarations; -pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; -pub(crate) mod provider_settings; pub(crate) mod registration; pub(crate) mod resource_costs_state; -pub(crate) mod results; -pub(crate) mod sqlite_connection; -pub(crate) mod state_column_evolution; -pub(crate) mod synced_copy; -pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_metadata; -pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 8d50af715b..c29ee9cf82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -16,14 +16,13 @@ use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, p use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; -use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; -use crate::bindings::_helpers::native_host::orchestration_environment::{ - callback_code_hash, orchestration_code_hash, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::constants::SENSOR_CONTEXT_PARAMETER; use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; +use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; +use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; pub(crate) fn evaluate_sensor( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index afe30f4fa9..ef8daac20d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -21,19 +21,17 @@ use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, p use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, -}; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, declared_at, -}; -use crate::bindings::_helpers::native_host::orchestration_environment::{ - callback_code_hash, orchestration_code_hash, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::constants::CALENDAR_WINDOW_PARAMETER; +use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; +use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; +use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; +use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; +use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; +use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; /// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. pub(crate) struct CalendarRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs index 0439de3ce2..df8641ddac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs @@ -2,20 +2,15 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::types::PyTuple; -use pyo3::wrap_pyfunction; use sqlbuild_host::entries::main::host_entry_labels::host_entry_labels; -use crate::bindings::_helpers::native_host::orchestration_declarations::declaration_site; -use crate::bindings::_helpers::native_host::synced_copy::WarehouseAccessRefusedError; +use sqlbuild_python_base::warehouse_access::main::register_synced_copy::register_synced_copy; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "HOST_ENTRIES", PyTuple::new(module.py(), host_entry_labels())?, )?; - module.add( - "WarehouseAccessRefusedError", - module.py().get_type::(), - )?; - module.add_function(wrap_pyfunction!(declaration_site, module)?) + register_synced_copy(module)?; + sqlbuild_python_base::orchestration_reads::main::register_orchestration_declarations::register_orchestration_declarations(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs deleted file mode 100644 index 023d192e00..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! Rust-owned SQLite connection and its rows; Python views exist only for connection overrides. - -use std::collections::VecDeque; -use std::path::PathBuf; -use std::sync::{Arc, Mutex, PoisonError}; - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; -use pyo3::types::PyBytes; -use rusqlite::types::Value; -use rusqlite::{Connection, Error, ErrorCode, OpenFlags, params_from_iter}; -use sqlbuild_state::execution_history::constants::MEMORY_PATH; -use sqlbuild_state::execution_history::models::SqlParam; - -use sqlbuild_python_base::boundary::main::imported::imported; - -/// Preserve DB-API exception classes at the public storage-error boundary. -fn database_error(py: Python<'_>, error: Error) -> PyErr { - let class: &str = match &error { - Error::SqliteFailure(error, _) if error.code == ErrorCode::ConstraintViolation => { - "IntegrityError" - } - Error::SqliteFailure(error, _) - if matches!( - error.code, - ErrorCode::DatabaseCorrupt | ErrorCode::NotADatabase - ) => - { - "DatabaseError" - } - _ => "OperationalError", - }; - match imported(py, "sqlite3", class).and_then(|class| class.call1((error.to_string(),))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} - -/// Parameters of history SQL contain no Python-owned values. -pub(crate) fn sqlite_params(params: &[SqlParam]) -> PyResult> { - params - .iter() - .map(|value| { - Ok(match value { - SqlParam::Null => Value::Null, - SqlParam::Text(value) => Value::Text(value.clone()), - SqlParam::Integer(value) => Value::Integer(*value), - SqlParam::Flag(value) => Value::Integer(i64::from(*value)), - SqlParam::Timestamp(_) | SqlParam::TextList(_) => { - return Err(PyValueError::new_err("Postgres-only history parameter")); - } - }) - }) - .collect() -} - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteConnection")] -pub(crate) struct SqliteConnection { - connection: Mutex>, -} - -impl SqliteConnection { - /// Open literal filenames, using absolute paths because bundled SQLite enables URI handling globally. - pub(crate) fn open(py: Python<'_>, path: &str) -> PyResult { - let flags = OpenFlags::default() & !OpenFlags::SQLITE_OPEN_URI; - let database: PathBuf = if path == MEMORY_PATH { - PathBuf::from(path) - } else { - std::env::current_dir()?.join(path) - }; - let connection = py - .detach(|| Connection::open_with_flags(database, flags)) - .map_err(|error| database_error(py, error))?; - Ok(Self { - connection: Mutex::new(Some(connection)), - }) - } - - fn with_connection( - &self, - py: Python<'_>, - body: impl FnOnce(&Connection) -> rusqlite::Result + Send, - ) -> PyResult { - let result = py.detach(|| { - let guard = self - .connection - .lock() - .unwrap_or_else(PoisonError::into_inner); - guard.as_ref().map(body) - }); - match result { - Some(result) => result.map_err(|error| database_error(py, error)), - None => Err(PyErr::from_value( - imported(py, "sqlite3", "ProgrammingError")? - .call1(("Cannot operate on a closed database.",))?, - )), - } - } - - pub(crate) fn execute_native( - &self, - py: Python<'_>, - sql: &str, - params: &[Value], - ) -> PyResult { - self.with_connection(py, |connection| { - let mut statement = connection.prepare_cached(sql)?; - let columns = Arc::new( - statement - .column_names() - .into_iter() - .map(str::to_owned) - .collect(), - ); - let count: usize = statement.column_count(); - let mut cursor = statement.query(params_from_iter(params))?; - let mut rows: VecDeque = VecDeque::new(); - while let Some(row) = cursor.next()? { - rows.push_back(SqliteRow { - columns: Arc::clone(&columns), - values: (0..count) - .map(|index| row.get(index)) - .collect::>()?, - }); - } - Ok(SqliteCursor { - rows: Mutex::new(rows), - last_row_id: connection.last_insert_rowid(), - }) - }) - } -} - -#[pymethods] -impl SqliteConnection { - #[pyo3(signature = (sql, parameters = None))] - fn execute( - &self, - py: Python<'_>, - sql: &str, - parameters: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - let mut params: Vec = Vec::new(); - if let Some(parameters) = parameters { - for value in parameters.try_iter()? { - let value = value?; - params.push(if value.is_none() { - Value::Null - } else if let Ok(value) = value.extract::() { - Value::Integer(value) - } else { - Value::Text(value.extract()?) - }); - } - } - self.execute_native(py, sql, ¶ms) - } - - fn executemany( - &self, - py: Python<'_>, - sql: &str, - parameters: &Bound<'_, PyAny>, - ) -> PyResult { - let mut result = SqliteCursor { - rows: Mutex::default(), - last_row_id: 0, - }; - for row in parameters.try_iter()? { - result = self.execute(py, sql, Some(&row?))?; - } - Ok(result) - } - - #[getter] - pub(crate) fn in_transaction(&self, py: Python<'_>) -> PyResult { - self.with_connection(py, |connection| Ok(!connection.is_autocommit())) - } - - pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { - py.detach(|| { - self.connection - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); - }); - Ok(()) - } -} - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] -pub(crate) struct SqliteRow { - pub(crate) columns: Arc>, - pub(crate) values: Vec, -} - -impl SqliteRow { - pub(crate) fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { - match self.values.get(index) { - Some(Value::Null) => Ok(py.None().into_bound(py)), - Some(Value::Integer(value)) => value.into_bound_py_any(py), - Some(Value::Real(value)) => value.into_bound_py_any(py), - Some(Value::Text(value)) => value.into_bound_py_any(py), - Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), - None => Err(PyIndexError::new_err("tuple index out of range")), - } - } - - pub(crate) fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let index: usize = self - .columns - .iter() - .position(|column| column.eq_ignore_ascii_case(name)) - .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; - self.value(py, index) - } -} - -#[pymethods] -impl SqliteRow { - fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(index) = key.extract::() { - let index = if index < 0 { - self.values.len().checked_add_signed(index) - } else { - Some(index as usize) - }; - return self.value(key.py(), index.unwrap_or(usize::MAX)); - } - if let Ok(name) = key.extract::() { - return self.column(key.py(), &name); - } - Err(PyTypeError::new_err("Index must be int or string")) - } -} - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] -pub(crate) struct SqliteCursor { - pub(crate) rows: Mutex>, - pub(crate) last_row_id: i64, -} - -impl SqliteCursor { - pub(crate) fn next_row(&self) -> Option { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .pop_front() - } -} - -#[pymethods] -impl SqliteCursor { - fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { - slf - } - - fn __next__(&self) -> Option { - self.next_row() - } - - fn fetchone(&self) -> Option { - self.next_row() - } - - fn fetchall(&self) -> Vec { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .drain(..) - .collect() - } - - #[getter] - fn lastrowid(&self) -> i64 { - self.last_row_id - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index f53791763d..6ef103c2b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -39,17 +39,17 @@ use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; -use crate::bindings::_helpers::native_host::constants::HISTORY_STREAM_PAGES; -use crate::bindings::_helpers::native_host::warehouse_catalog::read_catalog; -use crate::bindings::_helpers::native_host::warehouse_statistics::read_statistics; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::constants::HISTORY_STREAM_PAGES; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; +use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; /// One read-only adapter session serves every page of one synchronization. pub(crate) struct HistorySession<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs deleted file mode 100644 index 8de268820c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Thread attributes native capture and delivery code share. - -/// Marks a thread whose writes command-output capture ignores, such as one running a command-output sink. -pub(crate) const OUTPUT_SUPPRESSED_ATTRIBUTE: &str = "_sqlbuild_command_output_suppressed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/reentrant_lock.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/reentrant_lock.rs deleted file mode 100644 index 44c5319046..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/reentrant_lock.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! A native reentrant lock whose contended waits release the GIL and honor Python signals. - -use std::sync::{Condvar, Mutex, PoisonError}; -use std::thread::{ThreadId, current}; -use std::time::Duration; - -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{PyResult, Python}; - -const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); -const POISONED: &str = "native reentrant lock is poisoned"; - -#[derive(Default)] -struct LockState { - owner: Option, - depth: usize, -} - -/// Serialize native operations that may reenter through Python callbacks. -#[derive(Default)] -pub(crate) struct ReentrantLock { - state: Mutex, - available: Condvar, -} - -/// Release one acquisition even when the protected callback raises or panics. -struct LockGuard<'a>(&'a ReentrantLock); - -impl ReentrantLock { - /// Run `body` under the lock, checking pending Python signals while another thread owns it. - pub(crate) fn with_lock( - &self, - py: Python<'_>, - body: impl FnOnce() -> PyResult, - ) -> PyResult { - while !py - .detach(|| self.acquire_or_wait()) - .map_err(PyRuntimeError::new_err)? - { - py.check_signals()?; - } - let _guard = LockGuard(self); - body() - } - - fn acquire_or_wait(&self) -> Result { - let owner: ThreadId = current().id(); - let mut state = self.state.lock().map_err(|_| POISONED)?; - if state.owner.is_none() || state.owner == Some(owner) { - state.owner = Some(owner); - state.depth += 1; - return Ok(true); - } - let (_state, _) = self - .available - .wait_timeout(state, SIGNAL_POLL_INTERVAL) - .map_err(|_| POISONED)?; - Ok(false) - } -} - -impl Drop for LockGuard<'_> { - fn drop(&mut self) { - let mut state = self.0.state.lock().unwrap_or_else(PoisonError::into_inner); - state.depth -= 1; - if state.depth == 0 { - state.owner = None; - self.0.available.notify_one(); - } - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs index 6d43ccd662..3531325d57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs @@ -23,14 +23,14 @@ use sqlbuild_core::command_output::models::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; -use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; use crate::bindings::_helpers::output_capture::values::{ freeze, object, output_error, pass_text, python_text, unencodable, }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; +use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::fields::field::field as native_field; const THREAD_NAME: &str = "sqlbuild-output-exporter"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs index 1f460da449..7eafd8aa8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs @@ -11,10 +11,10 @@ use sqlbuild_core::command_output::errors::UnencodableText; use sqlbuild_core::command_output::types::PassText; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use crate::bindings::_helpers::macro_bridge::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; const SURROGATE_PASS: &str = "surrogatepass"; @@ -22,10 +22,7 @@ const UTF8: &str = "utf-8"; /// The exception class `name` of the output capture exceptions module, raised with `message`. pub(crate) fn output_error(py: Python<'_>, name: ExecutorHost, message: &str) -> PyErr { - match host_entry(py, name).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + host_error(py, name, message) } /// `CommandOutputValidationError(message)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 0c4c04e73b..88b9602038 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -11,7 +11,6 @@ use sqlbuild_core::plan_display::constants::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_command::support::keyword_relay; use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; @@ -25,18 +24,19 @@ use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeli use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, preview_changed}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; use crate::bindings::_helpers::plan_output::plan_json::{CHANGED_KEY, format_plan_json}; use crate::bindings::_helpers::plan_output::plan_text::format_plan; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::keyword_relay::keyword_relay; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index f528acb224..389a15a7f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -9,7 +9,6 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; use sqlbuild_rules::rule_catalogue::models::RuleListing; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -24,6 +23,7 @@ use crate::bindings::_helpers::rules::rule_run::run_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 0feb29e408..e1894604bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; -use crate::bindings::_helpers::compile_outputs::json_report::indented_json; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, cursor_details, owner, @@ -19,6 +18,7 @@ use crate::bindings::_helpers::plan_output::values::{ any, dict, enum_value, iso, items, list, mapped, optional_enum_value, relation_name, text, }; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The plan JSON key of what `--changed` would run, hold back and skip. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 9d89bd8ec8..0e6f46fa55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -44,7 +44,6 @@ use crate::bindings::_helpers::planner_identity::metadata::{ }; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns; -use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -54,6 +53,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::type_system::main::types_equal::types_equal as native_types_equal; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs index a849e84b30..1e9fe96a0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -1,15 +1,15 @@ //! The planner's Python change-detection result models. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, PyErr}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_analysis::planner_changes::errors::UnknownReplayPolicy; use sqlbuild_analysis::planner_changes::models::{ Backfill, BackfillAction, ChangeKind, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, }; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::ErrorId; @@ -17,10 +17,7 @@ use sqlbuild_python_base::records::types::RecordId; /// `PlannerInputError(message)` for an unknown replay policy. pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> PyErr { - match error_class(py, ErrorId::PlannerInput).and_then(|class| class.call1((error.message(),))) { - Ok(value) => PyErr::from_value(value), - Err(error) => error, - } + new_plain_error(py, ErrorId::PlannerInput, &error.message()) } /// `TYPES_MODULE.(value)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index a44a84b087..e81fdf2378 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -3,7 +3,6 @@ use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::planner_changes::actions::resolve_model_actions; use crate::bindings::_helpers::planner_changes::detect::detect_changes; use crate::bindings::_helpers::planner_entries::test_planning::{ @@ -42,6 +41,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::utc_now::utc_now; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs index 158c02992f..083a50daba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs @@ -13,11 +13,11 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; use crate::bindings::_helpers::planner_entries::plan_output::plan_selected_sql_tests; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 5e3d4b7500..a90cb8ea1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -1,5 +1,6 @@ //! Planner execution edges, their injected origins, stable execution order and buildability over compiled keys. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ @@ -31,7 +32,6 @@ use crate::bindings::_helpers::references::explicit_references::{ GateRow, HookRow, native_execution_gate_edges, }; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; @@ -258,10 +258,7 @@ pub(crate) fn build_downstream_deps<'py>( } fn planner_input_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::PlannerInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::PlannerInput, &message) } fn edge_origins<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 67786ad20a..60e731100f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -25,10 +25,12 @@ use crate::bindings::_helpers::model_config::typed_config::{config_value, model_ use crate::bindings::_helpers::planner_identity::hashing::{ baselined_function_definition_hash, function_definition_hash, }; -use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, SPACED, sorted_dumps}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::planner_identity::constants::COMPACT; +use sqlbuild_python_base::planner_identity::constants::SPACED; +use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs index 638074cc97..2b7ae821ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod graph; pub(crate) mod hashing; pub(crate) mod identities; -pub(crate) mod json_values; pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index eea07c7e45..b405173d37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, typed_values}; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs index 64352579f9..2439da6be4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; -use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::utc_now::utc_now; #[pymethods] impl PlannerRuntime { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index dd8cb1258d..4df5d150db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -1,6 +1,7 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::cursor::conversion::CursorSession; @@ -39,7 +40,6 @@ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -522,10 +522,7 @@ pub(crate) fn cursor_safety_configs<'py>( } fn compile_input_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::CompileInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::CompileInput, &message) } /// Resolve typed cursor overrides to generic start/end for one model. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs index 0002846aac..f44d0979aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -8,7 +8,7 @@ use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -17,10 +17,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; pub(super) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { - match error_class(py, class).and_then(|class| class.call1((message,))) { - Ok(value) => PyErr::from_value(value), - Err(error) => error, - } + new_plain_error(py, class, message) } pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index beb35ff240..9d439d3bb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::typed_config::{ config_node_of, config_text_is, config_text_object, config_value, model_values, }; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::{ @@ -33,6 +32,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; const APPEND_STRATEGY: &str = "append"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index f1c4e39cbc..8e3a4d36af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::planner_resolution::values::{ }; use crate::bindings::_helpers::planner_strategy::values::input_error; use crate::bindings::_helpers::references::source_relations::render_source_read; -use crate::bindings::_helpers::type_system::equality::types_equal; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -17,6 +16,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_sqltext::resolution::main::column_name_key::column_name_key; use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 4aa941d326..0f49154e68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::cursor::scalars::compare_values; use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::cursor::scalars::to_inclusive; use crate::bindings::_helpers::model_config::typed_config::{config_cursor_bound, model_values}; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, @@ -49,6 +48,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 07cdde6266..f8eef566ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::observations::source_capture::capture_source_columns; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -15,6 +14,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use std::collections::{BTreeSet, HashSet}; fn entries_of<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 83bd4a15a1..61c4672fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::{native, replace, text, tuple}; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; @@ -25,6 +24,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use std::collections::BTreeSet; /// One warning per reason that left configured source freshness unknown. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 6de1471c12..49f022336a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::discovery::locations::yaml_entry_line; -use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -27,6 +26,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs index 352c9c7d9e..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -1,6 +1 @@ -pub(crate) mod document; -pub(crate) mod functions; -pub(crate) mod getters; pub(crate) mod registration; -pub(crate) mod style; -pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs index df45c7b2ab..b720539b4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::presentation_text::{document, functions, style, terminal}; +use sqlbuild_python_base::presentation_text::main::register_document::register_document; +use sqlbuild_python_base::presentation_text::main::register_functions::register_functions; +use sqlbuild_python_base::presentation_text::main::register_style::register_style; +use sqlbuild_python_base::presentation_text::main::register_terminal::register_terminal; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - document::register(module)?; - functions::register(module)?; - style::register(module)?; - terminal::register(module) + register_document(module)?; + register_functions(module)?; + register_style(module)?; + register_terminal(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs deleted file mode 100644 index dddb7bee3a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Semantic styling for human CLI output, as a Python class. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::cli_text::models::CliStyle as NativeStyle; - -/// Semantic CLI styling facade used by human-output formatters. -#[pyclass(name = "CliStyle", module = "sqlbuild._native", frozen)] -pub(crate) struct PythonCliStyle { - #[pyo3(get)] - pub(crate) use_color: bool, -} - -impl PythonCliStyle { - fn native(&self) -> NativeStyle { - NativeStyle { - use_color: self.use_color, - } - } -} - -#[pymethods] -impl PythonCliStyle { - #[new] - #[pyo3(signature = (*, use_color))] - pub(crate) fn new(use_color: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - use_color: use_color.is_truthy()?, - }) - } - - fn title(&self, text: &str) -> String { - self.native().title(text) - } - - fn section(&self, text: &str) -> String { - self.native().section(text) - } - - fn label(&self, text: &str) -> String { - self.native().label(text) - } - - fn value(&self, text: &str) -> String { - self.native().value(text) - } - - fn accent(&self, text: &str) -> String { - self.native().accent(text) - } - - fn accent_strong(&self, text: &str) -> String { - self.native().accent_strong(text) - } - - fn plan_section(&self, text: &str) -> String { - self.native().plan_section(text) - } - - fn object_name(&self, text: &str) -> String { - self.native().object_name(text) - } - - fn command(&self, text: &str) -> String { - self.native().command(text) - } - - fn muted(&self, text: &str) -> String { - self.native().muted(text) - } - - fn success(&self, text: &str) -> String { - self.native().success(text) - } - - fn success_strong(&self, text: &str) -> String { - self.native().success_strong(text) - } - - fn warning(&self, text: &str) -> String { - self.native().warning(text) - } - - fn warning_strong(&self, text: &str) -> String { - self.native().warning_strong(text) - } - - fn error(&self, text: &str) -> String { - self.native().error(text) - } - - fn error_strong(&self, text: &str) -> String { - self.native().error_strong(text) - } - - fn error_muted(&self, text: &str) -> String { - self.native().error_muted(text) - } - - fn log_label(&self, text: &str) -> String { - self.native().log_label(text) - } - - fn dbt_section(&self, text: &str) -> String { - self.native().dbt_section(text) - } - - fn dbt_label(&self, text: &str) -> String { - self.native().dbt_label(text) - } - - fn dbt_object_name(&self, text: &str) -> String { - self.native().dbt_object_name(text) - } - - fn dbt_execution_label(&self, text: &str) -> String { - self.native().dbt_execution_label(text) - } - - /// Style status text according to the status word. - #[pyo3(signature = (*, status, text = None))] - fn status(&self, status: &str, text: Option<&str>) -> String { - self.native().status_text(status, text.unwrap_or(status)) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 5c1aa4bdcd..b137473697 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -2,13 +2,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The project directory, defaulting to the working directory, with its inputs and adapter. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index a930a72f7d..9e65260a9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -14,9 +14,7 @@ use sqlbuild_core::query_command::main::render_query_json::render_query_json; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::build_progress::transient_status::delayed_status; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; @@ -27,6 +25,8 @@ use crate::bindings::_helpers::query_command::text_output::{LONG_FORMAT, render_ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::transient_status::delayed_status::delayed_status; +use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs index 114ac23eb3..1b5d9080b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{PyResult, Python}; use sqlbuild_core::query_command::main::render_query_text::render_query_text; use sqlbuild_core::query_command::models::{QueryDisplay, QueryLayout}; -use crate::bindings::_helpers::build_progress::terminal::terminal_columns; -use crate::bindings::_helpers::presentation_text::terminal::stdout_is_tty; +use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; +use sqlbuild_python_base::presentation_text::main::stdout_is_tty::stdout_is_tty; /// The format that always prints one record per row. pub(crate) const LONG_FORMAT: &str = "long"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs index d2f05ba5fe..936ab7854a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -9,10 +9,10 @@ use sqlbuild_recall::records::models::RecallValue; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::models::QueryResult; -use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; /// What `sqb recall query` reports about a ClickHouse query besides its rows. #[derive(Clone, Debug, Default, PartialEq, Eq)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs index b79b2393a3..657ffa9d37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs @@ -17,9 +17,6 @@ use sqlbuild_recall::writer::models::{ }; use sqlbuild_recall::writer::types::{OutputReplacement, PendingLifecycleEvent}; -use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; -use crate::bindings::_helpers::event_exporting::values::export_policy; -use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::constants::{ EVENT_PARAMETER, RECORD_PARAMETER, UTC_OFFSET, UTC_SUFFIX, @@ -27,7 +24,10 @@ use crate::bindings::_helpers::recall::constants::{ use crate::bindings::_helpers::recall::writer::RecallWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; use sqlbuild_python_base::records::main::fields::field::field as native_field; thread_local! { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs index 19428e5734..f7334ffdb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs @@ -2,22 +2,19 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyFloat}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_model_config::header_metadata::main::check_measurement_contract::check_measurement_contract; use sqlbuild_model_config::header_metadata::main::check_threshold_bound::check_threshold_bound; use sqlbuild_model_config::header_metadata::main::check_threshold_pair::check_threshold_pair; use sqlbuild_model_config::header_metadata::models::{ThresholdBound, ThresholdInput}; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; /// The `MeasurementAuditError` carrying `message`. fn measurement_error(py: Python<'_>, message: &str) -> PyErr { - match error_class(py, ErrorId::MeasurementAudit).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::MeasurementAudit, message) } /// How Python's threshold validation sees one limit value. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index bc281b155f..3dc6eb9540 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -73,10 +73,10 @@ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; use crate::bindings::_helpers::{ - analysis, analysis_session, build_progress, compile_command, compile_context, compiled_views, - compute_logs, dbt_events, dbt_interop, diagnostics, discovery, event_exporting, - execution_history, graph, model_config, planner_entries, planner_migrations, planner_phases, - presentation_text, project_assembly, result_output, rules, scopes, + analysis, analysis_session, build_progress, compile_command, compiled_views, compute_logs, + dbt_events, dbt_interop, discovery, event_exporting, execution_history, graph, model_config, + planner_entries, planner_migrations, planner_phases, project_assembly, result_output, rules, + scopes, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; @@ -359,12 +359,12 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ analysis_session::getters::GETTERS, build_progress::getters::GETTERS, compile_command::getters::GETTERS, - compile_context::getters::GETTERS, + sqlbuild_python_base::compile_context::constants::GETTERS, compiled_views::getters::GETTERS, compute_logs::getters::GETTERS, dbt_events::getters::GETTERS, dbt_interop::getters::GETTERS, - diagnostics::getters::GETTERS, + sqlbuild_python_base::diagnostics::constants::GETTERS, discovery::getters::GETTERS, event_exporting::getters::GETTERS, execution_history::getters::GETTERS, @@ -373,7 +373,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ planner_entries::getters::GETTERS, planner_migrations::getters::GETTERS, planner_phases::getters::GETTERS, - presentation_text::getters::GETTERS, + sqlbuild_python_base::presentation_text::constants::GETTERS, project_assembly::getters::GETTERS, result_output::getters::GETTERS, rules::getters::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs index 76a8f2f545..a31f6f8907 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs @@ -3,6 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_core::migration_values::main::migration_decision_traits::migration_decision_traits; use sqlbuild_core::migration_values::main::model_migration_blocks_build::model_migration_blocks_build as blocks_build; @@ -13,7 +14,6 @@ use crate::bindings::_helpers::cursor::scalars::parse_value; use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -157,10 +157,7 @@ pub(crate) fn column_migration_blocks_build(entry: &Bound<'_, PyAny>) -> PyResul /// The `PlannerInputError` carrying `message`. fn planner_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::PlannerInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::PlannerInput, &message) } /// A `--*-cursor-ts` value: text checked as an ISO timestamp and parsed, others unchanged. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 2d69b896aa..248b4cdcd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -8,7 +8,6 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; -use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -17,6 +16,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::type_system::main::types_equal::types_equal as native_types_equal; const MODEL_RESOURCE: &str = "model"; const SOURCE_RESOURCE: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs index 9518074dd2..02a4a5b0a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; @@ -16,7 +17,6 @@ use sqlbuild_refactor::refactoring::types::PhysicalColumn; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -43,10 +43,7 @@ fn edit_row(result: Result) -> PyResult { /// `ContractAdoptionError(message)`. fn adoption_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::ContractAdoption).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::ContractAdoption, &message) } /// `(edited contents, ownership conflict)`, raising the edit's adoption failure. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 7c1f54509c..0fa0c28a5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -36,15 +36,15 @@ use sqlbuild_refactor::refactoring::models::{ }; use sqlbuild_refactor::refactoring::types::Originals; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::refactor::offline_compile::{OfflineCompile, compile_for_refactor}; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, unit_test_rebindings, }; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs index f77fe0d5d1..97f7cf9351 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -6,14 +6,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { - host_entry(py, FacadeHost::SharedInputError) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value) + host_error(py, FacadeHost::SharedInputError, &message) } /// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs index 3b76b5bc5f..07a9f4a102 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -1,5 +1,6 @@ //! The public declaration indexes: enums, constants and resolved reusable model schemas. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::HashMap; use pyo3::prelude::{ @@ -7,17 +8,13 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; /// `CompileInputError(message)`. fn compile_input_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::CompileInput).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::CompileInput, &message) } fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 8ed14ac069..8ad46bdaf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -7,7 +7,8 @@ use sqlbuild_cache::store::models::NativeStore; use std::path::PathBuf; use std::sync::Mutex; -use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use sqlbuild_python_base::cache::main::open_store::open_store; +use sqlbuild_python_base::cache::main::save_store::save_store; use crate::bindings::_helpers::render::macro_call_classes::CallClassState; @@ -62,7 +63,7 @@ impl MacroCallMemo { path: PathBuf, environment: &str, ) -> PyResult<(Vec, usize)> { - if crate::bindings::_helpers::compile_context::delta::injection().is_some() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some() { return Err(pyo3::exceptions::PyRuntimeError::new_err( "a delta compile leaves the macro call store to full compiles", )); @@ -83,7 +84,7 @@ impl MacroCallMemo { /// Whether this compile may use the call store; a delta compile leaves it to full compiles. fn store_allowed(&self) -> bool { - crate::bindings::_helpers::compile_context::delta::injection().is_none() + sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() } /// Whether this compile recorded calls the attached store does not hold yet. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs index 2c0cba8b91..720656612e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs @@ -11,8 +11,8 @@ use sqlbuild_sqltext::sql_references::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::fields::field::field as native_field; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 0ba322af7c..ff60788c89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -10,8 +10,6 @@ use sqlbuild_host::models::{ SqlTestExecutionResult, SqlTestOutcome, }; -use crate::bindings::_helpers::compile_outputs::json_report::indented_json; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::aggregation::RetainedBuildResults; use crate::bindings::_helpers::result_output::assets::{ @@ -23,7 +21,9 @@ use crate::bindings::_helpers::result_output::checks::{ }; use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; const JSON_VERSION: i64 = 1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs index 54d84e1049..22b2bf3d8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -12,13 +12,13 @@ use sqlbuild_core::terminal_events::models::TerminalRequest; use crate::bindings::_helpers::result_output::integration_records::{ clone_record, degraded_record, degraded_warning, execution_succeeded, integration_record, }; -use crate::bindings::_helpers::result_output::integration_values::Checked; use crate::bindings::_helpers::result_output::projection::resource_identity; use crate::bindings::_helpers::result_output::terminal_index::{ TerminalEventClaim, TerminalEventIndex, current_index_object, }; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::result_output::types::Checked; pub(crate) const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs index 01e4edfff8..2b2705b7da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs @@ -11,9 +11,10 @@ use sqlbuild_core::integration_results::main::normalize_integration_record::norm use sqlbuild_core::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::result_output::integration_values::{json_value, validation_error}; use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::result_output::main::json_value::json_value; +use sqlbuild_python_base::result_output::main::validation_error::validation_error; /// One decoded part of a record: its fields as attributes, and as a dictionary. fn field(fields: &Py, py: Python<'_>, name: &str) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs index 3c2371c7a8..f820e6902f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs @@ -14,21 +14,24 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; use crate::bindings::_helpers::plan_output::values::any; use crate::bindings::_helpers::result_output::assets::microbatch_summary; use crate::bindings::_helpers::result_output::checks::{ measured_value, measurement_thresholds, python_check_status, }; -use crate::bindings::_helpers::result_output::integration_values::{ - Checked, bounded_metadata, optional_text, required_json, text, text_or_empty, -}; use crate::bindings::_helpers::result_output::projection::{ ResultIdentity, ResultKind, result_kind, }; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::result_output::main::bounded_metadata::bounded_metadata; +use sqlbuild_python_base::result_output::main::optional_text::optional_text; +use sqlbuild_python_base::result_output::main::required_json::required_json; +use sqlbuild_python_base::result_output::main::text::text; +use sqlbuild_python_base::result_output::main::text_or_empty::text_or_empty; +use sqlbuild_python_base::result_output::types::Checked; const FAILED_EVENT: &str = "resource_attempt_failed"; const SKIPPED_EVENT: &str = "resource_attempt_skipped"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index e68e0fd101..2a86603b89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -6,7 +6,6 @@ pub(crate) mod event_writer; pub(crate) mod getters; pub(crate) mod integration_envelope; pub(crate) mod integration_records; -pub(crate) mod integration_values; pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod resource_ids; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs index 3e5d66b5ff..f9f29a2ca4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs @@ -1,5 +1,6 @@ //! The invocation-scoped index of canonical lifecycle facts that result output projects from. +use sqlbuild_python_base::build_progress::main::progress::terminal_request::terminal_request; use std::sync::{Mutex, MutexGuard}; use pyo3::exceptions::PyRuntimeError; @@ -14,7 +15,6 @@ use sqlbuild_core::terminal_events::main::consume_terminal_fact::consume_termina use sqlbuild_core::terminal_events::main::latest_resource_terminal::latest_resource_terminal; use sqlbuild_core::terminal_events::main::lifecycle_terminal::lifecycle_terminal; use sqlbuild_core::terminal_events::models::{TerminalFact, TerminalIndex, TerminalRequest}; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; @@ -23,10 +23,6 @@ use sqlbuild_python_base::records::types::NativeAttr; /// The module attribute holding the `ContextVar` of the installed index. const CONTEXT_ATTRIBUTE: &str = "_terminal_event_index_context"; const CONTEXT_NAME: &str = "sqlbuild_terminal_event_index"; -const CLAIM_OPTIONS: (&[&str], &[&str]) = ( - &["resource_name"], - &["resource_id", "resource_attempt_id", "attempt_number"], -); struct IndexState { index: TerminalIndex, @@ -183,16 +179,9 @@ impl TerminalEventIndex { py: Python<'_>, options: Option>, ) -> PyResult> { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "claim_resource_terminal", options, CLAIM_OPTIONS)?; self.claim( py, - &TerminalRequest { - resource_name: arguments.value("resource_name")?.extract()?, - resource_id: arguments.value("resource_id")?.extract()?, - resource_attempt_id: arguments.value("resource_attempt_id")?.extract()?, - attempt_number: arguments.value("attempt_number")?.extract()?, - }, + &terminal_request(py, "claim_resource_terminal", options)?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 20e0242c8a..8af26d5baa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -17,7 +17,6 @@ use sqlbuild_cli::diff_report::models::{ use sqlbuild_cli::diff_report::types::DiffNamedValue; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_command::support::{boolean, text}; use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; @@ -30,6 +29,8 @@ use crate::bindings::_helpers::review_command::planning::ReviewSettings; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs index 6374d48028..bac0a07885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs @@ -13,13 +13,14 @@ use sqlbuild_analysis::review_command::constants::{ use sqlbuild_analysis::review_command::models::{IncrementalStart, ReviewGraph, ValidationCounts}; use crate::bindings::_helpers::build_command::command::run_build_command; -use crate::bindings::_helpers::build_command::support::{boolean, text}; use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::review_command::planning::ReviewSettings; use crate::bindings::_helpers::review_command::support::{quietly, review_error}; use crate::bindings::_helpers::sql_test_command::test_command::run_test_command; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index ff2fad1e64..e7dc5eeba3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -19,11 +19,9 @@ use sqlbuild_analysis::review_command::models::{ }; use sqlbuild_config::project::models::ReviewConfig; -use crate::bindings::_helpers::build_command::support::{boolean, text}; use crate::bindings::_helpers::change_selection::base_snapshot::{base_comparison, base_variant}; use crate::bindings::_helpers::change_selection::definition_snapshot::project_wide_keys; use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::compiled_views::model_reads; @@ -40,6 +38,9 @@ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::text::text; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs index 729b58f540..1f21885182 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/schema.rs @@ -7,7 +7,6 @@ use sqlbuild_analysis::review_command::constants::UNKNOWN_REVIEW_DATABASE_CODE; use sqlbuild_analysis::review_command::main::review_schema_name::review_schema_name; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_command::support::{boolean, none, text}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::janitor::janitor_keys::utc_datetime; @@ -15,6 +14,9 @@ use crate::bindings::_helpers::review_command::planning::{ReviewProject, ReviewS use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_command::main::boolean::boolean; +use sqlbuild_python_base::build_command::main::none::none; +use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 11f57e3588..59efe4c926 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -16,10 +16,6 @@ use sqlbuild_rules::custom_host_pool::types::{FactKey, HostPlan}; use sqlbuild_rules::custom_rule_cache::models::CustomEvaluation; use sqlbuild_rules::custom_rule_cache::types::ReadSet; -use crate::bindings::_helpers::cache::native_store::current_code_identity; -use crate::bindings::_helpers::compile_context::delta::{ - DeltaInjection, delta_unsupported, injection, -}; use crate::bindings::_helpers::rules::constants::{ CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULES_CACHE_FILE, }; @@ -40,6 +36,10 @@ use crate::bindings::_helpers::rules::fact_replay::{ use crate::bindings::_helpers::rules::rule_run::{bulk_cache_file, rules_bulk_cache_path}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; @@ -114,11 +114,11 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( let project_dir: PathBuf = run.project_dir.extract()?; let mut digests: FactDigests<'py> = FactDigests::new(run.project, &project_dir)?; let models: Vec<(String, String, Option)> = - match crate::bindings::_helpers::compile_context::delta::injection() { + match sqlbuild_python_base::compile_context::main::delta::injection::injection() { Some(injection) => { let Some(mut rows) = injection.model_subjects.clone() else { return Err( - crate::bindings::_helpers::compile_context::delta::delta_unsupported( + sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported( "the custom rules' subjects were not recorded", ), ); @@ -133,7 +133,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( } None => { let rows: Vec<(String, String, Option)> = digests.replay.model_rows()?; - crate::bindings::_helpers::compile_context::delta::capture(|capture| { + sqlbuild_python_base::compile_context::main::delta::capture::capture(|capture| { capture.model_subjects = Some(rows.clone()); }); rows @@ -191,7 +191,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( } cache.record(py, evaluations)?; } - crate::bindings::_helpers::compile_context::delta::capture(|capture| { + sqlbuild_python_base::compile_context::main::delta::capture::capture(|capture| { capture.fact_digests.extend( digests .digests @@ -378,7 +378,7 @@ impl<'py> FactDigests<'py> { return Ok(cached.clone()); } if let Some(injected) = - crate::bindings::_helpers::compile_context::delta::injected_fact_digest(key) + sqlbuild_python_base::compile_context::main::delta::injected_fact_digest::injected_fact_digest(key) { self.digests.insert(key.clone(), injected.clone()); return Ok(injected); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index 04cc1b6b99..9ff9c6b1a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -1,5 +1,6 @@ //! Run planned custom-rule invocations across isolated host processes and merge their results. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::BTreeSet; use std::fs::File; use std::io::{BufWriter, Write}; @@ -36,7 +37,6 @@ use crate::bindings::_helpers::rules::custom_host_observation::{ }; use crate::bindings::_helpers::rules::custom_host_payload::write_payload; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; const HOST_INPUTS: [&str; 3] = ["target", "rules-cache", "host-inputs"]; @@ -445,8 +445,5 @@ fn text(path: &Path) -> String { } pub(crate) fn rules_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::Rules).and_then(|class| class.call1((message,))) { - Ok(instance) => PyErr::from_value(instance), - Err(error) => error, - } + new_plain_error(py, ErrorId::Rules, &message) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 0a10185b46..97757d27ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -1,6 +1,7 @@ //! Custom-rule import closures, implementation fingerprints and statically found test cases. use sqlbuild_python_base::positional_record; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::path::{Path, PathBuf}; use pyo3::prelude::{ @@ -18,7 +19,6 @@ use sqlbuild_pysource::rule_evidence::models::{ }; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -151,10 +151,7 @@ fn custom_rule_test_evidence<'py>( /// The native `error_id` exception raised with `message`. fn rules_error(py: Python<'_>, error_id: ErrorId, message: String) -> PyErr { - match error_class(py, error_id).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, error_id, &message) } /// Whether custom-rule code may import one absolute module name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 261009650c..c3d355114f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::compiled_views::relation_keys::model_key_row; use crate::bindings::_helpers::compiled_views::relation_keys::{ RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, }; -use crate::bindings::_helpers::native_threads::interruptible_join::{Joined, join_interruptibly}; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, @@ -38,6 +37,8 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ use crate::bindings::_helpers::rules::rule_cache::stored_entries; use crate::bindings::_helpers::rules::rules_config::rules_config_of; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interruptibly; +use sqlbuild_python_base::native_threads::models::Joined; use sqlbuild_python_base::records::main::fields::field::field as record_field; /// The lint thresholds used when the project leaves them unset. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 2f1268aef5..314da78b83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -25,7 +25,6 @@ use sqlbuild_rules::models::{ RowsEvaluation, RowsRequest, RulesConfig, }; -use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; @@ -39,14 +38,15 @@ use crate::bindings::_helpers::rules::rule_run::bulk_cache_file; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; use crate::bindings::_helpers::scopes::scope_reports::metadata_projection_value; -use crate::bindings::_helpers::type_system::equality::types_equal; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_python_base::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -325,7 +325,8 @@ fn built_request( request.models = built_models(&rows, &request, model_rows)?; let request: EvaluateRequest = crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; - let delta: bool = crate::bindings::_helpers::compile_context::delta::injection().is_some(); + let delta: bool = + sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some(); let memo: Option<(String, PathBuf)> = rows .memo .take() @@ -334,7 +335,7 @@ fn built_request( Ok(RowsRequest { request, memo, - retain: crate::bindings::_helpers::compile_context::delta::capturing(), + retain: sqlbuild_python_base::compile_context::main::delta::capturing::capturing(), }) }) } @@ -375,9 +376,9 @@ fn evaluated<'py>( RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; if let Some(retained) = evaluation.retained.take() { - crate::bindings::_helpers::compile_context::delta::capture(|capture| { + sqlbuild_python_base::compile_context::main::delta::capture::capture(|capture| { capture.rules = - Some(crate::bindings::_helpers::compile_context::delta::CapturedRules { retained }); + Some(sqlbuild_python_base::compile_context::models::CapturedRules { retained }); }); } let classes = FindingClasses::new(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs index f346fa0808..61478d0317 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -1,5 +1,6 @@ //! Custom-rule API decisions: finding positions, harness option TOML and project-tree paths. +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyAttributeError; @@ -28,10 +29,7 @@ const DEFAULT_POSITION: i64 = 1; /// `RuleUsageError(message)`. fn usage_error(py: Python<'_>, message: String) -> PyErr { - match error_class(py, ErrorId::RuleUsage).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::RuleUsage, &message) } /// `RuleUsageError(prefix + str(value))`, keeping the value's text exactly as Python spells it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 73856f15cd..5a0e5b77ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -16,8 +16,6 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::cache::native_store::current_code_identity; -use crate::bindings::_helpers::compile_driver::stage_work::count_stage; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::rules::catalogue::{ resolved_ignore_selectors, rule_catalogue, select_rules, @@ -48,6 +46,8 @@ use crate::bindings::_helpers::rules::sql_rules::run_planned_sql_rules; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index 147b24a768..07fab2e972 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -6,13 +6,13 @@ use pyo3::prelude::{ }; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; use crate::bindings::_helpers::rules::config_json::{asdict_json, dumped_json}; use crate::bindings::_helpers::rules::constants::RULES_NATIVE_API_VERSION; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::types::ErrorId; @@ -99,11 +99,7 @@ pub(crate) fn rule_selected(config: &Bound<'_, PyAny>, code: &str) -> PyResult, message: &str, cause: Option) -> PyErr { - let raised = || -> PyResult { - let error: Bound<'_, PyAny> = error_class(py, ErrorId::Rules)?.call1((message,))?; - Ok(PyErr::from_value(error)) - }; - let error: PyErr = raised().unwrap_or_else(|error| error); + let error: PyErr = new_plain_error(py, ErrorId::Rules, message); if let Some(cause) = cause { error.set_cause(py, Some(cause)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index da28595296..d5b9c70d43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -154,7 +154,7 @@ pub(crate) fn run_planned_sql_rules<'py>( let project_files: Option> = if !identities.is_empty() && run.prepared().is_none() && !run.model_selection - && crate::bindings::_helpers::compile_context::delta::injection().is_none() + && sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() { Some(lint_sources(&run.project_dir, None, None)?) } else { @@ -391,7 +391,7 @@ fn linted_files( let py: Python<'_> = run.project.py(); let project_dir: &Path = &run.project_dir; let prepared = prepared_result(run, py); - if crate::bindings::_helpers::compile_context::delta::injection().is_some() { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some() { return crate::bindings::_helpers::compile_delta::lint_hooks::changed_files(project_dir); } Ok(match prepared { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 2c1da94267..c977e40b26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -15,8 +15,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::ScenarioRunResult; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::scenario_command::conversions::{ capture_row, scenario_result_row, write_local_runtime_target, write_warehouse_runtime_target, }; @@ -25,7 +23,9 @@ use crate::bindings::_helpers::scenario_command::setup::{ }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// How a finished scenario is reported. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 10d1ee5cab..243048f163 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -23,7 +23,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; @@ -32,11 +31,12 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_sce use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 4df42b8a0e..54a4e75a47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -25,7 +25,6 @@ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; @@ -44,6 +43,7 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; const LOCAL_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 03bde8e5d8..35ed2fce64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -19,7 +19,7 @@ use sqlbuild_scopes::scope_index::models::{ DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, }; -use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs index 55849d967d..7f8cd890c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs @@ -9,9 +9,9 @@ use sqlbuild_scopes::relationship_names::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::types::CompilerDetach; /// A test file's relative path and each block's name, SQL and mode. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 50198ee537..52e8f26d74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -13,7 +13,6 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::scopes::scope_loading::load_or_build_scope_index; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; @@ -22,6 +21,7 @@ use crate::bindings::_helpers::scopes::scope_reports::{ ExplainArguments, ListArguments, ReportArguments, scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, serialize_scope_report, }; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs index 10ec0814cc..1150bea0e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -1,8 +1,8 @@ //! Report a scope index's error findings as individual compile diagnostics. -use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 313269d102..dbbc0fd927 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -363,7 +363,7 @@ pub(crate) fn scope_index_with_compile_usages<'py>( usages.extend(usage_records( model_schema_scope_usages(&schema_files, &resolver)?.as_any(), )?); - let digests: bool = crate::bindings::_helpers::compile_context::delta::capturing(); + let digests: bool = sqlbuild_python_base::compile_context::main::delta::capturing::capturing(); for model in native_field(inputs, "model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let macro_usages: Vec> = @@ -371,7 +371,7 @@ pub(crate) fn scope_index_with_compile_usages<'py>( let declaration_usages: Vec> = usage_records(&native_field(&model, "declaration_usages")?)?; if digests { - crate::bindings::_helpers::compile_context::image_facts::record_model_usages( + sqlbuild_python_base::compile_context::main::image_facts::record_model_usages::record_model_usages( &model, (¯o_usages, &declaration_usages), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 76788fac00..9062640ff8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -16,7 +16,6 @@ use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; @@ -33,6 +32,7 @@ use crate::bindings::_helpers::result_output::documents::write_execution_json_ou use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs index 237ef9cddc..fca32c6226 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs @@ -7,8 +7,8 @@ use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; -use crate::bindings::_helpers::type_system::normalization::normalization_error; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::type_system::main::normalization_error::normalization_error; const SEMANTIC_CONTEXT: &str = "native semantic completion"; /// The wheel's `TypeError` for `tokenize(sql, dialect=None)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 1196e1eba6..40e167e5d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -18,7 +18,6 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::binding_catalog::schema_validations; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compile_driver::stage_work::count_stage; use crate::bindings::_helpers::compiled_views::compiled_model_view::LineageInput; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_names, lineage_inputs}; use crate::bindings::_helpers::compiled_views::records::with_type_recovery; @@ -31,6 +30,7 @@ use crate::bindings::_helpers::semantic_checks::session_facts::{ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index ff665a2249..feeeae8e32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -6,11 +6,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::semantic_shapes::delta_shapes::semantic_reasons; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_project::types::InferredRow; +use sqlbuild_python_base::compile_context::models::ModelCoverage; use sqlbuild_python_base::records::main::fields::field::field as native_field; const TYPE_CHECKED_DIALECTS: [&str; 4] = ["duckdb", "snowflake", "postgresql", "bigquery"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs index c3b3682244..49f37bd4a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs @@ -5,14 +5,16 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; -use crate::bindings::_helpers::compile_context::coverage::{ - coverage_reasons, merged_coverage, merged_order, -}; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; -use crate::bindings::_helpers::compile_context::delta::guarded; -use crate::bindings::_helpers::compile_context::delta::{ - ModelShape, capture, capturing, delta_unsupported, injection, -}; +use sqlbuild_python_base::compile_context::main::coverage::coverage_reasons::coverage_reasons; +use sqlbuild_python_base::compile_context::main::coverage::merged_coverage::merged_coverage; +use sqlbuild_python_base::compile_context::main::coverage::merged_order::merged_order; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::compile_context::models::ModelCoverage; +use sqlbuild_python_base::compile_context::types::ModelShape; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The partial-check reasons of the `fresh` models, merged in a delta with the stored models'. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index 3b2737ba2f..da8ce29f46 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::analysis::schema_request::{ }; use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; -use crate::bindings::_helpers::compile_driver::stage_work::count_stage; use crate::bindings::_helpers::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ @@ -23,6 +22,7 @@ use crate::bindings::_helpers::semantic_shapes::shapes::{ use crate::bindings::models::BindingPositions; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index f31a057c49..e9ef355756 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -13,13 +13,14 @@ use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; -use crate::bindings::_helpers::build_progress::terminal::{flush, write}; use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; use crate::bindings::_helpers::graph::python_node_graph::{ python_node_graph, run_selection_from_inputs, }; use crate::bindings::_helpers::graph::python_selection::python_node_names; use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use sqlbuild_python_base::build_progress::main::terminal::flush::flush; +use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::native_host::main::flush_console::flush_console; use sqlbuild_python_base::native_host::main::module_installed::module_installed; use sqlbuild_python_base::native_host::main::python_executable::python_executable; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 4ac1e5519d..cd434692d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -25,13 +25,11 @@ use sqlbuild_host::types::ResultEnum; use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::build_progress::nested_progress::NestedCommandProgressCallbacks; -use crate::bindings::_helpers::build_progress::progress::current_native_progress_projector; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::{ format_expectation_detail, format_expectation_name, write_execution_header, }; use crate::bindings::_helpers::build_progress::result_text::{test_label, test_result_label}; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::change_selection::base_snapshot::{ BaseComparison, base_comparison, base_variant, }; @@ -45,7 +43,6 @@ use crate::bindings::_helpers::compile_driver::static_context::compile_static_co use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -61,7 +58,10 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index ab41dafc8f..6e04c994fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -12,9 +12,9 @@ use sqlbuild_sqltext::sql_references::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compile_context::diagnostic_collector::report_compile_diagnostic; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs index 2c783afa20..f477b7d8b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -6,6 +6,7 @@ use pyo3::types::PyString; use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_state::events::constants::{ COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, }; @@ -15,7 +16,6 @@ use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -57,10 +57,7 @@ pub(crate) fn layout(kind: &str) -> PyResult { /// `MigrationStateError(message)`. pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { - match error_class(py, ErrorId::MigrationState).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_plain_error(py, ErrorId::MigrationState, message) } /// The adapter-qualified name of `layout`'s table in one schema. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index f75a9c2a43..c3769c7844 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -16,10 +16,10 @@ use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; use crate::bindings::_helpers::state_rows::freshness::record_event_id; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 7070281c01..51ec6fea5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,4 +1,2 @@ -pub(crate) mod equality; -pub(crate) mod normalization; pub(crate) mod nullability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index 1077f61865..5c16c08005 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::type_system::{normalization, nullability}; +use crate::bindings::_helpers::type_system::nullability; +use sqlbuild_python_base::type_system::main::register_normalization::register_normalization; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - normalization::register(module)?; + register_normalization(module)?; nullability::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index ff8e8b4e6e..cf006c486b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -20,11 +20,11 @@ use sqlbuild_config::workspace::models::{PlaygroundRequest, SkillInstallRequest} use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::FacadeHost; -use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 2dc7f9c480..69ffd18969 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -14,5 +14,3 @@ mod since_cache; mod splice; #[path = "tests/test_types.rs"] mod test_types; -#[path = "tests/test_yaml_cache.rs"] -mod yaml_cache; diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index 9df3e0dfcf..948e5a6d98 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -3,22 +3,21 @@ use std::path::{Path, PathBuf}; use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; use sqlbuild_cache::project_reuse::types::ImageSections; -use sqlbuild_config::models::ConfigValue; use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; -use crate::bindings::_helpers::compile_context::delta::BaseRules; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; -use crate::bindings::_helpers::compile_context::delta::PendingRules; -use crate::bindings::_helpers::compile_context::delta::{ - CapturedDiagnostic, CapturedModel, CapturedTest, -}; use crate::bindings::_helpers::compile_delta::image::{ FACTS_SECTION, GATE_SECTION, IMAGE_FORMAT, ImageRelations, ImageRules, ProjectImage, RULE_DIGESTS_SECTION, }; +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_context::models::ModelCoverage; +use sqlbuild_python_base::compile_context::models::PendingRules; /// The rows of a request without models: a zero model count. const RULE_ROWS: [u8; 8] = [0; 8]; @@ -241,11 +240,6 @@ pub(crate) fn names(values: &[&str]) -> BTreeSet { values.iter().map(|value| (*value).to_owned()).collect() } -/// A YAML string value. -pub(crate) fn text(value: &str) -> ConfigValue { - ConfigValue::String(value.to_owned()) -} - /// One model's semantic coverage facts. pub(crate) fn coverage( name: &str, diff --git a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs index 57501219d9..b3d4fcc24f 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs @@ -1,12 +1,12 @@ use std::collections::BTreeSet; -use crate::bindings::_helpers::compile_context::coverage::{ - coverage_reasons, merged_coverage, merged_order, -}; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compile_delta::driver::names_uncompiled_model; use crate::bindings::tests::helpers::{coverage, names, order, strings}; use crate::bindings::tests::test_types::{CoverageTestCase, UncompiledModelTestCase}; +use sqlbuild_python_base::compile_context::main::coverage::coverage_reasons::coverage_reasons; +use sqlbuild_python_base::compile_context::main::coverage::merged_coverage::merged_coverage; +use sqlbuild_python_base::compile_context::main::coverage::merged_order::merged_order; +use sqlbuild_python_base::compile_context::models::ModelCoverage; const DISABLED: &str = "SQL analysis disabled"; const OPEN_INPUT: &str = "open input: raw_orders"; diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index af538f698a..aa09bf8593 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -1,8 +1,7 @@ use sqlbuild_cache::project_reuse::types::ImageSections; -use sqlbuild_config::models::ConfigValue; -use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compile_delta::image::ProjectImage; +use sqlbuild_python_base::compile_context::models::ModelCoverage; /// A stored report, a delta report and what splicing the delta into the stored one gives. pub(crate) struct SpliceTestCase { @@ -16,14 +15,6 @@ pub(crate) struct SpliceTestCase { pub(crate) expected_report: Option<&'static str>, } -/// A loaded YAML document kept by the delta compile's document store. -pub(crate) struct DocumentTestCase { - pub(crate) description: &'static str, - pub(crate) document: ConfigValue, - /// How many truncations of the stored document still read back as it. - pub(crate) expected_truncations_read: usize, -} - /// Stored image bytes damaged before they are read back. pub(crate) struct DamagedImageTestCase { pub(crate) description: &'static str, diff --git a/fensu.toml b/fensu.toml index dd9592d42d..42491832ec 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1444,33 +1444,9 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgre reason = "PostgreSQL and SQLite execution history each read a page through their own connection, locking and cursor keys; filters and record conversion come from conversion.rs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs", "crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs", "crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs"] reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs"] -reason = "Per-module constructors for the Python exception each module raises; they differ in the exception class and its arguments. One shared constructor for all of them is a separate change." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs"] reason = "Each builds one staying-Python object from its own keyword set." @@ -1516,7 +1492,7 @@ paths = ["crates/sqlbuild-cli/src/workers/_helpers/child.rs"] reason = "terminate and kill are the two escalation steps for a worker child; each maps every child kind to its own signal." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." [[targets.python.rule_exceptions]] @@ -1703,12 +1679,12 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 206367 } +thresholds = { max_crate_code_lines = 197887 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] -thresholds = { max_helpers_container_buckets = 101 } +thresholds = { max_helpers_container_buckets = 98 } reason = "The bindings helpers container is above the bucket limit; this caps it at its current bucket count until cohesive groups move into their own modules or crates. Lower the cap as buckets move out." [targets.rust.rule_options.XSB076] @@ -1743,7 +1719,11 @@ allowed_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", - "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.", + "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs=Defines the synced-copy adapter.", + "crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs=Registers the synced copy's refusal error with Python.", + "crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs=Declares the synced-copy entries.", + "crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs=Tells a plan preview's refused warehouse access apart.", + "crates/sqlbuild-python-base/src/warehouse_access/models.rs=Defines the synced-copy adapter the plan preview reads through.", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=Preview branch: table types and retention stay unknown.", @@ -1766,6 +1746,18 @@ exceptions = [ "UsageKind=Wire enum that serde builds from Python usage records.", ] +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-base/src/compile_context/models.rs" +symbols = ["PythonProcessClock"] +reason = "PythonProcessClock implements Debug manually because it holds the interpreter token, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-base/src/native_host/models.rs" +symbols = ["AuditProjection", "FamilyProjection"] +reason = "The header projections implement Debug manually because their cached public tuples sit in a PyOnceLock, which does not implement Debug; other models derive it." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -1839,7 +1831,7 @@ reason = "Every native entry point installs the build identity before anything r [[targets.rust.rule_exceptions]] rule = "FPRSS130" -path = "crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs" +path = "crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs" symbols = ["signal_interpreter_exit"] reason = "The existing interpreter-exit signal is process-wide and monotonic: every native daemon wait must observe any exit hook before Python finalization. The atomic replaces the same global threading.Event, not per-command lifecycle state." @@ -1951,7 +1943,7 @@ paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] @@ -1990,20 +1982,52 @@ reason = "The delivery thread claims each call in the dict it shares with the na [targets.rust.rule_options.XSB072] edge_paths = [ "crates/sqlbuild-python-base/src/native_host", + "crates/sqlbuild-python-base/src/orchestration_reads", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/records/main/registry/native_attr.rs", + "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ "crates/sqlbuild-python-base/src/boundary/_helpers/config_values.rs=13", "crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs=11", + "crates/sqlbuild-python-base/src/boundary/_helpers/text_units.rs=1", "crates/sqlbuild-python-base/src/boundary/main/imported.rs=1", "crates/sqlbuild-python-base/src/boundary/models.rs=2", + "crates/sqlbuild-python-base/src/build_command/_helpers/support.rs=7", + "crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs=7", + "crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs=23", + "crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs=2", + "crates/sqlbuild-python-base/src/build_progress/models.rs=18", + "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs=13", + "crates/sqlbuild-python-base/src/cache/_helpers/sql_test_scan_store.rs=3", + "crates/sqlbuild-python-base/src/cli/_helpers/launch_state.rs=8", + "crates/sqlbuild-python-base/src/command_runtime/_helpers/adapters.rs=1", + "crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs=4", + "crates/sqlbuild-python-base/src/compile_context/models.rs=7", + "crates/sqlbuild-python-base/src/contracts/_helpers/resource_identities.rs=1", + "crates/sqlbuild-python-base/src/contracts/_helpers/state_literals.rs=11", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/console_records.rs=1", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs=8", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/redaction.rs=1", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/trackers.rs=9", + "crates/sqlbuild-python-base/src/diagnostics/models.rs=3", + "crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs=3", + "crates/sqlbuild-python-base/src/event_exporting/_helpers/values.rs=1", + "crates/sqlbuild-python-base/src/event_exporting/models.rs=11", + "crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs=3", + "crates/sqlbuild-python-base/src/frontier/_helpers/python_types.rs=29", + "crates/sqlbuild-python-base/src/frontier/models.rs=3", + "crates/sqlbuild-python-base/src/planner_identity/_helpers/json_values.rs=4", + "crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs=2", + "crates/sqlbuild-python-base/src/presentation_text/_helpers/terminal.rs=3", + "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs=4", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/error_classes.rs=9", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registration.rs=26", "crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs=2", + "crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", @@ -2016,11 +2040,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", @@ -2030,27 +2054,19 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", @@ -2059,11 +2075,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", @@ -2082,9 +2097,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", @@ -2095,7 +2107,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs=3", @@ -2111,7 +2123,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", @@ -2126,15 +2137,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", @@ -2149,7 +2158,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs=7", @@ -2157,7 +2166,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs=3", @@ -2165,7 +2174,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=9", @@ -2179,9 +2188,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=6", @@ -2190,10 +2199,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", @@ -2205,7 +2210,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", @@ -2219,24 +2224,21 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/snapshot.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs=1", @@ -2252,8 +2254,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs=29", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", @@ -2295,7 +2295,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", @@ -2303,17 +2303,17 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs=1", @@ -2333,7 +2333,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", @@ -2346,7 +2346,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", @@ -2356,10 +2356,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=3", @@ -2399,8 +2398,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs=4", @@ -2434,16 +2433,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", @@ -2453,7 +2447,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs=1", @@ -2461,10 +2455,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs=11", @@ -2472,10 +2466,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", @@ -2490,7 +2484,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", @@ -2502,9 +2495,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", @@ -2521,13 +2514,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", @@ -2578,13 +2571,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", "crates/sqlbuild-python/src/bindings/models.rs=0", ] @@ -2592,13 +2583,29 @@ pending_budgets = [ [targets.rust.rule_options.XSB073] edge_paths = [ "crates/sqlbuild-python-base/src/native_host", + "crates/sqlbuild-python-base/src/orchestration_reads", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", + "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ "crates/sqlbuild-python-base/src/boundary/models.rs=2", + "crates/sqlbuild-python-base/src/build_command/_helpers/support.rs=2", + "crates/sqlbuild-python-base/src/build_progress/_helpers/transient_status.rs=2", + "crates/sqlbuild-python-base/src/build_progress/models.rs=6", + "crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs=1", + "crates/sqlbuild-python-base/src/compile_context/models.rs=2", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs=2", + "crates/sqlbuild-python-base/src/diagnostics/_helpers/trackers.rs=2", + "crates/sqlbuild-python-base/src/diagnostics/models.rs=2", + "crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs=6", + "crates/sqlbuild-python-base/src/event_exporting/models.rs=5", + "crates/sqlbuild-python-base/src/frontier/_helpers/python_types.rs=15", + "crates/sqlbuild-python-base/src/frontier/models.rs=1", + "crates/sqlbuild-python-base/src/planner_identity/_helpers/json_values.rs=2", + "crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs=1", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/error_classes.rs=1", "crates/sqlbuild-python-base/src/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", @@ -2613,14 +2620,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=1", @@ -2639,8 +2642,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=5", @@ -2687,8 +2688,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", @@ -2702,8 +2701,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", @@ -2713,8 +2710,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=4", @@ -2766,7 +2761,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", @@ -2787,7 +2781,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", @@ -2881,35 +2874,25 @@ module_layers = [ "auditing/render=project", "authored_values=front", "build_command=commands", - "build_command/support=infra", "build_progress=output", - "build_progress/progress=infra", - "build_progress/terminal=infra", - "build_progress/transient_lines=infra", - "build_progress/transient_status=infra", - "cache=infra", + "cache=registry", "change_selection=commands", "check=commands", "cli=commands", - "cli/launch_state=infra", "clone=commands", "command_runtime=commands", - "command_runtime/adapters=infra", "command_runtime/concurrency=front", "compile_attachment=compile", "compile_attachment/cursor_roles=project", "compile_attachment/effective_config=front", "compile_command=compile", - "compile_context=infra", "compile_delta=compile", "compile_driver=compile", "compile_driver/config_validation=front", "compile_driver/plan_pipeline=commands", "compile_driver/plan_result=commands", - "compile_driver/stage_work=infra", "compile_driver/static_context=commands", "compile_outputs=compile", - "compile_outputs/json_report=infra", "compiled_project=project", "compiled_views=project", "compute_logs=output", @@ -2917,9 +2900,7 @@ module_layers = [ "consumption=plan", "contracts=project", "contracts/effective_config=front", - "contracts/resource_identities=infra", "contracts/spec_resolution=front", - "contracts/state_literals=infra", "cost_command=commands", "cost_report=output", "cost_telemetry=output", @@ -2932,14 +2913,10 @@ module_layers = [ "dbt_events=commands", "dbt_interop=commands", "debug_command=commands", - "diagnostics=infra", + "diagnostics=registry", "diff_report=commands", "discovery=front", "event_exporting=output", - "event_exporting/constants=infra", - "event_exporting/models=infra", - "event_exporting/scope=infra", - "event_exporting/values=infra", "execution_history=front", "exported_constants=registry", "fingerprints=project", @@ -2947,8 +2924,6 @@ module_layers = [ "freshness=plan", "freshness/freshness_command=commands", "frontier=compile", - "frontier/encoder=infra", - "frontier/python_types=infra", "graph=project", "hand_built=commands", "hand_built/compiled_model=compile", @@ -2965,7 +2940,7 @@ module_layers = [ "manifest=compile", "microbatches=plan", "model_config=front", - "native_host=infra", + "native_host=front", "native_host/audit_results_state=front", "native_host/consumption_state=front", "native_host/model_lookup=project", @@ -2974,7 +2949,6 @@ module_layers = [ "native_host/orchestration_triggers=project", "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", - "native_threads=infra", "observations=plan", "output_capture=output", "plan_command=commands", @@ -2984,7 +2958,6 @@ module_layers = [ "planner_identity=plan", "planner_identity/graph=project", "planner_identity/hashing=project", - "planner_identity/json_values=infra", "planner_migrations=plan", "planner_phases=plan", "planner_phases/models=project", @@ -3009,9 +2982,8 @@ module_layers = [ "planner_strategy/models=project", "planner_strategy/types=project", "planner_warehouse=plan", - "presentation_text=infra", + "presentation_text=registry", "project_assembly=compile", - "python_values=infra", "query_command=commands", "query_command/text_output=output", "recall=output", @@ -3024,7 +2996,6 @@ module_layers = [ "relation_naming=project", "render=front", "result_output=output", - "result_output/integration_values=infra", "review_command=commands", "rules=compile", "scenario_command=commands", @@ -3053,8 +3024,6 @@ module_layers = [ "sqltext=front", "state_rows=front", "type_system=project", - "type_system/equality=infra", - "type_system/normalization=infra", "workspace_commands=commands", ] pending_budgets = [ @@ -3095,6 +3064,86 @@ reason = "Each is a short stack walk over a JSON syntax tree with its own stop a paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs"] reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs", "crates/sqlbuild-analysis/src/manifest/_helpers/values.rs", "crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs", "crates/sqlbuild-config/src/project/_helpers/fields.rs", "crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs", "crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs", "crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs"] +reason = "Each fills its own ordered key/value list in its own crate; a shared ordered-map helper across these crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs"] +reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/getters.rs"] +reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/semantic_validation/models.rs", "crates/sqlbuild-orchestration/src/api/models.rs", "crates/sqlbuild-orchestration/src/credentials/models.rs", "crates/sqlbuild-python-base/src/native_host/models.rs"] +reason = "Hand-written Debug implementations that each choose which of their own fields to show, so secrets, caches and large catalogs stay out of logs." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs"] +reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs"] +reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs"] +reason = "The capture encoder sorts by canonical capture key and freshness planning by resource identity; only the decorate-sort-undecorate shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/models.rs", "crates/sqlbuild-rules/src/engine/_helpers/retained.rs"] +reason = "The project image decoder and the retained-rows reader each decode their own count-prefixed binary format; only the list loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs"] +reason = "One writes a persistent line through the line coordinator, the other reads a finished resource's duration from the projector; both only pass keyword options to one method." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs"] +reason = "Each context object sets its own context variable and keeps its own token under its own lock; only the set-and-store shape is shared." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs"] +reason = "The transient status line and the build progress callbacks each own their spinner thread and stop event; merging the two spinners is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs"] +reason = "The transient status line and the build progress callbacks each own their spinner thread and stop event; merging the two spinners is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs"] +reason = "Each reads its own Python mapping into its own native pairs." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs"] +reason = "Each reads its own Python record's fields into its own native record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs", "crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs"] +reason = "Each public entry forwards to its own helper with the same arguments." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index f5f0c091dd..f4a5d3bcbf 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -500,6 +500,6 @@ ) SYNCED_COPY_PATTERN: str = ( r"\b(?:SyncedCopyAdapter|SyncedCopyConnection|PLAN_SYNCED_COPY_ENV|" - r"WarehouseAccessRefusedError)\b" + r"WarehouseAccessRefusedError|is_warehouse_access_refused|register_synced_copy)\b" ) RUST_LINE_COMMENT: str = "//" diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index e02c35787b..316c2ae063 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -35,19 +35,18 @@ crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs ColumnMig crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs TransientLineCoordinator.lock crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs TransientLineCoordinator.lock crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs CliNamespace.command -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs CliNamespace.dbt_args -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs CliNamespace.json -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs CliNamespace.lineage_format -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs CliNamespace.query_format -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs TransientLineCoordinator.lock +crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.command +crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.dbt_args +crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.json +crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.lineage_format +crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.query_format +crates/sqlbuild-python-base/src/build_progress/models.rs TransientLineCoordinator.lock crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs VerboseInspectionWriter.write crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs SqlValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs BuildProgressCallbacks._spin_until_stopped -crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs NestedCommandProgressCallbacks._spin_until_stopped -crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs TransientStatusReporter._spin_until_stopped -crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs TransientStatusReporter.close +crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs BuildProgressCallbacks._spin_until_stopped +crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs NestedCommandProgressCallbacks._spin_until_stopped +crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs TransientStatusReporter._spin_until_stopped +crates/sqlbuild-python-base/src/build_progress/models.rs TransientStatusReporter.close crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs PythonNodeGraph.selected_sql_refs crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs PythonNodeGraph.selected_sql_refs crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.command @@ -186,49 +185,55 @@ crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs Event crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs SourceWriteStrategy.value crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs DeclarationKind.value crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs ScopeKind.value -crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs EventExporterCommandScope.cached_extensions +crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs EventExporterCommandScope.cached_extensions crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs TransientStatusReporter.complete crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs SourceFreshnessAgeStatus.value crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessStrategy.value crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BackfillAction.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.analysis_shapes -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.dialect -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.expression_shapes -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.quoted_ignore_case -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs BindingCatalog.schemas -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs CollectionRendering.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs ColumnLineageConfidence.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs ColumnTransformKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs CompiledResourceType.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs DeclarationKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs DiagnosticPhase.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs DiagnosticSeverity.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs EventExportSeverity.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs FutureCursorAction.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs GrantKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs HardCodedRelationOwnerKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs InferredNullability.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs LoaderConnectionMode.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs MaterializationType.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs MicrobatchLimitAction.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs MissingMigrationOriginPolicy.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs OwnershipRootKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs PlanAction.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs PlanReason.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs ProjectCatalog.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs ResourceKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs RetentionDecreasePolicy.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs ScopeKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs SourceWriteStrategy.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs SqlReferenceKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs SqlValueKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs TableTypeDowngradePolicy.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs TableType.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs TableTypeSource.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs UsageKind.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs VisibilityReason.name +crates/sqlbuild-python-base/src/frontier/models.rs BackfillAction.name +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.analysis_shapes +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.analysis_shapes +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.dialect +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.dialect +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.expression_shapes +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.expression_shapes +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.native +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.native +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.quoted_ignore_case +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.quoted_ignore_case +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.schemas +crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.schemas +crates/sqlbuild-python-base/src/frontier/models.rs CollectionRendering.name +crates/sqlbuild-python-base/src/frontier/models.rs ColumnLineageConfidence.name +crates/sqlbuild-python-base/src/frontier/models.rs ColumnTransformKind.name +crates/sqlbuild-python-base/src/frontier/models.rs CompiledResourceType.name +crates/sqlbuild-python-base/src/frontier/models.rs DeclarationKind.name +crates/sqlbuild-python-base/src/frontier/models.rs DiagnosticPhase.name +crates/sqlbuild-python-base/src/frontier/models.rs DiagnosticSeverity.name +crates/sqlbuild-python-base/src/frontier/models.rs EventExportSeverity.name +crates/sqlbuild-python-base/src/frontier/models.rs FutureCursorAction.name +crates/sqlbuild-python-base/src/frontier/models.rs GrantKind.name +crates/sqlbuild-python-base/src/frontier/models.rs HardCodedRelationOwnerKind.name +crates/sqlbuild-python-base/src/frontier/models.rs InferredNullability.name +crates/sqlbuild-python-base/src/frontier/models.rs LoaderConnectionMode.name +crates/sqlbuild-python-base/src/frontier/models.rs MaterializationType.name +crates/sqlbuild-python-base/src/frontier/models.rs MicrobatchLimitAction.name +crates/sqlbuild-python-base/src/frontier/models.rs MissingMigrationOriginPolicy.name +crates/sqlbuild-python-base/src/frontier/models.rs OwnershipRootKind.name +crates/sqlbuild-python-base/src/frontier/models.rs PlanAction.name +crates/sqlbuild-python-base/src/frontier/models.rs PlanReason.name +crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs ProjectCatalog.__dict__ +crates/sqlbuild-python-base/src/frontier/models.rs ResourceKind.name +crates/sqlbuild-python-base/src/frontier/models.rs RetentionDecreasePolicy.name +crates/sqlbuild-python-base/src/frontier/models.rs ScopeKind.name +crates/sqlbuild-python-base/src/frontier/models.rs SourceWriteStrategy.name +crates/sqlbuild-python-base/src/frontier/models.rs SqlReferenceKind.name +crates/sqlbuild-python-base/src/frontier/models.rs SqlValueKind.name +crates/sqlbuild-python-base/src/frontier/models.rs TableTypeDowngradePolicy.name +crates/sqlbuild-python-base/src/frontier/models.rs TableType.name +crates/sqlbuild-python-base/src/frontier/models.rs TableTypeSource.name +crates/sqlbuild-python-base/src/frontier/models.rs UsageKind.name +crates/sqlbuild-python-base/src/frontier/models.rs VisibilityReason.name crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs CompilerStage.value crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs ResourceKind.value crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs DeclarationKind.value @@ -366,7 +371,7 @@ crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PlanReason.va crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonIdentityStatus.value crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonRunPhase.value crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs WarningSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs CliStyle.use_color +crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs CliStyle.use_color crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.detail crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.file_path crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.help