From 7f9ba6539b5436b6eaae00bbfa38c9902a9afeb2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:53:28 +0000 Subject: [PATCH] refactor: move the compile layer of the Python bindings into sqlbuild-python-compile sqlbuild-python is still over the crate size limit. The compile layer (compile command and driver, input loops, compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing) moves into a new crate above sqlbuild-python-project and beside sqlbuild-python-planner. Neither depends on the other. Command handlers that also reach the planner stay in sqlbuild-python. Old paths are deleted and callers updated; there are no re-exports. main/ entries hold the bodies they used to forward to, except 13 whose bodies are over the entry size budgets. The sqlbuild-python cap drops to its new size, XSB072/XSB073 budgets are re-keyed with unchanged totals, and the native field fallback list and compiler-differential scan paths follow the moved files. --- Cargo.lock | 36 +- Cargo.toml | 5 +- crates/README.md | 5 +- crates/sqlbuild-python-compile/Cargo.toml | 37 + .../analysis/_helpers/binding_locations.rs | 25 + .../src/analysis/_helpers}/getters.rs | 10 +- .../src/analysis/_helpers}/mod.rs | 2 +- .../src/analysis/_helpers/queries.rs | 31 + .../analysis/_helpers}/required_columns.rs | 0 .../_helpers}/sql_test_comparisons.rs | 54 +- .../src/analysis/_helpers/sql_tests.rs | 15 + .../src/analysis/constants.rs | 12 + .../main/locate_model_binding_diagnostics.rs | 40 + .../src/analysis/main/mod.rs | 5 + .../src/analysis/main/register_queries.rs | 19 + .../main/register_sql_test_comparisons.rs | 23 + .../src/analysis/main/register_sql_tests.rs | 11 + .../analysis/main/single_comparison_sql.rs | 27 + .../src/analysis/mod.rs | 7 + .../src/analysis/models.rs | 35 + .../src/analysis/types.rs | 7 + .../src/analysis_session/_helpers}/getters.rs | 10 +- .../src/analysis_session/_helpers/mod.rs | 4 + .../_helpers}/model_analyses.rs | 73 +- .../src/analysis_session/_helpers}/oracle.rs | 21 +- .../_helpers}/outcome_objects.rs | 21 +- .../src/analysis_session/constants.rs | 12 + .../analysis_session/main/analyze_models.rs | 44 ++ .../main/dynamic_contract_proof.rs | 11 + .../src/analysis_session/main/mod.rs | 5 + .../analysis_session/main/register_oracle.rs | 13 + .../main/register_outcome_objects.rs | 12 + .../main/session_outcome_analysis.rs | 15 + .../src/analysis_session/mod.rs | 6 + .../src/analysis_session/models.rs | 34 + .../src/auditing/_helpers}/identity.rs | 11 +- .../src/auditing/_helpers}/measurement.rs | 15 +- .../src/auditing/_helpers/mod.rs | 3 + .../src/auditing/_helpers}/schema_audits.rs | 13 +- .../src/auditing/main/mod.rs | 3 + .../src/auditing/main/register_identity.rs | 11 + .../src/auditing/main/register_measurement.rs | 11 + .../auditing/main/register_schema_audits.rs | 11 + .../src/auditing/mod.rs | 4 + .../_helpers}/declaration_scope.rs | 0 .../src/compile_attachment/_helpers/mod.rs | 6 + .../_helpers}/model_config.rs | 4 +- .../_helpers}/model_inputs.rs | 51 +- .../_helpers}/sql_expansion_context.rs | 37 +- .../_helpers}/storage_policies.rs | 0 .../compile_attachment/_helpers}/target.rs | 70 +- .../build_compile_target_context_function.rs | 10 + .../main/build_declaration_scope.rs | 22 + .../main/build_declaration_scope_function.rs | 10 + .../main/build_model_inputs.rs | 17 + .../main/build_model_inputs_function.rs | 10 + .../build_sql_expansion_context_function.rs | 10 + .../main/connection_is_default_target.rs | 17 + .../main/effective_target_namespace.rs | 19 + .../src/compile_attachment/main/mod.rs | 14 + .../native_model_config_session_function.rs | 10 + .../src/compile_attachment/main/present.rs | 8 + .../main/project_location.rs | 17 + .../main/rebind_declaration_scope_function.rs | 10 + .../main/resolved_target.rs | 31 + .../main/sql_expansion_inputs.rs | 12 + .../src/compile_attachment/mod.rs | 5 + .../src/compile_attachment/models.rs | 27 + .../_helpers/artifacts/artifact_batch.rs | 28 + .../compile_command/_helpers/artifacts/mod.rs | 5 + .../_helpers/artifacts}/prepared_artifacts.rs | 25 +- .../_helpers/artifacts}/surrogates.rs | 0 .../_helpers/artifacts}/target_writer.rs | 112 +-- .../artifacts}/test_artifact_cache.rs | 0 .../_helpers/invocation}/getters.rs | 39 +- .../_helpers/invocation/mod.rs | 6 + .../_helpers/invocation}/report.rs | 99 +-- .../_helpers/invocation}/reuse_attempt.rs | 162 ++-- .../_helpers/invocation/reuse_recording.rs | 327 ++++++++ .../_helpers/invocation}/runtime_identity.rs | 25 +- .../_helpers/invocation}/startup_inputs.rs | 13 +- .../src/compile_command/_helpers/mod.rs | 2 + .../src}/compile_command/constants.rs | 92 ++- .../src/compile_command/main/artifacts/mod.rs | 7 + .../main/artifacts/planning_diagnostics.rs | 25 + .../artifacts/register_prepared_artifacts.rs | 9 + .../main/artifacts/register_target_writer.rs | 12 + .../main/artifacts/remove_failed_compile.rs | 13 + .../main/artifacts/write_compile_target.rs | 28 + .../artifacts/write_rule_failed_models.rs | 23 + .../main/artifacts/write_static_target.rs | 18 + .../main/invocation/compile_reuse_timings.rs | 28 + .../main/invocation/diagnostic_item.rs | 10 + .../invocation/format_compile_error_json.rs | 66 ++ .../format_compile_json_with_notice.rs | 17 + .../main/invocation/format_compile_text.rs | 17 + .../main/invocation/interpreter_values.rs | 18 + .../compile_command/main/invocation/mod.rs | 16 + .../main/invocation/new_artifact_recorder.rs | 16 + .../main/invocation/print_to.rs | 20 + .../main/invocation/register_report.rs | 14 + .../main/invocation/register_reuse_attempt.rs | 23 + .../invocation/register_reuse_recording.rs | 14 + .../invocation/register_runtime_identity.rs | 10 + .../invocation/register_startup_inputs.rs | 11 + .../main/invocation/reuse_debug.rs | 10 + .../main/invocation/run_compile.rs | 35 + .../main/invocation/write_reusable_compile.rs | 91 +++ .../src/compile_command/main/mod.rs | 2 + .../src/compile_command/mod.rs | 7 + .../src/compile_command/models.rs | 382 +++++++++ .../src/compile_command/types.rs | 7 + .../src/compile_delta/_helpers}/driver.rs | 176 +---- .../src/compile_delta/_helpers/image.rs | 391 ++++++++++ .../src/compile_delta/_helpers}/mod.rs | 1 - .../compile_delta/_helpers}/report_hooks.rs | 99 +-- .../src/compile_delta/_helpers/rules_rows.rs | 55 ++ .../src/compile_delta/_helpers}/splice.rs | 11 +- .../src/compile_delta/_helpers/verify.rs | 83 ++ .../src/compile_delta/constants.rs | 19 + .../src/compile_delta/main/driver/commit.rs | 43 + .../main/driver/delta_enabled.rs | 9 + .../compile_delta/main/driver/env_enabled.rs | 8 + .../src/compile_delta/main/driver/mod.rs | 5 + .../src/compile_delta/main/driver/run.rs | 57 ++ .../src/compile_delta/main/driver/trace.rs | 13 + .../main/lint_hooks/changed_files.rs | 28 + .../main/lint_hooks/file_keys.rs | 50 ++ .../src/compile_delta/main/lint_hooks/mod.rs | 3 + .../main/lint_hooks/model_identities.rs | 39 + .../src/compile_delta/main/mod.rs | 6 + .../report_hooks/complete_source_texts.rs | 40 + .../compile_delta/main/report_hooks/mod.rs | 4 + .../main/report_hooks/record_compile_facts.rs | 14 + .../main/report_hooks/record_test_chains.rs | 10 + .../main/report_hooks/text_model_lines.rs | 39 + .../src/compile_delta/main/rules_rows/mod.rs | 1 + .../main/rules_rows/whole_project_request.rs} | 57 +- .../src/compile_delta/main/splice/mod.rs | 1 + .../main/verify/compiled_digests.rs | 27 + .../compile_delta/main/verify/differences.rs | 74 ++ .../src/compile_delta/main/verify/disarm.rs | 11 + .../main/verify/full_compile_finished.rs | 36 + .../main/verify/hold_expected.rs | 12 + .../src/compile_delta/main/verify/mod.rs | 6 + .../main/verify/write_mismatch.rs | 11 + .../src/compile_delta/mod.rs | 9 + .../src/compile_delta/models.rs | 421 ++++++++++ .../src/compile_delta/tests.rs | 12 + .../src/compile_delta/tests/helpers.rs | 263 +++++++ .../src/compile_delta}/tests/test_coverage.rs | 6 +- .../compile_delta}/tests/test_delta_facts.rs | 6 +- .../src/compile_delta}/tests/test_image.rs | 7 +- .../src/compile_delta}/tests/test_splice.rs | 7 +- .../src/compile_delta/tests/test_types.rs | 58 ++ .../_helpers/assembly}/assembly_analysis.rs | 17 +- .../_helpers/assembly}/assembly_checks.rs | 0 .../_helpers/assembly}/assembly_facts.rs | 11 +- .../_helpers/assembly}/assembly_models.rs | 18 +- .../_helpers/assembly}/assembly_project.rs | 34 +- .../_helpers/assembly}/assembly_sql_tests.rs | 4 +- .../compile_driver/_helpers/assembly/mod.rs | 6 + .../src/compile_driver/_helpers/mod.rs | 3 + .../compile_driver/_helpers/setup}/inputs.rs | 104 +-- .../_helpers/setup}/integration_loaders.rs | 0 .../src/compile_driver/_helpers/setup/mod.rs | 9 + .../compile_driver/_helpers/setup/pipeline.rs | 71 ++ .../_helpers/setup/pipeline_entries.rs | 86 ++ .../compile_driver/_helpers/setup}/project.rs | 148 +--- .../_helpers/setup}/python_plan_entries.rs | 136 +--- .../_helpers/setup}/relation_targets.rs | 87 +-- .../_helpers/setup}/target_defaults.rs | 0 .../compile_driver/_helpers/setup}/targets.rs | 235 +----- .../_helpers/stages}/analysis.rs | 37 +- .../_helpers/stages}/analysis_cache.rs | 40 +- .../_helpers/stages}/assembly.rs | 13 +- .../_helpers/stages}/command.rs | 175 +---- .../_helpers/stages}/deferral.rs | 168 +--- .../compile_driver/_helpers/stages}/diff.rs | 74 +- .../_helpers/stages}/discovery.rs | 72 +- .../src/compile_driver/_helpers/stages/mod.rs | 9 + .../compile_driver/_helpers/stages}/phases.rs | 26 +- .../_helpers/stages/selection_phases.rs | 87 +++ .../src/compile_driver/main/mod.rs | 2 + .../main/setup/attached_inputs.rs | 38 + .../main/setup/build_compiled_project.rs | 37 + .../main/setup/build_project_graph.rs | 37 + ...ild_project_graph_from_compiled_project.rs | 23 + .../setup/build_project_relation_targets.rs | 41 + .../setup/build_python_relation_targets.rs | 42 + .../setup/build_python_run_plan_outputs.rs | 89 +++ .../setup/build_static_pipeline_result.rs | 38 + .../main/setup/compile_adapter_context.rs | 39 + .../main/setup/compile_project_phase.rs | 34 + .../main/setup/compiled_phase.rs | 104 +++ .../main/setup/expanded_loader_namespace.rs | 26 + .../main/setup/managed_write_schema.rs | 40 + .../src/compile_driver/main/setup/mod.rs | 29 + .../main/setup/plan_has_executable_work.rs | 33 + .../main/setup/preserved_loader_schemas.rs | 51 ++ .../src/compile_driver/main/setup/progress.rs | 11 + .../main/setup/project_graph.rs | 16 + .../main/setup/project_target_settings.rs | 26 + .../setup/python_read_source_names_for.rs | 48 ++ .../main/setup/register_inputs.rs | 10 + .../main/setup/register_pipeline.rs | 11 + .../main/setup/register_pipeline_entries.rs | 24 + .../main/setup/register_project.rs | 17 + .../setup/register_python_plan_entries.rs | 14 + .../main/setup/register_relation_targets.rs | 12 + .../main/setup/register_targets.rs | 22 + .../main/setup/run_compile_only_pipeline.rs | 58 ++ .../main/setup/selected_target_settings.rs | 31 + .../main/setup/timing_tracker.rs | 9 + .../main/stages/analyze_compile_project.rs | 16 + .../main/stages/apply_unit_test_selectors.rs | 15 + .../main/stages/build_deferred_locations.rs | 43 + .../main/stages/discover_project_inputs.rs | 57 ++ .../compile_driver/main/stages/elapsed_ms.rs | 9 + .../main/stages/gather_deferred_relations.rs | 27 + .../src/compile_driver/main/stages/mod.rs | 25 + .../compile_driver/main/stages/monotonic.rs | 9 + .../stages/record_analysis_cache_metrics.rs | 23 + .../main/stages/register_analysis.rs | 10 + .../main/stages/register_analysis_cache.rs | 10 + .../main/stages/register_assembly.rs | 12 + .../main/stages/register_command.rs | 12 + .../main/stages/register_deferral.rs | 16 + .../main/stages/register_diff.rs | 12 + .../main/stages/register_discovery.rs | 12 + .../main/stages/register_selection_phases.rs | 15 + .../resolve_configured_rule_selection.rs | 38 + .../stages/resolve_deferred_target_config.rs | 30 + .../stages/resolve_model_deferral_inputs.rs | 72 ++ .../main/stages/resolved_request.rs | 15 + .../main/stages/run_compile_command.rs | 41 + .../main/stages/run_diff_pipeline.rs | 67 ++ .../main/stages/store_and_print.rs | 50 ++ .../stages/unit_test_selectors_applied.rs | 47 ++ .../main/stages/unrecorded_compile.rs | 34 + .../src/compile_driver/mod.rs | 6 + .../src/compile_driver/models.rs | 115 +++ .../src/compile_driver/types.rs | 8 + .../src/compile_outputs/_helpers/artifacts.rs | 132 ++++ .../src/compile_outputs/_helpers/mod.rs | 3 + .../src/compile_outputs/_helpers/reuse.rs | 99 +++ .../_helpers/runtime_target.rs | 123 +++ .../compile_outputs/main/artifact_digest.rs | 8 + .../main/check_compile_reuse.rs | 42 + .../main/compiled_sql_test_path.rs | 17 + .../src/compile_outputs/main/mod.rs | 13 + .../main/planned_sql_test_path.rs | 19 + .../main/publish_staged_artifacts.rs | 37 + .../compile_outputs/main/register_reuse.rs | 12 + .../main/register_runtime_target.rs | 10 + .../main/remove_stale_artifacts.rs | 28 + .../main/staged_artifact_files.rs | 15 + .../main/write_compiled_artifacts.rs | 23 + .../main/write_python_check_runtime_target.rs | 58 ++ .../main/write_runtime_target.rs | 58 ++ .../main/write_test_runtime_target.rs | 47 ++ .../src/compile_outputs/mod.rs | 6 + .../src/compile_outputs/models.rs | 354 +++++++++ .../src/compile_outputs/types.rs | 4 + .../src/dag_artifact/_helpers/artifact.rs | 23 + .../src/dag_artifact/_helpers/mod.rs | 4 + .../src/dag_artifact/_helpers}/projection.rs | 22 +- .../src/dag_artifact/_helpers}/rows.rs | 0 .../src/dag_artifact/_helpers}/selection.rs | 13 +- .../src/dag_artifact/main/build_dag_json.rs | 37 + .../src/dag_artifact/main/mod.rs | 4 + .../dag_artifact/main/register_artifact.rs | 10 + .../dag_artifact/main/register_projection.rs | 17 + .../dag_artifact/main/register_selection.rs | 11 + .../src/dag_artifact/mod.rs | 4 + .../src/frontier/_helpers/cache_directory.rs | 13 + .../frontier/_helpers}/compile_frontier.rs | 42 +- .../src/frontier/_helpers/mod.rs | 3 + .../src/frontier/_helpers}/stage_capture.rs | 54 +- .../src/frontier/constants.rs | 13 + .../src/frontier/main/bridged.rs | 14 + .../src/frontier/main/cache_directory.rs | 16 + .../src/frontier/main/mod.rs | 7 + .../src/frontier/main/produced.rs | 15 + .../frontier/main/register_cache_directory.rs | 10 + .../main/register_compile_frontier.rs | 10 + .../frontier/main/register_stage_capture.rs | 20 + .../src/frontier/main/write_plan_capture.rs | 27 + .../src/frontier/mod.rs | 6 + .../src/frontier/models.rs | 9 + .../hand_built/_helpers}/compiled_model.rs | 12 +- .../src/hand_built/_helpers}/mod.rs | 0 .../src/hand_built/main/compiled_model.rs | 16 + .../src/hand_built/main/mod.rs | 1 + .../src/hand_built/mod.rs | 4 + .../src/input_loops/_helpers/mod.rs | 3 + .../src/input_loops/_helpers/model/mod.rs | 9 + .../input_loops/_helpers/model/model_loop.rs | 352 +++++++++ .../_helpers/model}/model_loop_calls.rs | 3 +- .../_helpers/model}/model_loop_config.rs | 3 +- .../_helpers/model}/model_loop_contracts.rs | 4 +- .../_helpers/model}/model_loop_external.rs | 3 +- .../_helpers/model}/model_loop_header.rs | 21 +- .../_helpers/model}/model_loop_hooks.rs | 33 +- .../_helpers/model}/model_loop_host.rs | 8 +- .../_helpers/model}/model_loop_inputs.rs | 0 .../input_loops/_helpers/model_loop/mod.rs | 7 + .../_helpers/model_loop}/model_loop_macros.rs | 8 +- .../_helpers/model_loop}/model_loop_named.rs | 2 +- .../model_loop}/model_loop_opt_out.rs | 20 +- .../model_loop}/model_loop_prepass.rs | 2 +- .../model_loop}/model_loop_python_hooks.rs | 8 +- .../_helpers/model_loop}/model_loop_schema.rs | 5 +- .../model_loop}/model_loop_visibility.rs | 3 +- .../_helpers/resources}/attachment.rs | 113 +-- .../_helpers/resources}/audit_loop.rs | 33 +- .../_helpers/resources}/expanded_offsets.rs | 10 +- .../_helpers/resources}/function_loop.rs | 19 +- .../_helpers/resources}/loop_classes.rs | 129 +-- .../src/input_loops/_helpers/resources/mod.rs | 10 + .../resources}/resource_attachment.rs | 22 +- .../resources}/resource_diagnostics.rs | 0 .../_helpers/resources}/scenario_loop.rs | 19 +- .../_helpers/resources}/source_loop.rs | 17 +- .../_helpers/resources}/sql_test_loop.rs | 19 +- .../src/input_loops/constants.rs | 17 + .../src/input_loops/main/attach_checks.rs | 97 +++ .../src/input_loops/main/attach_resources.rs | 17 + .../src/input_loops/main/mod.rs | 8 + .../input_loops/main/model_loop_classes.rs | 120 +++ .../src/input_loops/main/pooled.rs | 9 + .../input_loops/main/register_attachment.rs | 12 + .../main/register_expanded_offsets.rs | 10 + .../input_loops/main/register_model_loop.rs | 9 + .../src/input_loops/main/table_backed.rs | 14 + .../src/input_loops/mod.rs | 6 + .../src/input_loops/models.rs} | 580 ++++---------- crates/sqlbuild-python-compile/src/lib.rs | 23 + .../_helpers/graph/column_lineage_graph.rs | 301 +++++++ .../src/lineage/_helpers/graph/mod.rs | 4 + .../_helpers/graph}/project_lineage.rs | 106 +-- .../lineage/_helpers/graph}/project_schema.rs | 0 .../_helpers/graph}/relation_fingerprint.rs | 9 +- .../_helpers/lineage/lineage_cache.rs | 109 +-- .../lineage/_helpers/lineage/lineage_index.rs | 126 +++ .../_helpers/lineage/lineage_objects.rs | 30 + .../src/lineage/_helpers/lineage/mod.rs | 3 + .../src/lineage/_helpers/mod.rs | 2 + .../main/build_direct_semantic_uses.rs | 49 ++ .../main/build_project_column_lineage.rs | 39 + .../src/lineage/main/graph_index.rs | 56 ++ .../src/lineage/main/index_views.rs | 47 ++ .../src/lineage/main/lineage_fingerprint.rs | 15 + .../src/lineage/main/lineage_key.rs | 13 + .../src/lineage/main/lineage_keys.rs | 10 + .../src/lineage/main/mod.rs | 12 + .../src/lineage/main/read_lineage_cache.rs | 56 ++ .../main/register_column_lineage_graph.rs | 9 + .../lineage/main/register_project_lineage.rs | 13 + .../main/register_relation_fingerprint.rs | 11 + .../src/lineage/main/write_lineage_cache.rs | 53 ++ .../src/lineage/mod.rs | 5 + .../src/lineage/models.rs | 520 +++++++++++++ .../src/manifest/_helpers/builder.rs | 19 + .../src/manifest/_helpers/mod.rs | 2 + .../src/manifest/_helpers}/rows.rs | 0 .../src/manifest/main/build_manifest.rs} | 69 +- .../src/manifest/main/mod.rs | 2 + .../src/manifest/main/register_builder.rs | 12 + .../src/manifest/mod.rs | 5 + .../src/manifest/models.rs | 39 + .../_helpers}/declared_relations.rs | 47 +- .../src/project_assembly/_helpers}/getters.rs | 37 +- .../src/project_assembly/_helpers/mod.rs | 6 + .../_helpers}/native_resources.rs | 38 +- .../project_assembly/_helpers/resources.rs | 47 ++ .../_helpers/syntax_checks.rs | 95 +++ .../_helpers}/syntax_validation.rs | 123 +-- .../src/project_assembly/constants.rs | 32 + .../main/assemble_native_project_resources.rs | 28 + .../main/declared_relations.rs | 13 + .../src/project_assembly/main/mod.rs | 15 + .../main/model_placeholders.rs | 22 + .../main/model_syntax_checks.rs | 21 + .../main/register_native_resources.rs | 11 + .../main/register_resources.rs | 11 + .../main/register_syntax_checks.rs | 15 + .../main/register_syntax_validation.rs | 16 + .../project_assembly/main/sql_entry_label.rs | 14 + .../main/standalone_pivot_rows.rs | 41 + .../main/syntax_error_message.rs | 20 + .../main/syntax_error_messages.rs | 26 + .../src/project_assembly/main/syntax_valid.rs | 17 + .../main/validate_hook_sql_syntax.rs | 45 ++ .../main/validate_sql_syntax.rs | 39 + .../src/project_assembly/mod.rs | 7 + .../src/project_assembly/models.rs | 92 +++ .../src/project_assembly/types.rs | 6 + .../rules/_helpers/checking}/dsl_headers.rs | 54 +- .../_helpers/checking}/early_builtin_rules.rs | 118 +-- .../src/rules/_helpers/checking/evaluation.rs | 15 + .../src/rules/_helpers/checking}/findings.rs | 0 .../src/rules/_helpers/checking}/getters.rs | 12 +- .../src/rules/_helpers/checking}/lint.rs | 20 +- .../src/rules/_helpers/checking/mod.rs | 10 + .../checking}/native_header_parser.rs | 65 +- .../checking}/project_tree_queries.rs | 33 +- .../src/rules/_helpers/checking/results.rs | 29 + .../src/rules/_helpers/checking}/rows.rs | 21 +- .../configuration}/builtin_catalogue.rs | 0 .../_helpers/configuration}/catalogue.rs | 41 +- .../_helpers/configuration}/config_json.rs | 0 .../rules/_helpers/configuration}/guidance.rs | 99 +-- .../src/rules/_helpers/configuration/mod.rs | 7 + .../_helpers/configuration}/rules_config.rs | 87 +-- .../_helpers/configuration}/selection.rs | 16 +- .../rules/_helpers/configuration}/skills.rs | 2 +- .../_helpers/custom}/custom_evaluation.rs | 97 +-- .../_helpers/custom}/custom_host_columns.rs | 2 +- .../custom}/custom_host_observation.rs | 11 +- .../_helpers/custom}/custom_host_payload.rs | 10 +- .../rules/_helpers/custom}/custom_hosts.rs | 16 +- .../_helpers/custom}/custom_rule_cache.rs | 12 +- .../_helpers/custom}/custom_rule_evidence.rs | 27 +- .../src/rules/_helpers/custom/mod.rs | 7 + .../src/rules/_helpers/fact}/fact_encoder.rs | 16 +- .../src/rules/_helpers/fact}/fact_replay.rs | 21 +- .../src/rules/_helpers/fact}/fact_tree.rs | 4 +- .../src/rules/_helpers/fact/mod.rs | 3 + .../src/rules/_helpers/fix}/fix_facts.rs | 2 +- .../src/rules/_helpers/fix}/fix_session.rs | 111 +-- .../src/rules/_helpers/fix}/fix_verdict.rs | 0 .../src/rules/_helpers/fix/mod.rs | 3 + .../src/rules/_helpers/lint/early_sql_lint.rs | 231 ++++++ .../rules/_helpers/lint}/lint_expansion.rs | 19 +- .../src/rules/_helpers/lint}/lint_headers.rs | 0 .../src/rules/_helpers/lint}/lint_inputs.rs | 8 +- .../src/rules/_helpers/lint}/lint_pipeline.rs | 256 +----- .../rules/_helpers/lint}/lint_preparation.rs | 7 +- .../rules/_helpers/lint}/lint_sql_bodies.rs | 16 +- .../src/rules/_helpers/lint}/lint_text.rs | 98 +-- .../src/rules/_helpers/lint/mod.rs | 9 + .../src/rules/_helpers/lint}/run_lint.rs | 35 +- .../src/rules/_helpers/mod.rs | 8 + .../src/rules/_helpers/rule/mod.rs | 5 + .../src/rules/_helpers/rule}/rule_api.rs | 20 +- .../src/rules/_helpers/rule}/rule_cache.rs | 21 +- .../src/rules/_helpers/rule}/rule_dispatch.rs | 74 +- .../src/rules/_helpers/rule}/rule_plan.rs | 26 +- .../src/rules/_helpers/rule}/rule_run.rs | 31 +- .../rules/_helpers/sql}/fixture_schemas.rs | 0 .../src/rules/_helpers/sql/mod.rs | 4 + .../src/rules/_helpers/sql}/sql_document.rs | 8 +- .../src/rules/_helpers/sql}/sql_rules.rs | 72 +- .../rules/_helpers/sql}/sql_test_objects.rs | 0 .../src}/rules/constants.rs | 29 +- .../main/checking/format_rules_result.rs | 25 + .../format_skipped_type_proof_rules.rs | 11 + .../rules/main/checking/header_violations.rs | 14 + .../src/rules/main/checking/mod.rs | 11 + .../main/checking/register_dsl_headers.rs | 26 + .../main/checking/register_evaluation.rs | 11 + .../src/rules/main/checking/register_lint.rs | 16 + .../checking/register_project_tree_queries.rs | 19 + .../rules/main/checking/register_results.rs | 13 + .../src/rules/main/checking/rules_config.rs | 13 + .../src/rules/main/checking/rules_progress.rs | 26 + .../rules/main/checking/start_early_rules.rs | 44 ++ .../configuration/build_rule_catalogue.rs | 20 + .../rules/main/configuration/format_rule.rs | 20 + .../configuration/install_rules_skills.rs | 54 ++ .../main/configuration/load_rules_config.rs | 13 + .../load_rules_config_with_cache.rs | 22 + .../src/rules/main/configuration/mod.rs | 10 + .../main/configuration/register_catalogue.rs | 15 + .../main/configuration/register_guidance.rs | 13 + .../configuration/register_rules_config.rs | 11 + .../main/configuration/register_selection.rs | 12 + .../main/configuration/rules_config_of.rs | 26 + .../src/rules/main/custom/mod.rs | 5 + .../main/custom/register_custom_evaluation.rs | 20 + .../register_custom_host_observation.rs | 11 + .../main/custom/register_custom_rule_cache.rs | 11 + .../custom/register_custom_rule_evidence.rs | 25 + .../main/custom/stored_custom_rule_refusal.rs | 50 ++ .../src/rules/main/fact/mod.rs | 1 + .../rules/main/fact/register_fact_encoder.rs | 12 + .../rules/main/fix/compile_facts_function.rs | 10 + .../rules/main/fix/fix_verdict_function.rs | 10 + .../src/rules/main/fix/mod.rs | 4 + .../src/rules/main/fix/plan_rule_fixes.rs | 62 ++ .../main/fix/plan_rule_fixes_function.rs | 10 + .../main/lint/lint_file_role_function.rs | 10 + .../src/rules/main/lint/mod.rs | 10 + .../main/lint/prepare_lint_body_function.rs | 10 + .../rules/main/lint/project_relative_path.rs | 13 + .../main/lint/register_early_sql_lint.rs | 12 + .../main/lint/register_lint_expansion.rs | 10 + .../main/lint/register_lint_sql_bodies.rs | 10 + .../src/rules/main/lint/register_lint_text.rs | 18 + .../src/rules/main/lint/register_run_lint.rs | 13 + .../src/rules/main/lint/relative_parts.rs | 17 + .../src/rules/main/lint/unsafe_fault.rs | 54 ++ .../src/rules/main/mod.rs | 8 + .../src/rules/main/rule/evaluate_rules.rs | 17 + .../src/rules/main/rule/mod.rs | 7 + .../src/rules/main/rule/register_rule_api.rs | 17 + .../rules/main/rule/register_rule_cache.rs | 15 + .../rules/main/rule/register_rule_dispatch.rs | 16 + .../src/rules/main/rule/register_rule_plan.rs | 11 + .../src/rules/main/rule/register_rule_run.rs | 15 + .../src/rules/main/rule/run_rules.rs | 15 + .../sql/format_redundant_fixture_nulls.rs} | 24 +- .../src/rules/main/sql/mod.rs | 4 + .../main/sql/register_fixture_formatting.rs | 10 + .../rules/main/sql/register_sql_document.rs | 12 + .../src/rules/main/sql/register_sql_rules.rs | 11 + .../sqlbuild-python-compile/src/rules/mod.rs | 6 + .../src/rules/models.rs | 648 ++++++++++++++++ .../src/scopes/_helpers/mod.rs | 1 + .../src/scopes/_helpers}/scope_loading.rs | 76 +- .../scopes/main/load_or_build_scope_index.rs | 54 ++ .../src/scopes/main/mod.rs | 2 + .../src/scopes/main/register_scope_loading.rs | 10 + .../sqlbuild-python-compile/src/scopes/mod.rs | 4 + .../src/semantic_checks/_helpers/mod.rs | 2 + .../_helpers/recovery}/completion.rs | 20 +- .../_helpers/recovery}/failures.rs | 0 .../_helpers/recovery/help_catalogue.rs | 10 + .../semantic_checks/_helpers/recovery/mod.rs | 5 + .../_helpers/recovery}/resource_sql.rs | 10 +- .../_helpers/recovery}/type_recovery.rs | 14 +- .../_helpers/session}/inputs.rs | 2 +- .../_helpers/session}/metadata.rs | 12 +- .../semantic_checks/_helpers/session/mod.rs | 5 + .../_helpers/session}/session_facts.rs | 0 .../_helpers/session}/stage.rs | 82 +- .../_helpers/session}/stage_objects.rs | 0 .../src}/semantic_checks/constants.rs | 2 +- .../main/completed_semantic_project.rs | 60 ++ .../src/semantic_checks/main/mod.rs | 4 + .../main/register_help_catalogue.rs | 10 + .../main/register_resource_sql.rs | 10 + .../semantic_checks/main/register_stage.rs | 12 + .../src/semantic_checks/mod.rs | 6 + .../src/semantic_checks/models.rs | 14 + .../src/sql_test_glue/_helpers}/assembly.rs | 0 .../sql_test_glue/_helpers}/compiled_tests.rs | 54 +- .../src/sql_test_glue/_helpers/mod.rs | 2 + .../main/assemble_native_sql_tests.rs | 46 ++ .../src/sql_test_glue/main/mod.rs | 2 + .../main/register_compiled_tests.rs | 11 + .../src/sql_test_glue/mod.rs | 4 + crates/sqlbuild-python/Cargo.toml | 6 +- .../_helpers/analysis/binding_locations.rs | 73 -- .../src/bindings/_helpers/analysis/queries.rs | 39 - .../bindings/_helpers/analysis/sql_tests.rs | 20 - .../bindings/_helpers/analysis_session/mod.rs | 4 - .../_helpers/analysis_session/registration.rs | 7 +- .../_helpers/audit_command/command.rs | 2 +- .../src/bindings/_helpers/auditing/mod.rs | 3 - .../_helpers/auditing/registration.rs | 9 +- .../_helpers/build_command/command.rs | 5 +- .../_helpers/build_command/completion.rs | 7 +- .../_helpers/build_command/defer_clone.rs | 2 +- .../_helpers/build_command/execution.rs | 4 +- .../_helpers/build_command/invocation.rs | 2 +- .../_helpers/build_command/lifecycle.rs | 9 +- .../_helpers/build_command/planning.rs | 4 +- .../_helpers/change_selection/base_compile.rs | 4 +- .../change_selection/base_snapshot.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 13 +- .../src/bindings/_helpers/check/check_core.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 6 +- .../bindings/_helpers/clone/clone_command.rs | 2 +- .../bindings/_helpers/clone/clone_pipeline.rs | 2 +- .../_helpers/command_runtime/invocation.rs | 2 +- .../_helpers/compile_attachment/mod.rs | 6 - .../compile_attachment/registration.rs | 37 +- .../compile_command/artifact_batch.rs | 153 ---- .../bindings/_helpers/compile_command/mod.rs | 13 - .../_helpers/compile_command/models.rs | 51 -- .../_helpers/compile_command/registration.rs | 27 +- .../compile_command/reuse_recording.rs | 646 --------------- .../bindings/_helpers/compile_delta/image.rs | 734 ------------------ .../_helpers/compile_delta/lint_hooks.rs | 107 --- .../bindings/_helpers/compile_delta/verify.rs | 238 ------ .../bindings/_helpers/compile_driver/mod.rs | 24 - .../_helpers/compile_driver/pipeline.rs | 211 ----- .../compile_driver/pipeline_entries.rs | 231 ------ .../_helpers/compile_driver/plan_pipeline.rs | 28 +- .../_helpers/compile_driver/plan_result.rs | 15 +- .../_helpers/compile_driver/registration.rs | 47 +- .../compile_driver/selection_phases.rs | 172 ---- .../_helpers/compile_driver/static_context.rs | 11 +- .../_helpers/compile_outputs/artifacts.rs | 285 ------- .../bindings/_helpers/compile_outputs/mod.rs | 3 - .../_helpers/compile_outputs/registration.rs | 7 +- .../_helpers/compile_outputs/reuse.rs | 360 --------- .../compile_outputs/runtime_target.rs | 281 ------- .../_helpers/contract_command/command.rs | 6 +- .../_helpers/contract_command/write.rs | 4 +- .../bindings/_helpers/cost_command/refresh.rs | 2 +- .../_helpers/dag_artifact/artifact.rs | 59 -- .../src/bindings/_helpers/dag_artifact/mod.rs | 4 - .../_helpers/dag_artifact/registration.rs | 10 +- .../bindings/_helpers/dbt_interop/pipeline.rs | 4 +- .../_helpers/dbt_interop/pipeline_prologue.rs | 4 +- .../_helpers/dbt_interop/plan_output.rs | 2 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 2 +- .../_helpers/debug_command/run_debug.rs | 2 +- .../bindings/_helpers/diff_report/command.rs | 4 +- .../bindings/_helpers/diff_report/query.rs | 2 +- .../_helpers/exported_constants/constants.rs | 15 +- .../bindings/_helpers/format_command/cache.rs | 8 +- .../_helpers/format_command/command.rs | 2 +- .../_helpers/format_command/config.rs | 2 +- .../bindings/_helpers/format_command/run.rs | 15 +- .../_helpers/format_command/selection.rs | 4 +- .../_helpers/format_command/stages.rs | 4 +- .../_helpers/freshness/freshness_command.rs | 4 +- .../_helpers/frontier/cache_directory.rs | 28 - .../bindings/_helpers/frontier/constants.rs | 9 - .../src/bindings/_helpers/frontier/mod.rs | 4 - .../_helpers/frontier/registration.rs | 10 +- .../src/bindings/_helpers/input_loops/mod.rs | 26 - .../_helpers/input_loops/registration.rs | 11 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/janitor/janitor_phases.rs | 2 +- .../_helpers/lineage/column_lineage_graph.rs | 620 --------------- .../_helpers/lineage/lineage_command.rs | 16 +- .../_helpers/lineage/lineage_index.rs | 222 ------ .../_helpers/lineage/lineage_objects.rs | 177 ----- .../src/bindings/_helpers/lineage/mod.rs | 7 - .../bindings/_helpers/lineage/registration.rs | 13 +- .../bindings/_helpers/load/load_command.rs | 7 +- .../bindings/_helpers/load/load_selection.rs | 11 +- .../_helpers/location_command/command.rs | 2 +- .../location_command/snapshot_lineage.rs | 4 +- .../src/bindings/_helpers/manifest/mod.rs | 2 - .../_helpers/manifest/registration.rs | 4 +- .../src/bindings/_helpers/mod.rs | 3 - .../bindings/_helpers/plan_command/plan.rs | 2 +- .../bindings/_helpers/plan_command/rules.rs | 18 +- .../bindings/_helpers/project_assembly/mod.rs | 7 - .../_helpers/project_assembly/models.rs | 44 -- .../_helpers/project_assembly/registration.rs | 22 +- .../_helpers/project_assembly/resources.rs | 99 --- .../project_assembly/syntax_checks.rs | 161 ---- .../bindings/_helpers/query_command/dag.rs | 4 +- .../_helpers/query_command/project.rs | 2 +- .../_helpers/record_behaviours/constants.rs | 14 +- .../_helpers/refactor/offline_compile.rs | 5 +- .../_helpers/refactor/refactor_command.rs | 2 +- .../_helpers/review_command/comparison.rs | 2 +- .../_helpers/review_command/impact.rs | 4 +- .../_helpers/review_command/planning.rs | 4 +- .../bindings/_helpers/rules/early_sql_lint.rs | 461 ----------- .../src/bindings/_helpers/rules/evaluation.rs | 20 - .../src/bindings/_helpers/rules/mod.rs | 50 -- .../bindings/_helpers/rules/registration.rs | 113 +-- .../src/bindings/_helpers/rules/results.rs | 87 --- .../_helpers/scenario_command/setup.rs | 4 +- .../src/bindings/_helpers/scopes/mod.rs | 1 - .../bindings/_helpers/scopes/registration.rs | 5 +- .../bindings/_helpers/scopes/scope_command.rs | 2 +- .../bindings/_helpers/seed_command/command.rs | 2 +- .../semantic_checks/help_catalogue.rs | 15 - .../bindings/_helpers/semantic_checks/mod.rs | 11 - .../_helpers/semantic_checks/registration.rs | 10 +- .../_helpers/sql_test_command/python_tests.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 4 +- .../bindings/_helpers/sql_test_glue/mod.rs | 2 - .../_helpers/sql_test_glue/registration.rs | 4 +- .../src/bindings/main/register.rs | 23 +- crates/sqlbuild-python/src/bindings/tests.rs | 8 - .../src/bindings/tests/helpers.rs | 258 ------ .../src/bindings/tests/test_types.rs | 59 -- fensu.toml | 631 +++++++++------ scripts/compiler_differential/constants.py | 4 +- scripts/fensu_policy/constants.py | 1 + scripts/native_module_paths/constants.py | 1 + tests/native_field_fallbacks.txt | 118 +-- 684 files changed, 14883 insertions(+), 11653 deletions(-) create mode 100644 crates/sqlbuild-python-compile/Cargo.toml create mode 100644 crates/sqlbuild-python-compile/src/analysis/_helpers/binding_locations.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-compile/src/analysis/_helpers}/getters.rs (74%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-compile/src/analysis/_helpers}/mod.rs (81%) create mode 100644 crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-compile/src/analysis/_helpers}/required_columns.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis => sqlbuild-python-compile/src/analysis/_helpers}/sql_test_comparisons.rs (84%) create mode 100644 crates/sqlbuild-python-compile/src/analysis/_helpers/sql_tests.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/locate_model_binding_diagnostics.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/register_queries.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/register_sql_test_comparisons.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/register_sql_tests.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/main/single_comparison_sql.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/models.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-compile/src/analysis_session/_helpers}/getters.rs (67%) create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-compile/src/analysis_session/_helpers}/model_analyses.rs (83%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-compile/src/analysis_session/_helpers}/oracle.rs (82%) rename crates/{sqlbuild-python/src/bindings/_helpers/analysis_session => sqlbuild-python-compile/src/analysis_session/_helpers}/outcome_objects.rs (93%) create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/analyze_models.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/dynamic_contract_proof.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/register_oracle.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/register_outcome_objects.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/main/session_outcome_analysis.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/analysis_session/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/auditing => sqlbuild-python-compile/src/auditing/_helpers}/identity.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/auditing => sqlbuild-python-compile/src/auditing/_helpers}/measurement.rs (78%) create mode 100644 crates/sqlbuild-python-compile/src/auditing/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/auditing => sqlbuild-python-compile/src/auditing/_helpers}/schema_audits.rs (88%) create mode 100644 crates/sqlbuild-python-compile/src/auditing/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/auditing/main/register_identity.rs create mode 100644 crates/sqlbuild-python-compile/src/auditing/main/register_measurement.rs create mode 100644 crates/sqlbuild-python-compile/src/auditing/main/register_schema_audits.rs create mode 100644 crates/sqlbuild-python-compile/src/auditing/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/declaration_scope.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/model_config.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/model_inputs.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/sql_expansion_context.rs (86%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/storage_policies.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_attachment => sqlbuild-python-compile/src/compile_attachment/_helpers}/target.rs (71%) create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_compile_target_context_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/build_sql_expansion_context_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/connection_is_default_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/effective_target_namespace.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/native_model_config_session_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/present.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/project_location.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/rebind_declaration_scope_function.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/resolved_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/main/sql_expansion_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_attachment/models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/artifact_batch.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/artifacts}/prepared_artifacts.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/artifacts}/surrogates.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/artifacts}/target_writer.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/artifacts}/test_artifact_cache.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/invocation}/getters.rs (68%) create mode 100644 crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/invocation}/report.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/invocation}/reuse_attempt.rs (81%) create mode 100644 crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/invocation}/runtime_identity.rs (80%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_command => sqlbuild-python-compile/src/compile_command/_helpers/invocation}/startup_inputs.rs (86%) create mode 100644 crates/sqlbuild-python-compile/src/compile_command/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-compile/src}/compile_command/constants.rs (64%) create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/planning_diagnostics.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_prepared_artifacts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_target_writer.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/remove_failed_compile.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_compile_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_rule_failed_models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_static_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/diagnostic_item.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_error_json.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_json_with_notice.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_text.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/interpreter_values.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/new_artifact_recorder.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/print_to.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_attempt.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_recording.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_runtime_identity.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_startup_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/reuse_debug.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_command/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_delta => sqlbuild-python-compile/src/compile_delta/_helpers}/driver.rs (88%) create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/_helpers/image.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_delta => sqlbuild-python-compile/src/compile_delta/_helpers}/mod.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_delta => sqlbuild-python-compile/src/compile_delta/_helpers}/report_hooks.rs (57%) create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/_helpers/rules_rows.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_delta => sqlbuild-python-compile/src/compile_delta/_helpers}/splice.rs (96%) create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/_helpers/verify.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/commit.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/delta_enabled.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/env_enabled.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/run.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/driver/trace.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/changed_files.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/file_keys.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/model_identities.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/complete_source_texts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_compile_facts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_test_chains.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/text_model_lines.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs => sqlbuild-python-compile/src/compile_delta/main/rules_rows/whole_project_request.rs} (51%) create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/splice/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/compiled_digests.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/differences.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/disarm.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/full_compile_finished.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/hold_expected.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/main/verify/write_mismatch.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/tests.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/tests/helpers.rs rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-compile/src/compile_delta}/tests/test_coverage.rs (95%) rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-compile/src/compile_delta}/tests/test_delta_facts.rs (93%) rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-compile/src/compile_delta}/tests/test_image.rs (93%) rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-compile/src/compile_delta}/tests/test_splice.rs (96%) create mode 100644 crates/sqlbuild-python-compile/src/compile_delta/tests/test_types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_analysis.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_checks.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_facts.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_models.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_project.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/assembly}/assembly_sql_tests.rs (97%) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/inputs.rs (82%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/integration_loaders.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/project.rs (73%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/python_plan_entries.rs (66%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/relation_targets.rs (63%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/target_defaults.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/setup}/targets.rs (74%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/analysis.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/analysis_cache.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/assembly.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/command.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/deferral.rs (58%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/diff.rs (72%) rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/discovery.rs (88%) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/stages}/phases.rs (92%) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/attached_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_compiled_project.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph_from_compiled_project.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_relation_targets.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_relation_targets.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_run_plan_outputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_static_pipeline_result.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_adapter_context.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_project_phase.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/compiled_phase.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/expanded_loader_namespace.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/managed_write_schema.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/plan_has_executable_work.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/progress.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_graph.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_target_settings.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline_entries.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_relation_targets.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/run_compile_only_pipeline.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/selected_target_settings.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/timing_tracker.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/analyze_compile_project.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/apply_unit_test_selectors.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/build_deferred_locations.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/discover_project_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/elapsed_ms.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/gather_deferred_relations.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/monotonic.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/record_analysis_cache_metrics.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_assembly.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_command.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_diff.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_discovery.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_selection_phases.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_configured_rule_selection.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_deferred_target_config.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolved_request.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_compile_command.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_diff_pipeline.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/store_and_print.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/unit_test_selectors_applied.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/stages/unrecorded_compile.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/types.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/_helpers/artifacts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/_helpers/reuse.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/_helpers/runtime_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/artifact_digest.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/check_compile_reuse.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/compiled_sql_test_path.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/planned_sql_test_path.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/publish_staged_artifacts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/register_reuse.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/register_runtime_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/remove_stale_artifacts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/staged_artifact_files.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/write_compiled_artifacts.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/write_python_check_runtime_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/write_runtime_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/main/write_test_runtime_target.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/models.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_outputs/types.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/_helpers/artifact.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/dag_artifact => sqlbuild-python-compile/src/dag_artifact/_helpers}/projection.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/dag_artifact => sqlbuild-python-compile/src/dag_artifact/_helpers}/rows.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/dag_artifact => sqlbuild-python-compile/src/dag_artifact/_helpers}/selection.rs (95%) create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/main/build_dag_json.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/main/register_artifact.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/main/register_projection.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/main/register_selection.rs create mode 100644 crates/sqlbuild-python-compile/src/dag_artifact/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/_helpers/cache_directory.rs rename crates/{sqlbuild-python/src/bindings/_helpers/frontier => sqlbuild-python-compile/src/frontier/_helpers}/compile_frontier.rs (59%) create mode 100644 crates/sqlbuild-python-compile/src/frontier/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/frontier => sqlbuild-python-compile/src/frontier/_helpers}/stage_capture.rs (67%) create mode 100644 crates/sqlbuild-python-compile/src/frontier/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/bridged.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/cache_directory.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/produced.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/register_cache_directory.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/register_compile_frontier.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/register_stage_capture.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/frontier/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/hand_built => sqlbuild-python-compile/src/hand_built/_helpers}/compiled_model.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/hand_built => sqlbuild-python-compile/src/hand_built/_helpers}/mod.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/hand_built/main/compiled_model.rs create mode 100644 crates/sqlbuild-python-compile/src/hand_built/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/hand_built/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/_helpers/model/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_calls.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_config.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_contracts.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_external.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_header.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_hooks.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_host.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model}/model_loop_inputs.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_macros.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_named.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_opt_out.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_prepass.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_python_hooks.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_schema.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/model_loop}/model_loop_visibility.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/attachment.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/audit_loop.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/expanded_offsets.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/function_loop.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/loop_classes.rs (71%) create mode 100644 crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/resource_attachment.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/resource_diagnostics.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/scenario_loop.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/source_loop.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops => sqlbuild-python-compile/src/input_loops/_helpers/resources}/sql_test_loop.rs (98%) create mode 100644 crates/sqlbuild-python-compile/src/input_loops/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/attach_checks.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/attach_resources.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/model_loop_classes.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/register_attachment.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/register_expanded_offsets.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/register_model_loop.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/table_backed.rs create mode 100644 crates/sqlbuild-python-compile/src/input_loops/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs => sqlbuild-python-compile/src/input_loops/models.rs} (63%) create mode 100644 crates/sqlbuild-python-compile/src/lib.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/graph/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/lineage => sqlbuild-python-compile/src/lineage/_helpers/graph}/project_lineage.rs (80%) rename crates/{sqlbuild-python/src/bindings/_helpers/lineage => sqlbuild-python-compile/src/lineage/_helpers/graph}/project_schema.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/lineage => sqlbuild-python-compile/src/lineage/_helpers/graph}/relation_fingerprint.rs (81%) rename crates/{sqlbuild-python/src/bindings => sqlbuild-python-compile/src/lineage}/_helpers/lineage/lineage_cache.rs (78%) create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_objects.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/build_direct_semantic_uses.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/build_project_column_lineage.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/graph_index.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/index_views.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/lineage_fingerprint.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/lineage_key.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/lineage_keys.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/read_lineage_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/register_column_lineage_graph.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/register_project_lineage.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/register_relation_fingerprint.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/main/write_lineage_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/lineage/models.rs create mode 100644 crates/sqlbuild-python-compile/src/manifest/_helpers/builder.rs create mode 100644 crates/sqlbuild-python-compile/src/manifest/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/manifest => sqlbuild-python-compile/src/manifest/_helpers}/rows.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/manifest/builder.rs => sqlbuild-python-compile/src/manifest/main/build_manifest.rs} (56%) create mode 100644 crates/sqlbuild-python-compile/src/manifest/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/manifest/main/register_builder.rs create mode 100644 crates/sqlbuild-python-compile/src/manifest/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/manifest/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/project_assembly => sqlbuild-python-compile/src/project_assembly/_helpers}/declared_relations.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/project_assembly => sqlbuild-python-compile/src/project_assembly/_helpers}/getters.rs (72%) create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/project_assembly => sqlbuild-python-compile/src/project_assembly/_helpers}/native_resources.rs (92%) create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/_helpers/resources.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs rename crates/{sqlbuild-python/src/bindings/_helpers/project_assembly => sqlbuild-python-compile/src/project_assembly/_helpers}/syntax_validation.rs (59%) create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/constants.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/assemble_native_project_resources.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/declared_relations.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/model_placeholders.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/model_syntax_checks.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/register_native_resources.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/register_resources.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_checks.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/sql_entry_label.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/standalone_pivot_rows.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_message.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/syntax_valid.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/validate_hook_sql_syntax.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/main/validate_sql_syntax.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/models.rs create mode 100644 crates/sqlbuild-python-compile/src/project_assembly/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/dsl_headers.rs (79%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/early_builtin_rules.rs (62%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/checking/evaluation.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/findings.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/getters.rs (58%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/lint.rs (67%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/checking/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/native_header_parser.rs (76%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/project_tree_queries.rs (88%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/checking/results.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/checking}/rows.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/builtin_catalogue.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/catalogue.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/config_json.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/guidance.rs (63%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/configuration/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/rules_config.rs (72%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/selection.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/configuration}/skills.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_evaluation.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_host_columns.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_host_observation.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_host_payload.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_hosts.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_rule_cache.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/custom}/custom_rule_evidence.rs (91%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/custom/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fact}/fact_encoder.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fact}/fact_replay.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fact}/fact_tree.rs (98%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/fact/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fix}/fix_facts.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fix}/fix_session.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/fix}/fix_verdict.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/fix/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/lint/early_sql_lint.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_expansion.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_headers.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_inputs.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_pipeline.rs (81%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_preparation.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_sql_bodies.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/lint_text.rs (60%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/lint/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/lint}/run_lint.rs (85%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/rule/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/rule}/rule_api.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/rule}/rule_cache.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/rule}/rule_dispatch.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/rule}/rule_plan.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/rule}/rule_run.rs (85%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/sql}/fixture_schemas.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/rules/_helpers/sql/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/sql}/sql_document.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/sql}/sql_rules.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/rules => sqlbuild-python-compile/src/rules/_helpers/sql}/sql_test_objects.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-compile/src}/rules/constants.rs (71%) create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/format_rules_result.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/format_skipped_type_proof_rules.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/header_violations.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/register_dsl_headers.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/register_evaluation.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/register_lint.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/register_results.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/rules_config.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/rules_progress.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/checking/start_early_rules.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/build_rule_catalogue.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/format_rule.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/install_rules_skills.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config_with_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/register_catalogue.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/register_guidance.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/register_rules_config.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/register_selection.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/configuration/rules_config_of.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_evaluation.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_host_observation.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/custom/stored_custom_rule_refusal.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fact/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fact/register_fact_encoder.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fix/compile_facts_function.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fix/fix_verdict_function.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fix/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes_function.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/lint_file_role_function.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/prepare_lint_body_function.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/project_relative_path.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/register_early_sql_lint.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_expansion.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_sql_bodies.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_text.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/register_run_lint.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/relative_parts.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/lint/unsafe_fault.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/evaluate_rules.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_api.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_cache.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_plan.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_run.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/rule/run_rules.rs rename crates/{sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs => sqlbuild-python-compile/src/rules/main/sql/format_redundant_fixture_nulls.rs} (84%) create mode 100644 crates/sqlbuild-python-compile/src/rules/main/sql/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/sql/register_fixture_formatting.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_document.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_rules.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/rules/models.rs create mode 100644 crates/sqlbuild-python-compile/src/scopes/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/scopes => sqlbuild-python-compile/src/scopes/_helpers}/scope_loading.rs (65%) create mode 100644 crates/sqlbuild-python-compile/src/scopes/main/load_or_build_scope_index.rs create mode 100644 crates/sqlbuild-python-compile/src/scopes/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/scopes/main/register_scope_loading.rs create mode 100644 crates/sqlbuild-python-compile/src/scopes/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/recovery}/completion.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/recovery}/failures.rs (100%) create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/help_catalogue.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/recovery}/resource_sql.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/recovery}/type_recovery.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/session}/inputs.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/session}/metadata.rs (98%) create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/session}/session_facts.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/session}/stage.rs (52%) rename crates/{sqlbuild-python/src/bindings/_helpers/semantic_checks => sqlbuild-python-compile/src/semantic_checks/_helpers/session}/stage_objects.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-compile/src}/semantic_checks/constants.rs (69%) create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/main/completed_semantic_project.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/main/register_help_catalogue.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/main/register_resource_sql.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/main/register_stage.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/semantic_checks/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-compile/src/sql_test_glue/_helpers}/assembly.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/sql_test_glue => sqlbuild-python-compile/src/sql_test_glue/_helpers}/compiled_tests.rs (85%) create mode 100644 crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/sql_test_glue/main/assemble_native_sql_tests.rs create mode 100644 crates/sqlbuild-python-compile/src/sql_test_glue/main/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/sql_test_glue/main/register_compiled_tests.rs create mode 100644 crates/sqlbuild-python-compile/src/sql_test_glue/mod.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs diff --git a/Cargo.lock b/Cargo.lock index 5de64631ec..ce19433345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4441,11 +4441,9 @@ dependencies = [ "rayon", "rusqlite", "rustc-hash", - "serde", "serde_json", "signal-hook", "sqlbuild-analysis", - "sqlbuild-attachments", "sqlbuild-cache", "sqlbuild-cli", "sqlbuild-config", @@ -4454,20 +4452,18 @@ dependencies = [ "sqlbuild-host", "sqlbuild-model-config", "sqlbuild-orchestration", - "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python-base", + "sqlbuild-python-compile", "sqlbuild-python-front", "sqlbuild-python-planner", "sqlbuild-python-project", "sqlbuild-recall", "sqlbuild-refactor", - "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", "sqlbuild-state", - "sqlclean", ] [[package]] @@ -4494,6 +4490,36 @@ dependencies = [ "sqlbuild-state", ] +[[package]] +name = "sqlbuild-python-compile" +version = "0.0.0" +dependencies = [ + "fensu-policy", + "pyo3", + "rayon", + "serde", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-project", + "sqlbuild-pysource", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-python-project", + "sqlbuild-render", + "sqlbuild-rules", + "sqlbuild-scopes", + "sqlbuild-sqltext", + "sqlclean", +] + [[package]] name = "sqlbuild-python-front" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 1af3b2707f..6386ddc847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/sqlbuild-python-front", "crates/sqlbuild-python-project", "crates/sqlbuild-python-planner", + "crates/sqlbuild-python-compile", "crates/sqlbuild-python", ] resolver = "2" @@ -63,9 +64,10 @@ order = [ "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", + "sqlbuild-python-compile", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -189,6 +191,7 @@ sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" } +sqlbuild-python-compile = { path = "crates/sqlbuild-python-compile" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index 5c7d6d822c..a307c90e7e 100644 --- a/crates/README.md +++ b/crates/README.md @@ -32,14 +32,15 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | | `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | | `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-compile/Cargo.toml b/crates/sqlbuild-python-compile/Cargo.toml new file mode 100644 index 0000000000..b2fb52557f --- /dev/null +++ b/crates/sqlbuild-python-compile/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sqlbuild-python-compile" +description = "The compile layer of the sqlbuild._native bindings: the compile driver and command, input loops, project assembly, analysis and the analysis session, rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier and auditing." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +fensu-policy.workspace = true +pyo3.workspace = true +rayon.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-cli.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-project.workspace = true +sqlbuild-pysource.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true +sqlbuild-render.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-scopes.workspace = true +sqlbuild-sqltext.workspace = true +sqlclean.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python-compile/src/analysis/_helpers/binding_locations.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/binding_locations.rs new file mode 100644 index 0000000000..479d720883 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/binding_locations.rs @@ -0,0 +1,25 @@ +//! A hand-built model's binding diagnostics at their authored locations. + +use crate::analysis::main::locate_model_binding_diagnostics::locate_model_binding_diagnostics; +use crate::analysis::models::LocatedRow; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_analysis::binding_locations::models::LocatedDiagnostic; + +use sqlbuild_python_project::analysis::models::Positional; + +pub(crate) fn located_row(item: LocatedDiagnostic) -> LocatedRow { + LocatedRow { + code: item.code, + message: item.message, + severity: item.severity, + line: item.line, + column: item.column, + span: item.location.map(Positional), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(locate_model_binding_diagnostics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/getters.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs rename to crates/sqlbuild-python-compile/src/analysis/_helpers/getters.rs index 445fe009fb..ef4363d715 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; use sqlbuild_python_project::analysis::models::BindingCatalog; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_binding_catalog, -}]; - -fn read_binding_catalog<'py>( +pub(crate) fn read_binding_catalog<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/mod.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs rename to crates/sqlbuild-python-compile/src/analysis/_helpers/mod.rs index f896a0e29c..2adf907fb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/mod.rs @@ -1,6 +1,6 @@ pub(crate) mod binding_locations; pub(crate) mod getters; pub(crate) mod queries; -mod required_columns; +pub(crate) mod required_columns; pub(crate) mod sql_test_comparisons; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs new file mode 100644 index 0000000000..963d0678cb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs @@ -0,0 +1,31 @@ +//! Validate, fingerprint and analyze SQL queries for the Python compiler. + +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; + +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; + +#[pyfunction] +pub(crate) fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) + }) + .map_err(value_error) +} + +/// A warehouse type name's JSON type tree, None without a parser, `ValueError` on a panic. +#[pyfunction] +pub(crate) fn parse_warehouse_type( + py: Python<'_>, + spelling: &str, + dialect: &str, +) -> PyResult> { + py.compiler_detach(|| { + sqlbuild_analysis::warehouse_types::main::parse_warehouse_type::parse_warehouse_type( + spelling, dialect, + ) + .map(|tree| tree.as_ref().map(serde_json::Value::to_string)) + }) + .map_err(value_error) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/required_columns.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs rename to crates/sqlbuild-python-compile/src/analysis/_helpers/required_columns.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/sql_test_comparisons.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs rename to crates/sqlbuild-python-compile/src/analysis/_helpers/sql_test_comparisons.rs index 47032bc536..a60b899a58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/sql_test_comparisons.rs @@ -2,9 +2,9 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyfunction}; use serde_json::{Value, json}; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql; @@ -22,11 +22,11 @@ const TOP_CLAUSE_DIALECT: &str = "tsql"; /// Operator, dialect and optional probed chain step of one comparison render. pub(crate) struct RenderSettings { - set_difference_operator: String, - sql_analysis_dialect: Option, - probe_step_index: Option, - output_step_index: Option, - actual_relations: HashMap, + pub(crate) set_difference_operator: String, + pub(crate) sql_analysis_dialect: Option, + pub(crate) probe_step_index: Option, + pub(crate) output_step_index: Option, + pub(crate) actual_relations: HashMap, } impl RenderSettings { @@ -59,7 +59,7 @@ fn optional_value FromPyObject<'py>>( /// Comparison SQL of each planned SQL test, or of one chain step's actual-output probe. #[pyfunction] #[pyo3(signature = (test_entries, **options))] -fn render_sql_test_comparison_sql( +pub(crate) fn render_sql_test_comparison_sql( py: Python<'_>, test_entries: Vec>, options: Option<&Bound<'_, PyDict>>, @@ -67,7 +67,7 @@ fn render_sql_test_comparison_sql( rendered_comparisons(py, &test_entries, &RenderSettings::read(options)?) } -fn rendered_comparisons( +pub(crate) fn rendered_comparisons( py: Python<'_>, test_entries: &[Bound<'_, PyAny>], settings: &RenderSettings, @@ -116,26 +116,6 @@ fn rendered_comparisons( .collect() } -/// The comparison SQL of one planned SQL test under `adapter`'s operator and dialect. -pub(crate) fn single_comparison_sql( - entry: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, -) -> PyResult { - let settings = RenderSettings { - set_difference_operator: adapter - .call_method0("render_set_difference_operator")? - .extract()?, - sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, - probe_step_index: None, - output_step_index: None, - actual_relations: HashMap::new(), - }; - rendered_comparisons(entry.py(), std::slice::from_ref(entry), &settings)? - .into_iter() - .next() - .ok_or_else(|| value_error("native SQL-test rendering returned an invalid batch response")) -} - /// The native comparison render request of one planned SQL test. pub(crate) fn comparison_request( entry: &Bound<'_, PyAny>, @@ -183,7 +163,7 @@ pub(crate) fn chain_step_request(step: &Bound<'_, PyAny>) -> PyResult { /// A bounded query for one direction of an expected-output difference of one chain step. #[pyfunction] #[pyo3(signature = (**options))] -fn render_sql_test_difference_sample_sql( +pub(crate) fn render_sql_test_difference_sample_sql( py: Python<'_>, options: Option<&Bound<'_, PyDict>>, ) -> PyResult { @@ -251,17 +231,3 @@ pub(crate) fn build_scenario_expected_comparison_sql( set_difference_operator, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_sql_test_comparison_sql, module)?)?; - module.add_function(wrap_pyfunction!( - render_sql_test_difference_sample_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(describe_missing_expected_columns, module)?)?; - module.add_function(wrap_pyfunction!( - build_scenario_expected_comparison_sql, - module - )?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/analysis/_helpers/sql_tests.rs b/crates/sqlbuild-python-compile/src/analysis/_helpers/sql_tests.rs new file mode 100644 index 0000000000..f17c4cae8b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/_helpers/sql_tests.rs @@ -0,0 +1,15 @@ +//! Extract, plan and render SQL tests natively. + +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; + +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; + +#[pyfunction] +pub(crate) fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json) + }) + .map_err(value_error) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/constants.rs b/crates/sqlbuild-python-compile/src/analysis/constants.rs new file mode 100644 index 0000000000..8a0341fd4e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of analysis. + +use crate::analysis::_helpers::getters::read_binding_catalog; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis::models::BindingCatalog; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_binding_catalog, +}]; diff --git a/crates/sqlbuild-python-compile/src/analysis/main/locate_model_binding_diagnostics.rs b/crates/sqlbuild-python-compile/src/analysis/main/locate_model_binding_diagnostics.rs new file mode 100644 index 0000000000..b76087e4c6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/locate_model_binding_diagnostics.rs @@ -0,0 +1,40 @@ +//! Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. + +use crate::analysis::_helpers::binding_locations::located_row; +use crate::analysis::models::LocatedRow; +use crate::analysis::types::ExpansionRow; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use pyo3::pyfunction; +use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; +use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; +use sqlbuild_python_project::analysis::models::Positional; + +/// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. +#[pyfunction] +pub fn locate_model_binding_diagnostics( + rows: Vec>, + texts: (String, String, String), + expansion: ExpansionRow, +) -> PyResult> { + let (authored, query, cleaned) = texts; + let rows: Vec = Positional::unwrap_all(rows); + let expansion: Option<(String, Vec>)> = expansion.map(|(expanded, passes)| { + ( + expanded, + passes.into_iter().map(Positional::unwrap_all).collect(), + ) + }); + let source: BindingSource<'_> = BindingSource { + authored: &authored, + query: &query, + cleaned: &cleaned, + expansion: expansion + .as_ref() + .map(|(expanded, passes)| (expanded.as_str(), passes.as_slice())), + }; + let located: Vec = + locate_binding_diagnostics(&rows, &source).map_err(PyValueError::new_err)?; + Ok(located.into_iter().map(located_row).collect()) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/main/mod.rs b/crates/sqlbuild-python-compile/src/analysis/main/mod.rs new file mode 100644 index 0000000000..577dceb73d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/mod.rs @@ -0,0 +1,5 @@ +pub mod locate_model_binding_diagnostics; +pub mod register_queries; +pub mod register_sql_test_comparisons; +pub mod register_sql_tests; +pub mod single_comparison_sql; diff --git a/crates/sqlbuild-python-compile/src/analysis/main/register_queries.rs b/crates/sqlbuild-python-compile/src/analysis/main/register_queries.rs new file mode 100644 index 0000000000..d8679405e7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/register_queries.rs @@ -0,0 +1,19 @@ +//! Register the queries functions and classes. + +use crate::analysis::_helpers::queries::parse_warehouse_type; +use crate::analysis::_helpers::queries::query_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog; +use sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references; + +/// Register the queries functions and classes. +pub fn register_queries(module: &Bound<'_, PyModule>) -> PyResult<()> { + register_binding_catalog(module)?; + crate::analysis::_helpers::required_columns::register(module)?; + register_explicit_references(module)?; + crate::analysis::_helpers::binding_locations::register(module)?; + module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/main/register_sql_test_comparisons.rs b/crates/sqlbuild-python-compile/src/analysis/main/register_sql_test_comparisons.rs new file mode 100644 index 0000000000..f0af98d867 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/register_sql_test_comparisons.rs @@ -0,0 +1,23 @@ +//! Register the sql test comparisons functions and classes. + +use crate::analysis::_helpers::sql_test_comparisons::build_scenario_expected_comparison_sql; +use crate::analysis::_helpers::sql_test_comparisons::describe_missing_expected_columns; +use crate::analysis::_helpers::sql_test_comparisons::render_sql_test_comparison_sql; +use crate::analysis::_helpers::sql_test_comparisons::render_sql_test_difference_sample_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the sql test comparisons functions and classes. +pub fn register_sql_test_comparisons(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_sql_test_comparison_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_sql_test_difference_sample_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(describe_missing_expected_columns, module)?)?; + module.add_function(wrap_pyfunction!( + build_scenario_expected_comparison_sql, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/main/register_sql_tests.rs b/crates/sqlbuild-python-compile/src/analysis/main/register_sql_tests.rs new file mode 100644 index 0000000000..3afd25abab --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/register_sql_tests.rs @@ -0,0 +1,11 @@ +//! Register the sql tests functions and classes. + +use crate::analysis::_helpers::sql_tests::extract_sql_tests_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the sql tests functions and classes. +pub fn register_sql_tests(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/main/single_comparison_sql.rs b/crates/sqlbuild-python-compile/src/analysis/main/single_comparison_sql.rs new file mode 100644 index 0000000000..9cf44841f6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/main/single_comparison_sql.rs @@ -0,0 +1,27 @@ +//! The comparison SQL of one planned SQL test under `adapter`'s operator and dialect. + +use crate::analysis::_helpers::sql_test_comparisons::RenderSettings; +use crate::analysis::_helpers::sql_test_comparisons::rendered_comparisons; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use std::collections::HashMap; + +/// The comparison SQL of one planned SQL test under `adapter`'s operator and dialect. +pub fn single_comparison_sql( + entry: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let settings = RenderSettings { + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + probe_step_index: None, + output_step_index: None, + actual_relations: HashMap::new(), + }; + rendered_comparisons(entry.py(), std::slice::from_ref(entry), &settings)? + .into_iter() + .next() + .ok_or_else(|| value_error("native SQL-test rendering returned an invalid batch response")) +} diff --git a/crates/sqlbuild-python-compile/src/analysis/mod.rs b/crates/sqlbuild-python-compile/src/analysis/mod.rs new file mode 100644 index 0000000000..bb17caf2f8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/mod.rs @@ -0,0 +1,7 @@ +//! SQL query validation, fingerprints and analysis for the Python compiler: required columns, binding locations and SQL test comparisons. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-compile/src/analysis/models.rs b/crates/sqlbuild-python-compile/src/analysis/models.rs new file mode 100644 index 0000000000..07820221a6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/models.rs @@ -0,0 +1,35 @@ +//! Public records of analysis. + +use sqlbuild_analysis::binding_locations::models::AuthoredSpan; +use sqlbuild_python_project::analysis::models::Positional; + +/// One located diagnostic as Python reads it, its span `(line, column, end line, end column)`. +#[derive(Debug)] +pub struct LocatedRow { + pub code: String, + pub message: String, + pub severity: String, + pub line: Option, + pub column: Option, + pub span: Option>, +} + +impl<'py> pyo3::IntoPyObject<'py> for LocatedRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.code, + self.message, + self.severity, + self.line, + self.column, + self.span, + ), + py, + ) + } +} diff --git a/crates/sqlbuild-python-compile/src/analysis/types.rs b/crates/sqlbuild-python-compile/src/analysis/types.rs new file mode 100644 index 0000000000..97b506babb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis/types.rs @@ -0,0 +1,7 @@ +//! Public traits and type aliases of analysis. + +use sqlbuild_analysis::semantic_validation::models::Expansion; +use sqlbuild_python_project::analysis::models::Positional; + +/// The expanded SQL and the expansion passes Python sends for a located model. +pub type ExpansionRow = Option<(String, Vec>>)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/getters.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs rename to crates/sqlbuild-python-compile/src/analysis_session/_helpers/getters.rs index f2f6b837c6..356fd65b69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs +++ b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_native_model_analysis_session, -}]; - -fn read_native_model_analysis_session<'py>( +pub(crate) fn read_native_model_analysis_session<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-compile/src/analysis_session/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/mod.rs new file mode 100644 index 0000000000..d077ec829d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod getters; +pub(crate) mod model_analyses; +pub(crate) mod oracle; +pub(crate) mod outcome_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs rename to crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs index 0a6d690dd0..5d46bb67fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs @@ -1,5 +1,7 @@ //! One compile's native model analysis session, with Python's catalog and metric bookkeeping. +use crate::analysis_session::models::ModelAnalyses; +use crate::analysis_session::models::ModelAnalysisRequest; use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -7,13 +9,10 @@ use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::project_assembly::declared_relations::DeclaredShapes; use sqlbuild_python_project::analysis::models::BindingCatalog; -use crate::bindings::_helpers::compile_driver::analysis_cache::{ - AnalysisCacheCounts, record_analysis_cache_metrics, -}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use crate::compile_driver::main::stages::record_analysis_cache_metrics::record_analysis_cache_metrics; +use crate::compile_driver::models::AnalysisCacheCounts; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::cache::main::content_digest::content_digest; use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; @@ -25,11 +24,9 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; use sqlbuild_python_project::analysis_session::main::case_sensitive_shapes::case_sensitive_shapes; -use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; use sqlbuild_python_project::analysis_session::main::return_types::return_types; use sqlbuild_python_project::analysis_session::main::shape_rows::shape_rows; -use sqlbuild_python_project::analysis_session::main::start_model_analysis_session::start_model_analysis_session; use sqlbuild_python_project::analysis_session::models::{ CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, }; @@ -48,62 +45,8 @@ const STORE_FAILURE_MESSAGE: &str = "native model analysis cache not read or sav const STORE_FILE_NAME: &str = "model-analyses.bin"; const STORE_VERSION: &str = "native-model-analysis-store-v1"; -/// What one compile's model analysis reads: the analysed models and the relation facts. -pub(crate) struct ModelAnalysisRequest<'a, 'py> { - pub(crate) model_inputs: Vec>, - pub(crate) profile: Bound<'py, PyAny>, - pub(crate) column_types: Bound<'py, PyAny>, - pub(crate) nullability: Bound<'py, PyAny>, - pub(crate) complete_schemas: Bound<'py, PyAny>, - pub(crate) dynamic_families: &'a [(String, Vec)], - pub(crate) rich_type_inference: bool, - /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. - pub(crate) cache: Bound<'py, PyAny>, - /// Native rows of the three declaration maps, when the caller built them alongside. - pub(crate) declared: Option<&'a DeclaredShapes>, -} - -/// Each analysed model's position in the finished session, those that succeeded, and the session. -pub(crate) struct ModelAnalyses<'py> { - pub(crate) positions: Bound<'py, PyDict>, - pub(crate) succeeded: Bound<'py, PyFrozenSet>, - /// The finished session, or `None` when no model was analysed. - pub(crate) session: Option>, -} - -/// Each model's analysis by name and the finished session; a native internal failure raises. -pub(crate) fn analyze_models<'py>( - request: &ModelAnalysisRequest<'_, 'py>, -) -> PyResult> { - let py: Python<'py> = request.profile.py(); - if request.model_inputs.is_empty() { - return Ok(ModelAnalyses { - positions: PyDict::new(py), - succeeded: PyFrozenSet::empty(py)?, - session: None, - }); - } - let catalog: Bound<'py, PyAny> = native_field(&request.profile, "binding_catalog")?; - let (rules, adapter_rules) = nullability_rules(&request.profile)?; - let session_request: RequestRow = session_request(request, &catalog, rules)?; - let (cache, adapter) = match adapter_rules { - Some((adapter, nullability)) => (None, Some((adapter, nullability))), - None => (analysis_store(request)?, None), - }; - let session: Bound<'py, NativeModelAnalysisSession> = Bound::new( - py, - start_model_analysis_session( - native_field(&catalog, "native")?.extract()?, - session_request, - cache, - adapter, - )?, - )?; - run_session(request, &catalog, &session) -} - /// The session request row, in the order the native session reads it. -fn session_request<'py>( +pub(crate) fn session_request<'py>( request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, rules: Vec<(String, String)>, @@ -189,7 +132,9 @@ fn declared_rows( } /// The analysis store's path and environment, or None where Python bypasses its cache. -fn analysis_store(request: &ModelAnalysisRequest<'_, '_>) -> PyResult> { +pub(crate) fn analysis_store( + request: &ModelAnalysisRequest<'_, '_>, +) -> PyResult> { let context: &Bound<'_, PyAny> = &request.cache; if context.is_none() { return Ok(None); @@ -207,7 +152,7 @@ fn analysis_store(request: &ModelAnalysisRequest<'_, '_>) -> PyResult( +pub(crate) fn run_session<'py>( request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, session: &Bound<'py, NativeModelAnalysisSession>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/oracle.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs rename to crates/sqlbuild-python-compile/src/analysis_session/_helpers/oracle.rs index 11dc02353e..7f59ef262f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs +++ b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/oracle.rs @@ -1,7 +1,7 @@ //! Test-only hooks comparing native CTE fact recovery and set-operation search with Python's. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; use sqlbuild_analysis::assembly::analysis_session::main::recover_cte_facts::recover_cte_facts; use sqlbuild_analysis::assembly::analysis_session::models::{CteFactRequest, CteFacts}; @@ -14,7 +14,7 @@ use sqlbuild_python_base::types::CompilerDetach; positional_record! { extract; /// One query's CTE fact recovery request, as the oracle tests send it. - struct RecoveryRow { + pub(crate) struct RecoveryRow { cleaned_sql: String, dialect: String, input_schemas: Shapes, @@ -28,7 +28,7 @@ positional_record! { positional_record! { into; /// The deferral, types, nullability, sorted direct outputs and sorted non-null outputs. - struct FactsRow { + pub(crate) struct FactsRow { deferral: Option, types: Pairs, nullability: Pairs, @@ -39,7 +39,10 @@ positional_record! { /// The facts native enrichment recovers for one query, or the reason it defers to Python. #[pyfunction] -fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult { +pub(crate) fn _oracle_cte_fact_recovery( + py: Python<'_>, + request: RecoveryRow, +) -> PyResult { let RecoveryRow { cleaned_sql, dialect, @@ -81,12 +84,6 @@ fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult bool { +pub(crate) fn _oracle_names_set_operation(sql: &str) -> bool { names_set_operation(sql) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(_oracle_cte_fact_recovery, module)?)?; - module.add_function(wrap_pyfunction!(_oracle_names_set_operation, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/outcome_objects.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs rename to crates/sqlbuild-python-compile/src/analysis_session/_helpers/outcome_objects.rs index bb49591f29..fff3107262 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python-compile/src/analysis_session/_helpers/outcome_objects.rs @@ -1,9 +1,9 @@ //! Finished session outcomes and pivot proof rows as Python's analysis objects. use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::positional_record; use sqlbuild_python_base::boundary::models::Request; @@ -16,11 +16,11 @@ use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kin const LINEAGE_NATIVE: &str = "native"; -type Object<'py> = Bound<'py, PyAny>; +pub(crate) type Object<'py> = Bound<'py, PyAny>; positional_record! { extract; /// A session outcome row: its analysis, lineage, binding diagnostics and cleaned SQL. - struct OutcomeObjects<'py> { + pub(crate) struct OutcomeObjects<'py> { succeeded: Object<'py>, columns: Object<'py>, source: String, @@ -120,13 +120,7 @@ pub(crate) fn session_outcome_analysis<'py>( ) } -/// A native dynamic pivot proof row as Python's proof, None when absent. -#[pyfunction] -pub(crate) fn dynamic_contract_proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - proof(row) -} - -fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = row.py(); if row.is_none() { return Ok(row.clone()); @@ -316,8 +310,3 @@ fn call<'py>( } callable.call((), Some(&arguments)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(session_outcome_analysis, module)?)?; - module.add_function(wrap_pyfunction!(dynamic_contract_proof, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/constants.rs b/crates/sqlbuild-python-compile/src/analysis_session/constants.rs new file mode 100644 index 0000000000..f506914742 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of analysis session. + +use crate::analysis_session::_helpers::getters::read_native_model_analysis_session; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_native_model_analysis_session, +}]; diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/analyze_models.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/analyze_models.rs new file mode 100644 index 0000000000..43f6b01a5a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/analyze_models.rs @@ -0,0 +1,44 @@ +//! Each model's analysis by name and the finished session; a native internal failure raises. + +use crate::analysis_session::_helpers::model_analyses::analysis_store; +use crate::analysis_session::_helpers::model_analyses::run_session; +use crate::analysis_session::_helpers::model_analyses::session_request; +use crate::analysis_session::models::ModelAnalyses; +use crate::analysis_session::models::ModelAnalysisRequest; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; +use sqlbuild_python_project::analysis_session::main::start_model_analysis_session::start_model_analysis_session; +use sqlbuild_python_project::analysis_session::models::{NativeModelAnalysisSession, RequestRow}; + +/// Each model's analysis by name and the finished session; a native internal failure raises. +pub fn analyze_models<'py>( + request: &ModelAnalysisRequest<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = request.profile.py(); + if request.model_inputs.is_empty() { + return Ok(ModelAnalyses { + positions: PyDict::new(py), + succeeded: PyFrozenSet::empty(py)?, + session: None, + }); + } + let catalog: Bound<'py, PyAny> = native_field(&request.profile, "binding_catalog")?; + let (rules, adapter_rules) = nullability_rules(&request.profile)?; + let session_request: RequestRow = session_request(request, &catalog, rules)?; + let (cache, adapter) = match adapter_rules { + Some((adapter, nullability)) => (None, Some((adapter, nullability))), + None => (analysis_store(request)?, None), + }; + let session: Bound<'py, NativeModelAnalysisSession> = Bound::new( + py, + start_model_analysis_session( + native_field(&catalog, "native")?.extract()?, + session_request, + cache, + adapter, + )?, + )?; + run_session(request, &catalog, &session) +} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/dynamic_contract_proof.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/dynamic_contract_proof.rs new file mode 100644 index 0000000000..75009f391c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/dynamic_contract_proof.rs @@ -0,0 +1,11 @@ +//! A native dynamic pivot proof row as Python's proof, None when absent. + +use crate::analysis_session::_helpers::outcome_objects::proof; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// A native dynamic pivot proof row as Python's proof, None when absent. +#[pyfunction] +pub fn dynamic_contract_proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + proof(row) +} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/mod.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/mod.rs new file mode 100644 index 0000000000..3556d6b090 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/mod.rs @@ -0,0 +1,5 @@ +pub mod analyze_models; +pub mod dynamic_contract_proof; +pub mod register_oracle; +pub mod register_outcome_objects; +pub mod session_outcome_analysis; diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/register_oracle.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/register_oracle.rs new file mode 100644 index 0000000000..e173515f86 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/register_oracle.rs @@ -0,0 +1,13 @@ +//! Register the oracle functions and classes. + +use crate::analysis_session::_helpers::oracle::_oracle_cte_fact_recovery; +use crate::analysis_session::_helpers::oracle::_oracle_names_set_operation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the oracle functions and classes. +pub fn register_oracle(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_cte_fact_recovery, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_names_set_operation, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/register_outcome_objects.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/register_outcome_objects.rs new file mode 100644 index 0000000000..d74dd3953b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/register_outcome_objects.rs @@ -0,0 +1,12 @@ +//! Register the outcome objects functions and classes. + +use crate::analysis_session::_helpers::outcome_objects::session_outcome_analysis; +use crate::analysis_session::main::dynamic_contract_proof::dynamic_contract_proof; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the outcome objects functions and classes. +pub fn register_outcome_objects(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(session_outcome_analysis, module)?)?; + module.add_function(wrap_pyfunction!(dynamic_contract_proof, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/main/session_outcome_analysis.rs b/crates/sqlbuild-python-compile/src/analysis_session/main/session_outcome_analysis.rs new file mode 100644 index 0000000000..51306b8c03 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/main/session_outcome_analysis.rs @@ -0,0 +1,15 @@ +//! The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. + +use crate::analysis_session::_helpers::outcome_objects; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. +#[inline] +pub fn session_outcome_analysis<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + outcome_objects::session_outcome_analysis(request) +} diff --git a/crates/sqlbuild-python-compile/src/analysis_session/mod.rs b/crates/sqlbuild-python-compile/src/analysis_session/mod.rs new file mode 100644 index 0000000000..2c03330d3b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/mod.rs @@ -0,0 +1,6 @@ +//! One compile's native model analysis session and its finished outcomes as Python's analysis objects. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/analysis_session/models.rs b/crates/sqlbuild-python-compile/src/analysis_session/models.rs new file mode 100644 index 0000000000..39a2fbde80 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/analysis_session/models.rs @@ -0,0 +1,34 @@ +//! Public records of analysis session. + +use crate::project_assembly::models::DeclaredShapes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::types::PyDict; +use pyo3::types::PyFrozenSet; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; + +/// What one compile's model analysis reads: the analysed models and the relation facts. +#[derive(Debug)] +pub struct ModelAnalysisRequest<'a, 'py> { + pub model_inputs: Vec>, + pub profile: Bound<'py, PyAny>, + pub column_types: Bound<'py, PyAny>, + pub nullability: Bound<'py, PyAny>, + pub complete_schemas: Bound<'py, PyAny>, + pub dynamic_families: &'a [(String, Vec)], + pub rich_type_inference: bool, + /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. + pub cache: Bound<'py, PyAny>, + /// Native rows of the three declaration maps, when the caller built them alongside. + pub declared: Option<&'a DeclaredShapes>, +} + +/// Each analysed model's position in the finished session, those that succeeded, and the session. +#[derive(Debug)] +pub struct ModelAnalyses<'py> { + pub positions: Bound<'py, PyDict>, + pub succeeded: Bound<'py, PyFrozenSet>, + /// The finished session, or `None` when no model was analysed. + pub session: Option>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs b/crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs rename to crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs index c1d329e11f..8c0fcea54b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs +++ b/crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs @@ -1,8 +1,8 @@ //! Audit gate identities of planned audits, built as `AuditGateIdentity` objects. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::auditing::main::audit_gate_identity::audit_gate_identity; use sqlbuild_analysis::auditing::models::{ GateIdentity, MeasurementFacts, PlannedAudit, ThresholdBound, Thresholds, @@ -88,7 +88,7 @@ fn keyword_call<'py>( /// Target-neutral and target-specific identity hashes for planned audits. #[pyfunction] #[pyo3(signature = (*, audits))] -fn build_audit_gate_identity<'py>( +pub(crate) fn build_audit_gate_identity<'py>( py: Python<'py>, audits: &Bound<'py, PyAny>, ) -> PyResult> { @@ -154,8 +154,3 @@ fn build_audit_gate_identity<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_audit_gate_identity, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs b/crates/sqlbuild-python-compile/src/auditing/_helpers/measurement.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs rename to crates/sqlbuild-python-compile/src/auditing/_helpers/measurement.rs index e15b03c0c3..40a7306a50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs +++ b/crates/sqlbuild-python-compile/src/auditing/_helpers/measurement.rs @@ -1,15 +1,13 @@ //! A measured value's audit outcome under its sample and threshold policy. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::auditing::main::evaluate_measurement::evaluate_measurement as evaluate; use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds}; use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds; +use crate::auditing::_helpers::identity::thresholds as read_thresholds; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::types::ErrorId; @@ -20,7 +18,7 @@ fn measurement_error(py: Python<'_>, message: &str) -> PyErr { /// The canonical outcome for one finite measured value. #[pyfunction] #[pyo3(signature = (*, measured_value, sample_count, minimum_samples, thresholds))] -fn evaluate_measurement<'py>( +pub(crate) fn evaluate_measurement<'py>( measured_value: f64, sample_count: Option, minimum_samples: Option, @@ -39,8 +37,3 @@ fn evaluate_measurement<'py>( }; host_entry(py, FacadeHost::AuditOutcome)?.getattr(member) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_measurement, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/auditing/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/auditing/_helpers/mod.rs new file mode 100644 index 0000000000..f112cbde93 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod identity; +pub(crate) mod measurement; +pub(crate) mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs b/crates/sqlbuild-python-compile/src/auditing/_helpers/schema_audits.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs rename to crates/sqlbuild-python-compile/src/auditing/_helpers/schema_audits.rs index 50b57c771c..eef4d704aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs +++ b/crates/sqlbuild-python-compile/src/auditing/_helpers/schema_audits.rs @@ -1,10 +1,8 @@ //! Authored schema audit lists parsed natively into `SchemaAuditInstance` objects. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use sqlbuild_model_config::header_metadata::main::parse_audit_list::parse_audit_list; use sqlbuild_model_config::header_metadata::models::ParsedAudit; @@ -41,7 +39,7 @@ fn raised( /// Parse an authored audit list, treating null as empty only when requested. #[pyfunction] #[pyo3(signature = (**request))] -fn parse_audit_instances<'py>( +pub(crate) fn parse_audit_instances<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("parse_audit_instances", request)?; @@ -60,8 +58,3 @@ fn parse_audit_instances<'py>( let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; audit_tuple(py, &classes, &parsed) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_audit_instances, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/auditing/main/mod.rs b/crates/sqlbuild-python-compile/src/auditing/main/mod.rs new file mode 100644 index 0000000000..2d2321f79c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/main/mod.rs @@ -0,0 +1,3 @@ +pub mod register_identity; +pub mod register_measurement; +pub mod register_schema_audits; diff --git a/crates/sqlbuild-python-compile/src/auditing/main/register_identity.rs b/crates/sqlbuild-python-compile/src/auditing/main/register_identity.rs new file mode 100644 index 0000000000..1ae0a1a636 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/main/register_identity.rs @@ -0,0 +1,11 @@ +//! Register the identity functions and classes. + +use crate::auditing::_helpers::identity::build_audit_gate_identity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the identity functions and classes. +pub fn register_identity(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_audit_gate_identity, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/auditing/main/register_measurement.rs b/crates/sqlbuild-python-compile/src/auditing/main/register_measurement.rs new file mode 100644 index 0000000000..d9f5ffd778 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/main/register_measurement.rs @@ -0,0 +1,11 @@ +//! Register the measurement functions and classes. + +use crate::auditing::_helpers::measurement::evaluate_measurement; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the measurement functions and classes. +pub fn register_measurement(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(evaluate_measurement, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/auditing/main/register_schema_audits.rs b/crates/sqlbuild-python-compile/src/auditing/main/register_schema_audits.rs new file mode 100644 index 0000000000..619468c133 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/main/register_schema_audits.rs @@ -0,0 +1,11 @@ +//! Register the schema audits functions and classes. + +use crate::auditing::_helpers::schema_audits::parse_audit_instances; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the schema audits functions and classes. +pub fn register_schema_audits(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_audit_instances, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/auditing/mod.rs b/crates/sqlbuild-python-compile/src/auditing/mod.rs new file mode 100644 index 0000000000..231c3b2d8b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/auditing/mod.rs @@ -0,0 +1,4 @@ +//! Audit gate identities, measured audit outcomes and authored schema audits. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/declaration_scope.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/declaration_scope.rs diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/mod.rs new file mode 100644 index 0000000000..95a0b5b93e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declaration_scope; +pub(crate) mod model_config; +pub(crate) mod model_inputs; +pub(crate) mod sql_expansion_context; +pub(crate) mod storage_policies; +pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_config.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_config.rs index d945fb86d7..6b54307203 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_config.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; -use crate::bindings::_helpers::input_loops::loop_classes::table_backed; +use crate::compile_attachment::_helpers::storage_policies::inherited_storage; +use crate::input_loops::main::table_backed::table_backed; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs index 884976fc17..e20a08393b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs @@ -10,9 +10,10 @@ use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_a use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; -use crate::bindings::_helpers::input_loops::loop_classes::model_loop_classes; -use crate::bindings::_helpers::input_loops::model_loop::{LoopSwitches, NativeModelLoop}; +use crate::compile_attachment::_helpers::model_config::native_model_config_session; +use crate::input_loops::main::model_loop_classes::model_loop_classes; +use crate::input_loops::models::LoopSwitches; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -275,29 +276,27 @@ pub(crate) fn build_model_inputs<'py>( &host_inputs, )?; let syntax: Bound<'py, PyAny> = native_field(context, "sql_lexical_syntax")?; - let native_loop: Bound<'py, PyAny> = - crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop::new( - ( - validator.unbind(), - contract_classes(py)?.unbind(), - classes.unbind(), - ), - native_field(&syntax, "native_mapping")?.extract()?, - reserved_cursor_markers(), - ( - LoopSwitches { - sql_analysis: native_field(&settings, "sql_analysis")?.extract()?, - no_sql_validation, - defer_sql_validation: defer_model_sql_validation, - require_sql_analysis: native_field(&settings, "require_sql_analysis")? - .extract()?, - }, - project_config_path.str()?.extract()?, - native_field(context, "sql_analysis_dialect")?.extract()?, - ), - ) - .and_then(|value| pyo3::Bound::new(py, value)) - .map(pyo3::Bound::into_any)?; + let native_loop: Bound<'py, PyAny> = crate::input_loops::models::NativeModelLoop::new( + ( + validator.unbind(), + contract_classes(py)?.unbind(), + classes.unbind(), + ), + native_field(&syntax, "native_mapping")?.extract()?, + reserved_cursor_markers(), + ( + LoopSwitches { + sql_analysis: native_field(&settings, "sql_analysis")?.extract()?, + no_sql_validation, + defer_sql_validation: defer_model_sql_validation, + require_sql_analysis: native_field(&settings, "require_sql_analysis")?.extract()?, + }, + project_config_path.str()?.extract()?, + native_field(context, "sql_analysis_dialect")?.extract()?, + ), + ) + .and_then(|value| pyo3::Bound::new(py, value)) + .map(pyo3::Bound::into_any)?; let files: Vec> = native_field(discovered_inputs, "model_files")? .try_iter()? .map(|file| file?.extract()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/sql_expansion_context.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/sql_expansion_context.rs index da924c246f..a8ff32f413 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/sql_expansion_context.rs @@ -1,14 +1,17 @@ //! The SQL expansion inputs lint uses to expand authored SQL outside a compile. +use crate::compile_attachment::main::present::present; +use crate::compile_attachment::models::ExpansionSources; +use crate::compile_attachment::models::ExpansionVars; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; -use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::compile_attachment::_helpers::declaration_scope::build_declaration_scope; +use crate::compile_attachment::_helpers::declaration_scope::rebind_declaration_scope; +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -24,7 +27,7 @@ use sqlbuild_python_front::render::main::declarations::public_declaration_indexe use sqlbuild_python_front::render::models::SqlExpansionInputs; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. -fn declaration_scope<'py>( +pub(crate) fn declaration_scope<'py>( discovered: &Bound<'py, PyAny>, static_scope: Bound<'py, PyAny>, sql_lexical_syntax: Bound<'py, PyAny>, @@ -83,27 +86,6 @@ fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult { - pub(crate) project_dir: Bound<'py, PyAny>, - pub(crate) value_renderer: Bound<'py, PyAny>, - pub(crate) vars: ExpansionVars<'py>, - pub(crate) discovered_inputs: Option>, - pub(crate) declaration_scope: Option>, - pub(crate) static_declaration_scope: Option>, -} - -/// Where expansion takes its target and vars from. -pub(crate) enum ExpansionVars<'py> { - /// For commands without `--target`: the project's default target, with these CLI vars. - ProjectDefault(Option>), - /// The compile's effective target name and vars, exactly as the command resolved them. - Compile { - target_name: Bound<'py, PyAny>, - vars: Bound<'py, PyAny>, - }, -} - /// Assemble SQL expansion inputs; a static scope lends only its index to private macros. pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { let py: Python<'_> = sources.project_dir.py(); @@ -197,11 +179,6 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult) -> Option> { - (!value.is_none()).then_some(value) -} - /// Assemble SQL expansion inputs; a static scope lends only its index to private macros. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/storage_policies.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/storage_policies.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/target.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs rename to crates/sqlbuild-python-compile/src/compile_attachment/_helpers/target.rs index 408b1c8897..f512396ace 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python-compile/src/compile_attachment/_helpers/target.rs @@ -1,23 +1,22 @@ //! The effective target, its namespace, and the shared project-local compile-cache directory. +use crate::compile_attachment::main::connection_is_default_target::connection_is_default_target; +use crate::compile_attachment::main::project_location::project_location; +use crate::compile_attachment::main::resolved_target::resolved_target; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use crate::bindings::_helpers::compile_command::constants::{ - COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, -}; -use crate::bindings::_helpers::frontier::cache_directory::cache_directory; -use crate::bindings::_helpers::frontier::constants::{ - COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, -}; -use pyo3::IntoPyObjectExt; +use crate::compile_command::constants::COMPILE_CACHE_DISABLE_ENV_VAR; +use crate::compile_command::constants::COMPILE_CACHE_DISABLE_VALUE; +use crate::compile_command::constants::RETIRED_CACHE_DIRECTORY_NAMES; +use crate::frontier::constants::COMPILER_CACHE_DIRECTORY_NAME; +use crate::frontier::constants::RETIRED_CACHE_NAMESPACE_SUFFIXES; +use crate::frontier::main::cache_directory::cache_directory; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::compile_attachment::main::project_vars::project_vars; -use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; -use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; use sqlbuild_python_front::model_config::main::config::run_id::run_id; @@ -26,30 +25,6 @@ const SCHEMA_KEY: &str = "schema"; /// Adapters whose database is a file, so a connection `database` is not a target location. pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; -/// The selected target's name and merged config, or `(None, None)`. -pub(crate) fn resolved_target<'py>( - discovered: &Bound<'py, PyAny>, - selected_target: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py: Python<'py> = discovered.py(); - let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; - let target_name: Bound<'py, PyAny> = resolve_target_name( - py, - &project_config.clone(), - &local_config.clone(), - &selected_target.clone(), - ) - .and_then(|value| value.into_bound_py_any(py))?; - if target_name.is_none() { - return Ok((target_name, py.None().into_bound(py))); - } - let target_config: Bound<'py, PyAny> = - resolve_target_config(py, &project_config, &local_config, &target_name.clone()) - .and_then(|value| value.into_bound_py_any(py))?; - Ok((target_name, target_config)) -} - /// Resolve the effective target and shared project-local compile-cache directory. #[pyfunction] #[pyo3(signature = (*, discovered_inputs, selected_target, no_cache))] @@ -125,33 +100,6 @@ fn optional_string<'py>( }) } -/// A project-level location: the target's value, then `[defaults]`, then the first set fallback. -pub(crate) fn project_location<'py>( - (target, default): (Bound<'py, PyAny>, Bound<'py, PyAny>), - fallbacks: Vec>, -) -> PyResult> { - let py: Python<'py> = target.py(); - for value in [target, default].into_iter().chain(fallbacks) { - if value.is_truthy()? { - return Ok(value); - } - } - Ok(py.None().into_bound(py)) -} - -/// Whether the top-level `[connection]` is the implicit default target: no named connections or targets. -pub(crate) fn connection_is_default_target(discovered: &Bound<'_, PyAny>) -> PyResult { - for config in ["project_config", "local_config"] { - let config: Bound<'_, PyAny> = native_field(discovered, config)?; - for field in ["connections", "targets"] { - if native_field(&config, field)?.is_truthy()? { - return Ok(false); - } - } - } - Ok(true) -} - /// The selected target's name, record, database and schema, resolved from `connection` and the adapter's defaults without compiling resources. pub(crate) fn effective_target_namespace<'py>( (discovered_inputs, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_compile_target_context_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_compile_target_context_function.rs new file mode 100644 index 0000000000..84974ccdd3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_compile_target_context_function.rs @@ -0,0 +1,10 @@ +//! `build_compile_target_context` as a function of `module`. + +use crate::compile_attachment::_helpers::target::build_compile_target_context; + +/// `build_compile_target_context` as a function of `module`. +pub fn build_compile_target_context_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_compile_target_context, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope.rs new file mode 100644 index 0000000000..efaa7e24dc --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope.rs @@ -0,0 +1,22 @@ +//! Build one canonical index and validate it before SQL expansion. + +use crate::compile_attachment::_helpers::declaration_scope; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build one canonical index and validate it before SQL expansion. +#[inline] +pub fn build_declaration_scope<'py>( + discovered_inputs: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, + sql_lexical_syntax: &Bound<'py, PyAny>, + compile_cache_dir: Option>, +) -> PyResult> { + declaration_scope::build_declaration_scope( + discovered_inputs, + loaded_macros, + sql_lexical_syntax, + compile_cache_dir, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope_function.rs new file mode 100644 index 0000000000..f6e2af2d67 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_declaration_scope_function.rs @@ -0,0 +1,10 @@ +//! `build_declaration_scope` as a function of `module`. + +use crate::compile_attachment::_helpers::declaration_scope::build_declaration_scope; + +/// `build_declaration_scope` as a function of `module`. +pub fn build_declaration_scope_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_declaration_scope, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs.rs new file mode 100644 index 0000000000..7664eae508 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs.rs @@ -0,0 +1,17 @@ +//! Attach schema metadata to discovered model files. + +use crate::compile_attachment::_helpers::model_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Attach schema metadata to discovered model files. +#[inline] +pub fn build_model_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + model_inputs::build_model_inputs(py, options) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs_function.rs new file mode 100644 index 0000000000..4fdb4ed06e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_model_inputs_function.rs @@ -0,0 +1,10 @@ +//! `build_model_inputs` as a function of `module`. + +use crate::compile_attachment::_helpers::model_inputs::build_model_inputs; + +/// `build_model_inputs` as a function of `module`. +pub fn build_model_inputs_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_model_inputs, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/build_sql_expansion_context_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_sql_expansion_context_function.rs new file mode 100644 index 0000000000..180f9480c1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/build_sql_expansion_context_function.rs @@ -0,0 +1,10 @@ +//! `build_sql_expansion_context` as a function of `module`. + +use crate::compile_attachment::_helpers::sql_expansion_context::build_sql_expansion_context; + +/// `build_sql_expansion_context` as a function of `module`. +pub fn build_sql_expansion_context_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_sql_expansion_context, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/connection_is_default_target.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/connection_is_default_target.rs new file mode 100644 index 0000000000..376da04f2a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/connection_is_default_target.rs @@ -0,0 +1,17 @@ +//! Whether the top-level `[connection]` is the implicit default target: no named connections or targets. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether the top-level `[connection]` is the implicit default target: no named connections or targets. +pub fn connection_is_default_target(discovered: &Bound<'_, PyAny>) -> PyResult { + for config in ["project_config", "local_config"] { + let config: Bound<'_, PyAny> = native_field(discovered, config)?; + for field in ["connections", "targets"] { + if native_field(&config, field)?.is_truthy()? { + return Ok(false); + } + } + } + Ok(true) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/effective_target_namespace.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/effective_target_namespace.rs new file mode 100644 index 0000000000..9ce4ff1974 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/effective_target_namespace.rs @@ -0,0 +1,19 @@ +//! The selected target's name, record, database and schema, resolved from `connection` and the adapter's defaults without compiling resources. + +use crate::compile_attachment::_helpers::target; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// The selected target's name, record, database and schema, resolved from `connection` and the adapter's defaults without compiling resources. +#[inline] +pub fn effective_target_namespace<'py>( + (discovered_inputs, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter_name, connection, adapter): (&str, &Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + target::effective_target_namespace( + (discovered_inputs, selected_target), + (adapter_name, connection, adapter), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/mod.rs new file mode 100644 index 0000000000..a8160de097 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/mod.rs @@ -0,0 +1,14 @@ +pub mod build_compile_target_context_function; +pub mod build_declaration_scope; +pub mod build_declaration_scope_function; +pub mod build_model_inputs; +pub mod build_model_inputs_function; +pub mod build_sql_expansion_context_function; +pub mod connection_is_default_target; +pub mod effective_target_namespace; +pub mod native_model_config_session_function; +pub mod present; +pub mod project_location; +pub mod rebind_declaration_scope_function; +pub mod resolved_target; +pub mod sql_expansion_inputs; diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/native_model_config_session_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/native_model_config_session_function.rs new file mode 100644 index 0000000000..e680b62048 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/native_model_config_session_function.rs @@ -0,0 +1,10 @@ +//! `native_model_config_session` as a function of `module`. + +use crate::compile_attachment::_helpers::model_config::native_model_config_session; + +/// `native_model_config_session` as a function of `module`. +pub fn native_model_config_session_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(native_model_config_session, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/present.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/present.rs new file mode 100644 index 0000000000..3846a1cce4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/present.rs @@ -0,0 +1,8 @@ +//! `value`, or `None` when it is Python `None`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; + +/// `value`, or `None` when it is Python `None`. +pub fn present(value: Bound<'_, PyAny>) -> Option> { + (!value.is_none()).then_some(value) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/project_location.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/project_location.rs new file mode 100644 index 0000000000..e7e35df836 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/project_location.rs @@ -0,0 +1,17 @@ +//! A project-level location: the target's value, then `[defaults]`, then the first set fallback. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +/// A project-level location: the target's value, then `[defaults]`, then the first set fallback. +pub fn project_location<'py>( + (target, default): (Bound<'py, PyAny>, Bound<'py, PyAny>), + fallbacks: Vec>, +) -> PyResult> { + let py: Python<'py> = target.py(); + for value in [target, default].into_iter().chain(fallbacks) { + if value.is_truthy()? { + return Ok(value); + } + } + Ok(py.None().into_bound(py)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/rebind_declaration_scope_function.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/rebind_declaration_scope_function.rs new file mode 100644 index 0000000000..0bc14e54f9 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/rebind_declaration_scope_function.rs @@ -0,0 +1,10 @@ +//! `rebind_declaration_scope` as a function of `module`. + +use crate::compile_attachment::_helpers::declaration_scope::rebind_declaration_scope; + +/// `rebind_declaration_scope` as a function of `module`. +pub fn rebind_declaration_scope_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(rebind_declaration_scope, module) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/resolved_target.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/resolved_target.rs new file mode 100644 index 0000000000..673b7ea8d2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/resolved_target.rs @@ -0,0 +1,31 @@ +//! The selected target's name and merged config, or `(None, None)`. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; + +/// The selected target's name and merged config, or `(None, None)`. +pub fn resolved_target<'py>( + discovered: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let target_name: Bound<'py, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; + if target_name.is_none() { + return Ok((target_name, py.None().into_bound(py))); + } + let target_config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; + Ok((target_name, target_config)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/main/sql_expansion_inputs.rs b/crates/sqlbuild-python-compile/src/compile_attachment/main/sql_expansion_inputs.rs new file mode 100644 index 0000000000..b5ec008927 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/main/sql_expansion_inputs.rs @@ -0,0 +1,12 @@ +//! Assemble SQL expansion inputs; a static scope lends only its index to private macros. + +use crate::compile_attachment::_helpers::sql_expansion_context; +use crate::compile_attachment::models::ExpansionSources; +use pyo3::prelude::PyResult; +use sqlbuild_python_front::render::models::SqlExpansionInputs; + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +#[inline] +pub fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { + sql_expansion_context::sql_expansion_inputs(sources) +} diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/mod.rs b/crates/sqlbuild-python-compile/src/compile_attachment/mod.rs new file mode 100644 index 0000000000..7c16ef03f6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/mod.rs @@ -0,0 +1,5 @@ +//! Compile-input attachment: the declaration scope, model config and the model inputs the native model loop attaches. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/compile_attachment/models.rs b/crates/sqlbuild-python-compile/src/compile_attachment/models.rs new file mode 100644 index 0000000000..fde579fcec --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_attachment/models.rs @@ -0,0 +1,27 @@ +//! Public records of compile attachment. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Where SQL expansion inputs come from: the project, its discovery and the scope to reuse. +#[derive(Debug)] +pub struct ExpansionSources<'py> { + pub project_dir: Bound<'py, PyAny>, + pub value_renderer: Bound<'py, PyAny>, + pub vars: ExpansionVars<'py>, + pub discovered_inputs: Option>, + pub declaration_scope: Option>, + pub static_declaration_scope: Option>, +} + +/// Where expansion takes its target and vars from. +#[derive(Debug)] +pub enum ExpansionVars<'py> { + /// For commands without `--target`: the project's default target, with these CLI vars. + ProjectDefault(Option>), + /// The compile's effective target name and vars, exactly as the command resolved them. + Compile { + target_name: Bound<'py, PyAny>, + vars: Bound<'py, PyAny>, + }, +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/artifact_batch.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/artifact_batch.rs new file mode 100644 index 0000000000..5f9922af30 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/artifact_batch.rs @@ -0,0 +1,28 @@ +//! Compiled artifacts queued for one native write, keeping unchanged files when asked. + +pub(crate) const PHYSICAL_WRITE_PHASE: &str = "physical_write_ms"; + +/// `contents` with platform line endings. +pub(crate) fn platform_lines(contents: Vec) -> Vec { + if cfg!(windows) { + contents + .iter() + .flat_map(|byte| { + if *byte == b'\n' { + vec![b'\r', b'\n'] + } else { + vec![*byte] + } + }) + .collect() + } else { + contents + } +} + +/// `sql` with trailing whitespace replaced by one newline. +pub(crate) fn sql_artifact(sql: &str) -> Vec { + let trimmed: &str = + sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); + format!("{trimmed}\n").into_bytes() +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/mod.rs new file mode 100644 index 0000000000..c87ecb5f45 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod artifact_batch; +pub(crate) mod prepared_artifacts; +pub(crate) mod surrogates; +pub(crate) mod target_writer; +pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs index d77574ef3b..560a96376a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs @@ -1,20 +1,19 @@ //! Prepare compile artifacts in the background and publish them only at the normal write phase. +use crate::compile_command::main::artifacts::planning_diagnostics::planning_diagnostics; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyRefMut, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; -use crate::bindings::_helpers::compile_command::reuse_recording::CompileArtifactWrites; -use crate::bindings::_helpers::compile_command::target_writer::{ - PlannedStaticSqlTests, plan_tests, -}; -use crate::bindings::_helpers::compile_command::target_writer::{ - Prepared, planning_diagnostics, publish_staged_target, staging_changed, write_static_target, -}; -use crate::bindings::_helpers::compile_outputs::artifacts::staged_artifact_files; +use crate::compile_command::_helpers::artifacts::target_writer::PlannedStaticSqlTests; +use crate::compile_command::_helpers::artifacts::target_writer::plan_tests; +use crate::compile_command::_helpers::artifacts::target_writer::publish_staged_target; +use crate::compile_command::_helpers::artifacts::target_writer::staging_changed; +use crate::compile_command::_helpers::artifacts::target_writer::write_static_target; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::Prepared; +use crate::compile_outputs::main::staged_artifact_files::staged_artifact_files; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -583,7 +582,3 @@ impl PreparedCompileArtifacts { } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/surrogates.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/surrogates.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs index ddd0e533d9..00daa4b9bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs @@ -1,14 +1,13 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. +use crate::compile_command::models::Prepared; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction}; use sqlbuild_cache::artifacts::constants::SQL_TEST_CACHE_FILE; use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; @@ -17,22 +16,25 @@ use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; -use crate::bindings::_helpers::compile_command::artifact_batch::ArtifactBatch; -use crate::bindings::_helpers::compile_command::constants::RACY_WINDOW_NS; -use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, artifact_kept, artifact_moved, artifact_tree_moved, -}; -use crate::bindings::_helpers::compile_command::test_artifact_cache::{ - IdentityContext, artifact_identity_payload, artifact_record_key_payload, identity_context, - payload_digests, read_cache, write_cache, yield_interpreter, -}; -use crate::bindings::_helpers::compile_outputs::artifacts::{ - publish_staged_artifacts, remove_stale_artifacts, -}; -use crate::bindings::_helpers::compile_outputs::runtime_target::{ - compiled_sql_test_path, planned_sql_test_path, -}; +use crate::analysis::main::single_comparison_sql::single_comparison_sql; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::IdentityContext; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::artifact_identity_payload; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::artifact_record_key_payload; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::identity_context; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::payload_digests; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::read_cache; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::write_cache; +use crate::compile_command::_helpers::artifacts::test_artifact_cache::yield_interpreter; +use crate::compile_command::_helpers::invocation::reuse_recording::artifact_kept; +use crate::compile_command::_helpers::invocation::reuse_recording::artifact_moved; +use crate::compile_command::_helpers::invocation::reuse_recording::artifact_tree_moved; +use crate::compile_command::constants::RACY_WINDOW_NS; +use crate::compile_command::models::ArtifactBatch; +use crate::compile_command::types::ArtifactRecorder; +use crate::compile_outputs::main::compiled_sql_test_path::compiled_sql_test_path; +use crate::compile_outputs::main::planned_sql_test_path::planned_sql_test_path; +use crate::compile_outputs::main::publish_staged_artifacts::publish_staged_artifacts; +use crate::compile_outputs::main::remove_stale_artifacts::remove_stale_artifacts; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -274,7 +276,7 @@ fn length(value: &Bound<'_, PyAny>, field: &str) -> PyResult { } /// Write the planner's compiled output under `target_dir`. -fn write_planned_target<'py>( +pub(crate) fn write_planned_target<'py>( target_dir: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, plan_output: &Bound<'py, PyAny>, @@ -352,14 +354,6 @@ fn write_planned_target<'py>( ) } -/// SQL tests planned ahead of the write phase, and where the compile notes its artifact writes. -#[derive(Clone, Copy, Default)] -pub(crate) struct Prepared<'a, 'py> { - /// `planned_tests()` hands over the background plan, when one was started. - pub(crate) planned_tests: Option<&'a Bound<'py, PyAny>>, - pub(crate) recorder: ArtifactRecorder<'a>, -} - /// A test needing a fresh artifact, with its cache slot and artifact identity when caching. type PendingTest = (Py, Option, Option); @@ -379,7 +373,7 @@ pub(crate) struct PlannedStaticSqlTests { } /// `zip(pending, artifacts, strict=True)`. -fn zipped( +pub(crate) fn zipped( planned: &PlannedStaticSqlTests, ) -> PyResult> { if planned.pending.len() != planned.artifacts.len() { @@ -519,7 +513,7 @@ pub(crate) fn plan_tests<'py>( count_stage(py, SQL_TEST_MODEL_CHAINS, 1)?; let chains: Bound<'py, PyAny> = sql_test_model_chain_names_by_key(project, &tests)?.into_any(); - crate::bindings::_helpers::compile_delta::report_hooks::record_test_chains( + crate::compile_delta::main::report_hooks::record_test_chains::record_test_chains( &tests, &chains, )?; Some((context, chains)) @@ -582,7 +576,7 @@ pub(crate) fn plan_tests<'py>( } /// The planned tests handed over by `planned_tests()`, when they target `target_dir`. -fn resolved_plan<'py>( +pub(crate) fn resolved_plan<'py>( target_dir: &Bound<'py, PyAny>, (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), prepared: Prepared<'_, 'py>, @@ -610,7 +604,7 @@ fn resolved_plan<'py>( } /// The planning errors of one test's artifact, as SQL test diagnostics. -fn artifact_diagnostics<'py>( +pub(crate) fn artifact_diagnostics<'py>( test: &Bound<'py, PyAny>, artifact: &PlanRow, ) -> PyResult>> { @@ -937,30 +931,6 @@ fn failed_relative(failed_dir: &Path, path: &Path) -> String { } } -/// Delete `target/compiled-failed/`, which only the latest compile whose Rules failed owns. -pub(crate) fn remove_failed_compile(target: &Path) -> PyResult<()> { - match std::fs::remove_dir_all(target.join(FAILED_COMPILED_DIRECTORY_NAME)) { - Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error.into()), - _ => Ok(()), - } -} - -/// Plan uncached SQL tests and report their errors without writing artifacts or cache. -pub(crate) fn planning_diagnostics<'py>( - target_dir: &Bound<'py, PyAny>, - (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - prepared: Prepared<'_, 'py>, -) -> PyResult> { - let py: Python<'py> = project.py(); - let planned: Bound<'py, PlannedStaticSqlTests> = - resolved_plan(target_dir, (adapter, project), prepared)?; - let mut diagnostics: Vec> = Vec::new(); - for ((test, _, _), artifact) in zipped(planned.get())? { - diagnostics.extend(artifact_diagnostics(test.bind(py), artifact)?); - } - PyTuple::new(py, diagnostics) -} - /// Publish staged files with the same unchanged-file and stale-file semantics as a write. pub(crate) fn publish_staged_target<'py>( (prepared, expected_files): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -1033,7 +1003,7 @@ pub(crate) fn staging_changed(py: Python<'_>, message: &str) -> PyResult /// Write offline compiled output files under `target_dir`. #[pyfunction] #[pyo3(signature = (**options))] -fn write_static_compile_target<'py>( +pub(crate) fn write_static_compile_target<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -1046,29 +1016,3 @@ fn write_static_compile_target<'py>( Prepared::default(), ) } - -/// Write the planner's compiled output files under `target_dir`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn write_compile_target<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "write_compile_target", - options, - (&["target_dir", "adapter", "plan_output"], &["manifest"]), - )?; - write_planned_target( - &arguments.value("target_dir")?, - &arguments.value("adapter")?, - &arguments.value("plan_output")?, - &arguments.value("manifest")?, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(write_static_compile_target, module)?)?; - module.add_function(wrap_pyfunction!(write_compile_target, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/test_artifact_cache.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/test_artifact_cache.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/getters.rs similarity index 68% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/getters.rs index 20d024226c..5bb4e66763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/getters.rs @@ -3,35 +3,12 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; -use crate::bindings::_helpers::compile_command::prepared_artifacts::PreparedCompileArtifacts; -use crate::bindings::_helpers::compile_command::reuse_recording::{ - CompileArtifactWrites, RecordedCompileOutput, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::compile_command::_helpers::artifacts::prepared_artifacts::PreparedCompileArtifacts; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::CompileReuseAttempt; +use crate::compile_command::models::RecordedCompileOutput; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_compile_reuse_attempt, - }, - GetterEntry { - class: class_of::, - read: read_prepared_compile_artifacts, - }, - GetterEntry { - class: class_of::, - read: read_compile_artifact_writes, - }, - GetterEntry { - class: class_of::, - read: read_recorded_compile_output, - }, -]; - -fn read_compile_reuse_attempt<'py>( +pub(crate) fn read_compile_reuse_attempt<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -56,7 +33,7 @@ fn read_compile_reuse_attempt<'py>( }) } -fn read_prepared_compile_artifacts<'py>( +pub(crate) fn read_prepared_compile_artifacts<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -73,7 +50,7 @@ fn read_prepared_compile_artifacts<'py>( }) } -fn read_compile_artifact_writes<'py>( +pub(crate) fn read_compile_artifact_writes<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -92,7 +69,7 @@ fn read_compile_artifact_writes<'py>( }) } -fn read_recorded_compile_output<'py>( +pub(crate) fn read_recorded_compile_output<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/mod.rs new file mode 100644 index 0000000000..49b063ff62 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod getters; +pub(crate) mod report; +pub(crate) mod reuse_attempt; +pub(crate) mod reuse_recording; +pub(crate) mod runtime_identity; +pub(crate) mod startup_inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs index 5fdcc2c834..096a2f2425 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs @@ -1,10 +1,10 @@ //! The `sqb compile` report: human text, the JSON report, and the JSON report of a stopped compile. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyDict, PyDictMethods, PyList, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods, }; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_core::cli_text::main::format_compile_text::format_compile_text as compile_text; use sqlbuild_core::cli_text::main::semantic_coverage_notice::semantic_coverage_notice as notice_text; @@ -13,13 +13,11 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; +use crate::compile_command::_helpers::artifacts::surrogates::SurrogateText; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::emit_json_report::emit_json_report; -use sqlbuild_python_base::compile_outputs::main::emit_orjson_report::emit_orjson_report; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -39,7 +37,7 @@ const MODEL_TYPE: &str = "model"; const SEED_TYPE: &str = "seed"; const FUNCTION_TYPES: [&str; 2] = ["udf", "table_fn"]; const SQL_TEST_TYPE: &str = "sql_test"; -const ERROR_SEVERITY: &str = "error"; +pub(crate) const ERROR_SEVERITY: &str = "error"; const WARNING_SEVERITY: &str = "warning"; const RICH_LINEAGE: &str = "rich"; const NO_LINEAGE: &str = "none"; @@ -319,7 +317,7 @@ fn shown_diagnostics( return Ok(Vec::new()); } let texts: Bound<'_, PyDict> = source_texts(&inputs.project)?; - crate::bindings::_helpers::compile_delta::report_hooks::complete_source_texts( + crate::compile_delta::main::report_hooks::complete_source_texts::complete_source_texts( &inputs.project, &inputs.diagnostics, &texts, @@ -387,7 +385,7 @@ pub(crate) fn format_compile_text<'py>( }); } let models: Vec = - crate::bindings::_helpers::compile_delta::report_hooks::text_model_lines(models); + crate::compile_delta::main::report_hooks::text_model_lines::text_model_lines(models); let lineage: LineageNote = if !inputs.lineage.is_none() && inputs.lineage_mode == RICH_LINEAGE { LineageNote::Rich } else if inputs.lineage.is_none() @@ -436,7 +434,7 @@ fn coverage_notice( Ok(notice_text(&reasons)) } -fn dict<'py>( +pub(crate) fn dict<'py>( py: Python<'py>, entries: Vec<(&str, Bound<'py, PyAny>)>, ) -> PyResult> { @@ -447,12 +445,15 @@ fn dict<'py>( Ok(built) } -fn any<'py, T: IntoPyObjectExt<'py>>(py: Python<'py>, value: T) -> PyResult> { +pub(crate) fn any<'py, T: IntoPyObjectExt<'py>>( + py: Python<'py>, + value: T, +) -> PyResult> { value.into_bound_py_any(py) } /// The installed SQLBuild version, or `unknown`. -fn sqlbuild_version(py: Python<'_>) -> PyResult> { +pub(crate) fn sqlbuild_version(py: Python<'_>) -> PyResult> { match imported(py, METADATA_MODULE, "version")?.call1((PACKAGE_NAME,)) { Ok(version) => Ok(version), Err(error) @@ -483,13 +484,8 @@ fn location_json<'py>(location: &Bound<'py, PyAny>) -> PyResult(diagnostic: &Bound<'py, PyAny>) -> PyResult> { - diagnostic_json(diagnostic) -} - /// `_diagnostic_to_json`: one diagnostic's JSON fields, omitting the unset ones. -fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = diagnostic.py(); let payload: Bound<'py, PyDict> = dict( py, @@ -954,16 +950,7 @@ pub(crate) fn format_compile_json( compile_json(py, options, None::) -> PyResult<()>>) } -/// `format_compile_json`, first handing the selected models' partial-check notice to `notice`. -pub(crate) fn format_compile_json_with_notice<'py>( - py: Python<'py>, - options: Option>, - notice: impl FnOnce(Bound<'py, PyString>) -> PyResult<()>, -) -> PyResult { - compile_json(py, options, Some(notice)) -} - -fn compile_json<'py>( +pub(crate) fn compile_json<'py>( py: Python<'py>, options: Option>, notice: Option) -> PyResult<()>>, @@ -1089,61 +1076,3 @@ fn compile_json<'py>( )?; emit_json_report(report.as_any()) } - -/// A compile that stopped on a raised error, as one JSON diagnostic report. -#[pyfunction] -#[pyo3(signature = (*, code, message, help_text, location=None))] -pub(crate) fn format_compile_error_json<'py>( - code: Bound<'py, PyAny>, - message: Bound<'py, PyAny>, - help_text: Bound<'py, PyAny>, - location: Option>, -) -> PyResult { - let py: Python<'py> = code.py(); - let location: Bound<'py, PyAny> = location.unwrap_or_else(|| py.None().into_bound(py)); - let (path, line, column) = if location.is_none() { - (location.clone(), location.clone(), location.clone()) - } else { - ( - native_field(&location, "path")?, - native_field(&location, "line")?, - native_field(&location, "column")?, - ) - }; - let diagnostic: Bound<'py, PyAny> = new_record_with( - py, - RecordId::CompilerDiagnostic, - &[ - ("phase", any(py, "compile")?), - ("severity", any(py, ERROR_SEVERITY)?), - ("code", code), - ("message", message), - ("path", path), - ("line", line), - ("column", column), - ("location", location), - ("help", help_text), - ], - )?; - let report: Bound<'py, PyDict> = dict( - py, - vec![ - ("version", sqlbuild_version(py)?), - ("command", any(py, "compile")?), - ("offline", any(py, true)?), - ("has_errors", any(py, true)?), - ("stopped", any(py, true)?), - ( - "diagnostics", - PyList::new(py, [diagnostic_json(&diagnostic)?])?.into_any(), - ), - ], - )?; - emit_orjson_report(report.as_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_compile_text, module)?)?; - module.add_function(wrap_pyfunction!(format_compile_json, module)?)?; - module.add_function(wrap_pyfunction!(format_compile_error_json, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs index dd5044785d..0fd4d7dae3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs @@ -1,13 +1,12 @@ //! Check whether a compile can replay the stored result, replay it on a hit and leave a launch record. +use crate::compile_command::main::invocation::reuse_debug::reuse_debug; use std::path::PathBuf; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction}; use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_cli::replay::main::output_file_identities::output_file_identities; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; @@ -16,32 +15,56 @@ use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; -use crate::bindings::_helpers::compile_command::constants::{ - COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, COMPILE_COMMAND_NAME, - COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, - EXCLUDED_ROOT_ATTRIBUTE, EXCLUDED_ROOT_DIRECTORIES, MILLISECONDS_PER_SECOND, - MISSING_ENVIRONMENT_VALUE, MISSING_FILE_DIGEST, MISSING_PATH_MTIME_NS, - NATIVE_REUSE_DIRECTORY_NAME, PRESENCE_ONLY_FILE_SUFFIXES, PROJECT_CONFIG_FILENAMES, - PROJECT_ROOT_PATH_MTIME_NS, RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS, - RETIRED_REUSE_DIRECTORY_NAMES, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_DISABLE_ENV_VAR, - REUSE_DISABLE_VALUE, REUSE_HIT_MESSAGE, REUSE_HIT_TIMING, REUSE_LOGGER_NAME, - REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, REUSE_MISS_TIMING, TARGET_DIRECTORY_NAME, - TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, -}; -use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; -use crate::bindings::_helpers::compile_command::runtime_identity::{ - invocation_identity, runtime_identity, -}; -use crate::bindings::_helpers::compile_delta::driver::{ - self, DELTA_VERIFY_ENV_VAR, DeltaOutcome, delta_enabled, env_enabled, -}; -use crate::bindings::_helpers::compile_delta::verify; -use crate::bindings::_helpers::compile_driver::command::run_compile_command; -use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; -use crate::bindings::_helpers::compile_outputs::reuse::{ - ReuseCheckPy, ReuseRulesPy, check_compile_reuse, -}; -use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::compile_command::_helpers::invocation::runtime_identity::invocation_identity; +use crate::compile_command::_helpers::invocation::runtime_identity::runtime_identity; +use crate::compile_command::constants::COMPILE_CACHE_DISABLE_ENV_VAR; +use crate::compile_command::constants::COMPILE_CACHE_DISABLE_VALUE; +use crate::compile_command::constants::COMPILE_COMMAND_NAME; +use crate::compile_command::constants::COMPILED_DIRECTORY_NAME; +use crate::compile_command::constants::COMPUTE_LOG_DIRECTORY_NAME; +use crate::compile_command::constants::EXCLUDED_DIRECTORIES; +use crate::compile_command::constants::MILLISECONDS_PER_SECOND; +use crate::compile_command::constants::MISSING_ENVIRONMENT_VALUE; +use crate::compile_command::constants::MISSING_FILE_DIGEST; +use crate::compile_command::constants::MISSING_PATH_MTIME_NS; +use crate::compile_command::constants::NATIVE_REUSE_DIRECTORY_NAME; +use crate::compile_command::constants::PRESENCE_ONLY_FILE_SUFFIXES; +use crate::compile_command::constants::PROJECT_CONFIG_FILENAMES; +use crate::compile_command::constants::PROJECT_ROOT_PATH_MTIME_NS; +use crate::compile_command::constants::RETIRED_REUSE_DIRECTORY_NAMES; +use crate::compile_command::constants::REUSE_BYPASS_TIMING; +use crate::compile_command::constants::REUSE_CHECK_TIMING; +use crate::compile_command::constants::REUSE_DISABLE_ENV_VAR; +use crate::compile_command::constants::REUSE_DISABLE_VALUE; +use crate::compile_command::constants::REUSE_HIT_MESSAGE; +use crate::compile_command::constants::REUSE_HIT_TIMING; +use crate::compile_command::constants::REUSE_LOGGER_NAME; +use crate::compile_command::constants::REUSE_MAX_ENTRY_BYTES; +use crate::compile_command::constants::REUSE_MAX_STORED_ENTRIES; +use crate::compile_command::constants::REUSE_MISS_TIMING; +use crate::compile_command::constants::TARGET_DIRECTORY_NAME; +use crate::compile_command::constants::TOTAL_TIMING; +use crate::compile_command::constants::TRACKED_ENVIRONMENT_PREFIXES; +use crate::compile_command::constants::UNTRACKED_ENVIRONMENT_NAMES; +use crate::compile_command::models::CompileReuseAttempt; +use crate::compile_delta::constants::DELTA_VERIFY_ENV_VAR; +use crate::compile_delta::constants::MISMATCH_EXIT_CODE; +use crate::compile_delta::main::driver::commit::commit; +use crate::compile_delta::main::driver::env_enabled::env_enabled; +use crate::compile_delta::main::driver::run::run; +use crate::compile_delta::main::driver::trace::trace; +use crate::compile_delta::main::verify::compiled_digests::compiled_digests; +use crate::compile_delta::main::verify::hold_expected::hold_expected; +use crate::compile_delta::main::verify::write_mismatch::write_mismatch; +use crate::compile_delta::models::DeltaOutcome; +use crate::compile_delta::models::Expected; +use crate::compile_driver::main::stages::elapsed_ms::elapsed_ms; +use crate::compile_driver::main::stages::monotonic::monotonic; +use crate::compile_driver::main::stages::run_compile_command::run_compile_command; +use crate::compile_outputs::main::check_compile_reuse::check_compile_reuse; +use crate::compile_outputs::models::ReuseCheckPy; +use crate::compile_outputs::models::ReuseRulesPy; +use crate::frontier::main::cache_directory::cache_directory; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -60,25 +83,20 @@ use sqlbuild_python_front::discovery::main::project::warn_deprecated_settings::w use sqlbuild_python_front::macro_bridge::main::keep_delta_claim::keep_delta_claim; const REUSE_REQUEST: FacadeHost = FacadeHost::CompileReuseRequest; -const HIT: &str = "hit"; +pub(crate) const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; -const MISS: &str = "miss"; +pub(crate) const MISS: &str = "miss"; /// The logging format that prints its one argument as given. -const LOG_TEXT: &str = "%s"; +pub(crate) const LOG_TEXT: &str = "%s"; const BYPASS: &str = "bypass"; fn reuse_logger(py: Python<'_>) -> PyResult> { imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } -/// Log one message at debug level on the compile reuse logger. -pub(crate) fn reuse_debug(py: Python<'_>, message: &str) -> PyResult<()> { - reuse_debug_with(py, LOG_TEXT, message) -} - /// Log `argument` through the logging `format` at debug level on the compile reuse logger. -fn reuse_debug_with<'py>( +pub(crate) fn reuse_debug_with<'py>( py: Python<'py>, format: &str, argument: impl pyo3::IntoPyObject<'py>, @@ -240,7 +258,7 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { } } -fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { +pub(crate) fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { name.into_bound_py_any(py) } @@ -365,7 +383,9 @@ fn replayed<'py>( /// Replay the stored compile when every input matches, otherwise prepare to store one. #[pyfunction] #[pyo3(signature = (*, request))] -fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn attempt_compile_reuse<'py>( + request: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = request.py(); let started: f64 = monotonic(py)?; let requested: Bound<'py, PyAny> = native_field(request, "project_dir")?; @@ -425,7 +445,7 @@ fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult(request: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn reuse_request<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); let flags: Bound<'py, PyAny> = native_field(request, "profile_flags")?; let mut profiling: bool = false; @@ -465,28 +485,8 @@ fn reuse_request<'py>(request: &Bound<'py, PyAny>) -> PyResult call_with(&host_entry(py, REUSE_REQUEST)?, &fields) } -/// Run one compile command request, replaying the previous compile when nothing changed. -#[pyfunction] -pub(crate) fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = request.py(); - let attempt: Bound<'py, PyAny> = attempt_compile_reuse(&reuse_request(request)?)?; - let exit_code: Bound<'py, PyAny> = native_field(&attempt, "exit_code")?; - if native_field(&attempt, "outcome")?.eq(outcome(py, HIT)?)? && !exit_code.is_none() { - return Ok(exit_code); - } - let missed: bool = native_field(&attempt, "outcome")?.eq(outcome(py, MISS)?)?; - let code: PyResult = if missed && delta_enabled() { - compiled_with_delta(request, &attempt) - } else { - run_compile_command(py, request.clone(), Some(attempt)) - }; - reset(); - verify::disarm(); - code.and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py)) -} - /// Compile the changed files as a delta when it can, otherwise (or to verify it) in full. -fn compiled_with_delta<'py>( +pub(crate) fn compiled_with_delta<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, ) -> PyResult { @@ -497,7 +497,7 @@ fn compiled_with_delta<'py>( if let Some(projector) = &projector { projector.get().hold_lines(); } - let ran: PyResult> = driver::run(request, attempt); + let ran: PyResult> = run(request, attempt); let held: Vec = projector .as_ref() .map(|projector| projector.get().release_lines()) @@ -528,8 +528,8 @@ fn compiled_with_delta<'py>( write_held()?; let (files, models) = (finished.changed_files, finished.rerun_models); let committing: std::time::Instant = std::time::Instant::now(); - let code: i64 = driver::commit(*finished, (request, attempt))?; - driver::trace(py, "delta record and print", committing)?; + let code: i64 = commit(*finished, (request, attempt))?; + trace(py, "delta record and print", committing)?; let seconds: f64 = elapsed_ms(py, started)? as f64 / MILLISECONDS_PER_SECOND; print_stderr( py, @@ -556,33 +556,34 @@ fn compiled_with_delta<'py>( start_capture(); return run_compile_command(py, request.clone(), Some(attempt.clone())); } - DeltaOutcome::Finished(finished) => verify::hold_expected(verify::Expected { + DeltaOutcome::Finished(finished) => hold_expected(Expected { stdout: Some(finished.stdout.clone()), exit_code: Some(finished.phases.0.exit_code), artifacts_written: finished.phases.0.artifacts_written, raised: None, - compiled: verify::compiled_digests(&compiled_root), + compiled: compiled_digests(&compiled_root), compiled_root: compiled_root.to_string_lossy().into_owned(), - failed: verify::compiled_digests(&failed_root), + failed: compiled_digests(&failed_root), failed_root: failed_root.to_string_lossy().into_owned(), }), - DeltaOutcome::Raised(error) => verify::hold_expected(verify::Expected { + DeltaOutcome::Raised(error) => hold_expected(Expected { raised: Some(error.to_string()), compiled_root: compiled_root.to_string_lossy().into_owned(), failed_root: failed_root.to_string_lossy().into_owned(), - ..verify::Expected::default() + ..Expected::default() }), } start_capture(); let full: PyResult = run_compile_command(py, request.clone(), Some(attempt.clone())); let raised: Option = full.as_ref().err().map(ToString::to_string); - let differences: Vec = verify::differences(raised); + let differences: Vec = + crate::compile_delta::main::verify::differences::differences(raised); if differences.is_empty() { print_stderr(py, &"delta compile verified".into_bound_py_any(py)?)?; return full; } let cache_dir: PathBuf = cache.extract()?; - verify::write_mismatch(&cache_dir, &differences); + write_mismatch(&cache_dir, &differences); for difference in &differences { print_stderr( py, @@ -590,16 +591,5 @@ fn compiled_with_delta<'py>( )?; } full?; - Ok(verify::MISMATCH_EXIT_CODE) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py: Python<'_> = module.py(); - module.add(RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS)?; - module.add( - EXCLUDED_ROOT_ATTRIBUTE, - PyFrozenSet::new(py, EXCLUDED_ROOT_DIRECTORIES)?, - )?; - module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?)?; - module.add_function(wrap_pyfunction!(run_compile, module)?) + Ok(MISMATCH_EXIT_CODE) } diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs new file mode 100644 index 0000000000..e75a86c9ad --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs @@ -0,0 +1,327 @@ +//! What one full compile printed and left in each artifact, and storing it for compile reuse. + +use crate::compile_command::main::invocation::print_to::print_to; +use crate::compile_command::main::invocation::reuse_debug::reuse_debug; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::CompileFacts; +use crate::compile_command::types::ArtifactRecorder; +use sqlbuild_cache::project_reuse::types::ImageSections; +use std::path::{Path, PathBuf}; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::types::{PyList, PyListMethods, PyString}; + +use sqlbuild_cache::project_reuse::models::RecordedArtifact; +use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; +use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; + +use crate::compile_command::_helpers::invocation::reuse_attempt::reuse_rules; +use crate::compile_command::constants::BYTES_PER_MEBIBYTE; +use crate::compile_command::constants::REUSE_LOGGER_NAME; +use crate::compile_command::constants::REUSE_STORE_DONE_PREFIX; +use crate::compile_command::constants::REUSE_STORE_NOTICE_BYTES; +use crate::compile_command::constants::REUSE_STORE_NOTICE_PATHS; +use crate::compile_command::constants::REUSE_STORE_SKIPPED_PREFIX; +use crate::compile_outputs::main::artifact_digest::artifact_digest; +use crate::compile_outputs::models::CompileRecordPy; +use crate::compile_outputs::models::NativeReuseAttempt; +use crate::compile_outputs::models::SettingsRow; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +pub(crate) const MISS_OUTCOME: &str = "miss"; +pub(crate) const HIT_OUTCOME: &str = "hit"; +pub(crate) const BYPASS_OUTCOME: &str = "bypass"; +const STORED: &str = "stored"; +const SKIPPED: &str = "skipped"; + +/// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. +pub(crate) fn absolute(path: &Bound<'_, PyAny>) -> PyResult { + absolute_path(path.py(), &path.extract::()?) +} + +/// `os.path.abspath(path)` of a native path. +fn absolute_path(py: Python<'_>, path: &Path) -> PyResult { + if cfg!(unix) + && let Some(found) = native_absolute(path) + { + return Ok(found); + } + imported(py, "os.path", "abspath")? + .call1((path,))? + .extract() +} + +fn native_absolute(path: &Path) -> Option { + let Ok(cwd) = std::env::current_dir() else { + return None; + }; + Some(posix_abspath(cwd.to_str()?, path.to_str()?)) +} + +/// Note the exact bytes an artifact holds after a write or an unchanged-file check. +pub(crate) fn artifact_written( + recorder: ArtifactRecorder<'_>, + path: &Bound<'_, PyAny>, + contents: &[u8], +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute(path)?, + RecordedArtifact::Written { + digest: artifact_digest(contents), + }, + ); + Ok(()) +} + +/// `artifact_written` for a native path. +pub(crate) fn artifact_written_at( + py: Python<'_>, + recorder: ArtifactRecorder<'_>, + path: &Path, + contents: &[u8], +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute_path(py, path)?, + RecordedArtifact::Written { + digest: artifact_digest(contents), + }, + ); + Ok(()) +} + +/// Note an earlier artifact kept as-is after its size and mtime were validated. +pub(crate) fn artifact_kept( + py: Python<'_>, + recorder: ArtifactRecorder<'_>, + path: &Path, + (size, mtime_ns): (u64, i64), +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute_path(py, path)?, + RecordedArtifact::Kept { size, mtime_ns }, + ); + Ok(()) +} + +/// Note that one staged artifact was published at another path. +pub(crate) fn artifact_moved( + recorder: ArtifactRecorder<'_>, + source: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder + .get() + .move_one(&absolute(source)?, absolute(destination)?); + Ok(()) +} + +/// Note that a staged artifact directory was published at another directory. +pub(crate) fn artifact_tree_moved( + recorder: ArtifactRecorder<'_>, + source: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder + .get() + .move_tree(&absolute(source)?, &absolute(destination)?); + Ok(()) +} + +/// The stderr lines and stdout report one compile printed, kept as the Python text it printed. +pub(crate) type PrintedOutput = (Vec>, Option>); + +pub(crate) fn reuse_logger(py: Python<'_>) -> PyResult> { + imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) +} + +/// Loaded module files, then the macro store's extra module paths, as Python lists them. +fn module_paths(py: Python<'_>) -> PyResult> { + let paths: Bound<'_, PyList> = PyList::empty(py); + let modules: Bound<'_, PyAny> = py.import("builtins")?.getattr("tuple")?.call1((py + .import("sys")? + .getattr("modules")? + .call_method0("values")?,))?; + let getattr: Bound<'_, PyAny> = py.import("builtins")?.getattr("getattr")?; + for module in modules.try_iter()? { + let path: Bound<'_, PyAny> = getattr.call1((module?, "__file__", py.None()))?; + if path.is_instance_of::() { + paths.append(path)?; + } + } + let extra: Bound<'_, PyAny> = py + .import("builtins")? + .getattr("sorted")? + .call1((host_entry(py, UserCodeHost::MacroStoreModulePaths)?.call0()?,))?; + for path in extra.try_iter()? { + let path: Bound<'_, PyAny> = path?; + if path.is_instance_of::() { + paths.append(path)?; + } + } + Ok(paths) +} + +/// The notice a long reuse record prints first, with when it started. +fn start_notice(py: Python<'_>, native: &Bound<'_, PyAny>) -> PyResult> { + let pending: u64 = native.call_method0("pending_digest_bytes")?.extract()?; + let paths: usize = native.call_method0("snapshot_paths")?.extract()?; + if pending < REUSE_STORE_NOTICE_BYTES && paths < REUSE_STORE_NOTICE_PATHS { + return Ok(None); + } + let mebibytes: f64 = pending as f64 / BYTES_PER_MEBIBYTE; + let message: String = + format!("Recording compile for reuse ({paths} paths, {mebibytes:.0} MiB to hash)..."); + print_to(py, "stderr", &message.into_bound_py_any(py)?)?; + Ok(Some(imported(py, "time", "monotonic")?.call0()?.extract()?)) +} + +fn finish_notice(py: Python<'_>, started: Option, prefix: &str) -> PyResult<()> { + let Some(started) = started else { + return Ok(()); + }; + let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; + let seconds: f64 = now - started; + print_to( + py, + "stderr", + &format!("{prefix}{seconds:.1} s)").into_bound_py_any(py)?, + ) +} + +/// Store a reusable compile natively; returns whether it was stored. +pub(crate) fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { + let py: Python<'py> = facts.native.py(); + let settings: Bound<'_, PyAny> = call_with( + &host_entry(py, FacadeHost::ProviderSettingsInputs)?, + &[( + "settings_classes", + native_field(&facts.input_reads, "settings_classes")?, + )], + )?; + let unsupported: Bound<'_, PyAny> = native_field(&settings, "unsupported_reason")?; + if !unsupported.is_none() { + let _ = reuse_logger(py)?.call_method1( + "debug", + ("Compile reuse is off for this project: %s", &unsupported), + )?; + } + let read_run_id: Bound<'_, PyAny> = native_field(&facts.input_reads, "read_run_id")?; + let stdout: Bound<'_, PyAny> = native_field(&facts.output, "stdout")?; + let eligible: bool = !stdout.is_none() + && facts.compile_cache_enabled + && !read_run_id.is_truthy()? + && unsupported.is_none(); + let notice_started: Option = if eligible { + start_notice(py, &facts.native)? + } else { + None + }; + let strings = |value: Bound<'py, PyAny>| -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() + }; + let settings_inputs: Option> = if unsupported.is_none() { + let mut rows: Vec = Vec::new(); + for item in native_field(&settings, "inputs")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + rows.push(SettingsRow { + case_sensitive: native_field(&item, "case_sensitive")?.extract()?, + names: strings(native_field(&item, "names")?)?, + prefixes: strings(native_field(&item, "prefixes")?)?, + env_files: strings(native_field(&item, "env_files")?)?, + secrets_dirs: strings(native_field(&item, "secrets_dirs")?)?, + }); + } + Some(rows) + } else { + None + }; + let dag_path: Option = if facts.dag_artifact_path.is_none() { + None + } else { + Some( + imported(py, "os.path", "abspath")? + .call1((&facts.dag_artifact_path,))? + .extract()?, + ) + }; + let payload = CompileRecordPy { + stdout: stdout.extract()?, + stderr_lines: strings(native_field(&facts.output, "stderr_lines")?)?, + exit_code: facts.exit_code.extract()?, + compile_cache_enabled: facts.compile_cache_enabled, + read_run_id: read_run_id.extract()?, + settings_inputs, + template_environment_names: strings(native_field( + &facts.input_reads, + "environment_names", + )?)?, + module_paths: module_paths(py)?.extract()?, + artifacts: facts + .artifact_writes + .extract::>()? + .artifact_rows(), + artifacts_written: facts.artifacts_written, + dag_path, + json_output: facts.json_output.extract()?, + rules: reuse_rules(py)?, + }; + if let Some(mut capture) = + sqlbuild_python_base::compile_context::main::delta::take_capture::take_capture() + { + capture.environment_names = + native_field(&facts.input_reads, "environment_names")?.extract()?; + let json_output: bool = facts.json_output.is_truthy()?; + match crate::compile_delta::models::ProjectImage::from_capture(capture, json_output) { + Ok(image) => { + let mut pending: Option = None; + if let Ok(sections) = image.encode() { + pending = Some(sections); + } + sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(pending); + } + Err(reason) => { + reuse_debug( + py, + &format!("Compile reuse kept no project image: {reason}"), + )?; + } + } + } + let (outcome, digested): (&str, usize) = facts + .native + .extract::>()? + .record(py, payload)?; + let _ = reuse_logger(py)?.call_method1( + "debug", + ("Compile reuse record digested %d project files", digested), + )?; + finish_notice( + py, + notice_started, + if outcome == SKIPPED { + REUSE_STORE_SKIPPED_PREFIX + } else { + REUSE_STORE_DONE_PREFIX + }, + )?; + Ok(outcome == STORED) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/runtime_identity.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/runtime_identity.rs index b2cbf7d862..fe407c6be0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/runtime_identity.rs @@ -1,11 +1,12 @@ //! Interpreter, native build and invocation facts that decide whether a stored compile is reusable. +use crate::compile_command::main::invocation::interpreter_values::interpreter_values; use std::collections::BTreeMap; use std::ffi::OsString; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -28,23 +29,9 @@ const REQUEST_CHOICES: [&str; 9] = [ "cli_vars", ]; -/// `sys.version`, `sys.executable`, `sys.platform` and `str(sys.implementation.cache_tag)`. -pub(crate) fn interpreter_values(py: Python<'_>) -> PyResult<[String; 4]> { - let sys: Bound<'_, PyModule> = py.import("sys")?; - Ok([ - native_field(&sys, "version")?.extract()?, - native_field(&sys, "executable")?.extract()?, - native_field(&sys, "platform")?.extract()?, - native_field(&sys, "implementation")? - .getattr("cache_tag")? - .str()? - .extract()?, - ]) -} - /// The interpreter facts every interpreter-sensitive cache key stamps, in one order. #[pyfunction] -fn interpreter_identity(py: Python<'_>) -> PyResult> { +pub(crate) fn interpreter_identity(py: Python<'_>) -> PyResult> { PyTuple::new(py, interpreter_values(py)?) } @@ -107,7 +94,3 @@ pub(crate) fn invocation_identity<'py>( .call((payload,), Some(&options))?; text.call_method1("encode", ("utf-8", "surrogateescape")) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(interpreter_identity, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/startup_inputs.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs rename to crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/startup_inputs.rs index 48444d2b0e..15bf84eb93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/startup_inputs.rs @@ -3,11 +3,9 @@ use std::path::PathBuf; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyList; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cache::startup_inputs::main::startup_stamps::startup_stamps; use sqlbuild_cache::startup_inputs::models::{StartupLocations, StartupStamp}; @@ -38,7 +36,7 @@ fn stat_error(py: Python<'_>, path: PathBuf, error: &std::io::Error) -> PyErr { /// `[(path, st_mtime_ns, st_size), ...]` for every startup input path. #[pyfunction] #[pyo3(signature = (*, directories, files, prefixes))] -fn startup_input_stamps<'py>( +pub(crate) fn startup_input_stamps<'py>( directories: &Bound<'py, PyAny>, files: &Bound<'py, PyAny>, prefixes: &Bound<'py, PyAny>, @@ -62,8 +60,3 @@ fn startup_input_stamps<'py>( .collect::>()?; PyList::new(py, rows) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(startup_input_stamps, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/mod.rs new file mode 100644 index 0000000000..1f7d2c84f2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod artifacts; +pub(crate) mod invocation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python-compile/src/compile_command/constants.rs similarity index 64% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs rename to crates/sqlbuild-python-compile/src/compile_command/constants.rs index 480a828b5a..967dcd4b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/constants.rs @@ -1,41 +1,70 @@ -//! Compile reuse storage, fingerprint, environment and notice constants. +//! Public constants of compile command. +use crate::compile_command::_helpers::artifacts::prepared_artifacts::PreparedCompileArtifacts; +use crate::compile_command::_helpers::invocation::getters::read_compile_artifact_writes; +use crate::compile_command::_helpers::invocation::getters::read_compile_reuse_attempt; +use crate::compile_command::_helpers::invocation::getters::read_prepared_compile_artifacts; +use crate::compile_command::_helpers::invocation::getters::read_recorded_compile_output; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::CompileReuseAttempt; +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_delta::constants::DELTA_VERIFY_ENV_VAR; +use crate::frontier::constants::CAPTURE_DIR_ENV_VAR; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; - -use crate::bindings::_helpers::compile_delta::driver::DELTA_VERIFY_ENV_VAR; -use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; /// The directory under the compiler cache directory holding the native reuse store. pub(crate) const NATIVE_REUSE_DIRECTORY_NAME: &str = "project-reuse-v3"; + /// The reuse directories older releases kept stored compiles in. pub(crate) const RETIRED_REUSE_DIRECTORY_NAMES: [&str; 2] = ["project-reuse-v1", "project-reuse-v2"]; + pub(crate) const REUSE_MAX_ENTRY_BYTES: i64 = 512 * 1024 * 1024; + pub(crate) const REUSE_MAX_STORED_ENTRIES: i64 = 8; -pub(crate) const REUSE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_REUSE"; + +pub const REUSE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_REUSE"; + pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; + pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; -pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; + +pub const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; + /// Sorted; tracked-prefix variables that never invalidate a stored compile. -pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 3] = [ +pub const UNTRACKED_ENVIRONMENT_NAMES: [&str; 3] = [ CAPTURE_DIR_ENV_VAR, REUSE_DISABLE_ENV_VAR, DELTA_VERIFY_ENV_VAR, ]; + pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; + pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; + pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = [PROJECT_CONFIG_FILENAME, "sqlbuild_project.yml"]; + /// The `sqlbuild._native` attribute the reuse check reads its racy window from. pub(crate) const RACY_WINDOW_ATTRIBUTE: &str = "COMPILE_REUSE_RACY_WINDOW_NS"; + /// The `sqlbuild._native` attribute the reuse check reads its excluded root directories from. pub(crate) const EXCLUDED_ROOT_ATTRIBUTE: &str = "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES"; + pub(crate) const RACY_WINDOW_NS: i64 = 2_000_000_000; + pub(crate) const MISSING_PATH_MTIME_NS: i64 = -1; + pub(crate) const PROJECT_ROOT_PATH_MTIME_NS: i64 = -2; -pub(crate) const TARGET_DIRECTORY_NAME: &str = "target"; + +pub const TARGET_DIRECTORY_NAME: &str = "target"; + pub(crate) const COMPILED_DIRECTORY_NAME: &str = "compiled"; + pub(crate) const MILLISECONDS_PER_SECOND: f64 = 1000.0; + /// Sorted; top-level project directories the reuse snapshot skips. pub(crate) const EXCLUDED_ROOT_DIRECTORIES: [&str; 16] = [ ".cache", @@ -55,25 +84,41 @@ pub(crate) const EXCLUDED_ROOT_DIRECTORIES: [&str; 16] = [ "target", "venv", ]; + /// Sorted; directories the reuse snapshot skips at any depth. pub(crate) const EXCLUDED_DIRECTORIES: [&str; 2] = [".git", "__pycache__"]; + pub(crate) const PRESENCE_ONLY_FILE_SUFFIXES: [&str; 2] = [".duckdb", ".duckdb.wal"]; + pub(crate) const REUSE_HIT_TIMING: &str = "project_reuse_hits"; + pub(crate) const REUSE_MISS_TIMING: &str = "project_reuse_misses"; + pub(crate) const REUSE_BYPASS_TIMING: &str = "project_reuse_bypasses"; + pub(crate) const REUSE_CHECK_TIMING: &str = "project_reuse_check_ms"; + pub(crate) const TOTAL_TIMING: &str = "total_ms"; + /// The reuse hit notice, split around its elapsed seconds. pub(crate) const REUSE_HIT_MESSAGE: [&str; 2] = ["Inputs unchanged; reused the previous compile (", " s)"]; + pub(crate) const REUSE_STORE_NOTICE_BYTES: u64 = 64 * 1024 * 1024; + pub(crate) const REUSE_STORE_NOTICE_PATHS: usize = 100_000; + pub(crate) const BYTES_PER_MEBIBYTE: f64 = 1024.0 * 1024.0; + pub(crate) const REUSE_STORE_DONE_PREFIX: &str = "Recorded compile for reuse ("; + pub(crate) const REUSE_STORE_SKIPPED_PREFIX: &str = "Did not record compile for reuse: files changed while it ran ("; + pub(crate) const COMPUTE_LOG_DIRECTORY_NAME: &str = "logs"; -pub(crate) const COMPILE_COMMAND_NAME: &str = "compile"; + +pub const COMPILE_COMMAND_NAME: &str = "compile"; + /// The required and optional keyword fields of `CompileReuseAttempt`. pub(crate) const REUSE_ATTEMPT_FIELDS: (&[&str], &[&str]) = ( &["outcome", "project_dir", "started"], @@ -86,8 +131,31 @@ pub(crate) const REUSE_ATTEMPT_FIELDS: (&[&str], &[&str]) = ( "json_output", ], ); + /// The variable that disables the compile cache when it holds `COMPILE_CACHE_DISABLE_VALUE`. -pub(crate) const COMPILE_CACHE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_CACHE"; -pub(crate) const COMPILE_CACHE_DISABLE_VALUE: &str = "1"; +pub const COMPILE_CACHE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_CACHE"; + +pub const COMPILE_CACHE_DISABLE_VALUE: &str = "1"; + /// The fact and reference cache directories older releases kept under each retired cache root. -pub(crate) const RETIRED_CACHE_DIRECTORY_NAMES: [&str; 2] = ["facts-v1", "references-v3"]; +pub const RETIRED_CACHE_DIRECTORY_NAMES: [&str; 2] = ["facts-v1", "references-v3"]; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_compile_reuse_attempt, + }, + GetterEntry { + class: class_of::, + read: read_prepared_compile_artifacts, + }, + GetterEntry { + class: class_of::, + read: read_compile_artifact_writes, + }, + GetterEntry { + class: class_of::, + read: read_recorded_compile_output, + }, +]; diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/mod.rs new file mode 100644 index 0000000000..9817113952 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/mod.rs @@ -0,0 +1,7 @@ +pub mod planning_diagnostics; +pub mod register_prepared_artifacts; +pub mod register_target_writer; +pub mod remove_failed_compile; +pub mod write_compile_target; +pub mod write_rule_failed_models; +pub mod write_static_target; diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/planning_diagnostics.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/planning_diagnostics.rs new file mode 100644 index 0000000000..841622267a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/planning_diagnostics.rs @@ -0,0 +1,25 @@ +//! Plan uncached SQL tests and report their errors without writing artifacts or cache. + +use crate::compile_command::_helpers::artifacts::target_writer::PlannedStaticSqlTests; +use crate::compile_command::_helpers::artifacts::target_writer::artifact_diagnostics; +use crate::compile_command::_helpers::artifacts::target_writer::resolved_plan; +use crate::compile_command::_helpers::artifacts::target_writer::zipped; +use crate::compile_command::models::Prepared; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + +/// Plan uncached SQL tests and report their errors without writing artifacts or cache. +pub fn planning_diagnostics<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + prepared: Prepared<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let planned: Bound<'py, PlannedStaticSqlTests> = + resolved_plan(target_dir, (adapter, project), prepared)?; + let mut diagnostics: Vec> = Vec::new(); + for ((test, _, _), artifact) in zipped(planned.get())? { + diagnostics.extend(artifact_diagnostics(test.bind(py), artifact)?); + } + PyTuple::new(py, diagnostics) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_prepared_artifacts.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_prepared_artifacts.rs new file mode 100644 index 0000000000..defdd432e8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_prepared_artifacts.rs @@ -0,0 +1,9 @@ +//! Register the prepared artifacts functions and classes. + +use crate::compile_command::_helpers::artifacts::prepared_artifacts::PreparedCompileArtifacts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the prepared artifacts functions and classes. +pub fn register_prepared_artifacts(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_target_writer.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_target_writer.rs new file mode 100644 index 0000000000..1b8b19b6fe --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/register_target_writer.rs @@ -0,0 +1,12 @@ +//! Register the target writer functions and classes. + +use crate::compile_command::_helpers::artifacts::target_writer::write_static_compile_target; +use crate::compile_command::main::artifacts::write_compile_target::write_compile_target; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the target writer functions and classes. +pub fn register_target_writer(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_static_compile_target, module)?)?; + module.add_function(wrap_pyfunction!(write_compile_target, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/remove_failed_compile.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/remove_failed_compile.rs new file mode 100644 index 0000000000..049f36585e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/remove_failed_compile.rs @@ -0,0 +1,13 @@ +//! Delete `target/compiled-failed/`, which only the latest compile whose Rules failed owns. + +use pyo3::prelude::PyResult; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; +use std::path::Path; + +/// Delete `target/compiled-failed/`, which only the latest compile whose Rules failed owns. +pub fn remove_failed_compile(target: &Path) -> PyResult<()> { + match std::fs::remove_dir_all(target.join(FAILED_COMPILED_DIRECTORY_NAME)) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error.into()), + _ => Ok(()), + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_compile_target.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_compile_target.rs new file mode 100644 index 0000000000..1d491ef384 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_compile_target.rs @@ -0,0 +1,28 @@ +//! Write the planner's compiled output files under `target_dir`. + +use crate::compile_command::_helpers::artifacts::target_writer::write_planned_target; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Write the planner's compiled output files under `target_dir`. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn write_compile_target<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "write_compile_target", + options, + (&["target_dir", "adapter", "plan_output"], &["manifest"]), + )?; + write_planned_target( + &arguments.value("target_dir")?, + &arguments.value("adapter")?, + &arguments.value("plan_output")?, + &arguments.value("manifest")?, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_rule_failed_models.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_rule_failed_models.rs new file mode 100644 index 0000000000..979f987edd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_rule_failed_models.rs @@ -0,0 +1,23 @@ +//! After Rules failed, write every model's compiled SQL to `target/compiled-failed/` and remove every other file there, so it holds exactly the expansion these Rules checked. A model with Rule errors starts with a header marking it not buildable and listing them. `target/compiled/` keeps the last passing compile; only an earlier manifest is removed, since it no longer describes `target/`. Returns the failing models' files, relative to `target/compiled-failed/` and sorted. + +use crate::compile_command::_helpers::artifacts::target_writer; +use crate::compile_command::types::ArtifactRecorder; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// After Rules failed, write every model's compiled SQL to `target/compiled-failed/` and remove every other file there, so it holds exactly the expansion these Rules checked. A model with Rule errors starts with a header marking it not buildable and listing them. `target/compiled/` keeps the last passing compile; only an earlier manifest is removed, since it no longer describes `target/`. Returns the failing models' files, relative to `target/compiled-failed/` and sorted. +#[inline] +pub fn write_rule_failed_models<'py>( + target_dir: &Bound<'py, PyAny>, + (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + diagnostics: &Bound<'py, PyAny>, + recorder: ArtifactRecorder<'_>, +) -> PyResult> { + target_writer::write_rule_failed_models( + target_dir, + (project, project_dir), + diagnostics, + recorder, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_static_target.rs b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_static_target.rs new file mode 100644 index 0000000000..d605f2c447 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_static_target.rs @@ -0,0 +1,18 @@ +//! Write offline compiled output under `target_dir`, reusing SQL tests planned ahead. + +use crate::compile_command::_helpers::artifacts::target_writer; +use crate::compile_command::models::Prepared; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Write offline compiled output under `target_dir`, reusing SQL tests planned ahead. +#[inline] +pub fn write_static_target<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + manifest: &Bound<'py, PyAny>, + prepared: Prepared<'_, 'py>, +) -> PyResult> { + target_writer::write_static_target(target_dir, (adapter, project), manifest, prepared) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs new file mode 100644 index 0000000000..55c686d579 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs @@ -0,0 +1,28 @@ +//! The hit, miss and bypass counters plus the reuse check time, keyed as `compile_timings` is. + +use crate::compile_command::_helpers::invocation::reuse_recording::BYPASS_OUTCOME; +use crate::compile_command::_helpers::invocation::reuse_recording::HIT_OUTCOME; +use crate::compile_command::_helpers::invocation::reuse_recording::MISS_OUTCOME; +use crate::compile_command::constants::REUSE_BYPASS_TIMING; +use crate::compile_command::constants::REUSE_CHECK_TIMING; +use crate::compile_command::constants::REUSE_HIT_TIMING; +use crate::compile_command::constants::REUSE_MISS_TIMING; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The hit, miss and bypass counters plus the reuse check time, keyed as `compile_timings` is. +pub fn compile_reuse_timings<'py>(attempt: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = attempt.py(); + let outcome: String = native_field(attempt, "outcome")?.str()?.to_string(); + let timings: Bound<'py, PyDict> = PyDict::new(py); + for (name, counted) in [ + (REUSE_HIT_TIMING, HIT_OUTCOME), + (REUSE_MISS_TIMING, MISS_OUTCOME), + (REUSE_BYPASS_TIMING, BYPASS_OUTCOME), + ] { + timings.set_item(name, i64::from(outcome == counted))?; + } + timings.set_item(REUSE_CHECK_TIMING, native_field(attempt, "check_ms")?)?; + Ok(timings) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/diagnostic_item.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/diagnostic_item.rs new file mode 100644 index 0000000000..5407926419 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/diagnostic_item.rs @@ -0,0 +1,10 @@ +//! One diagnostic as the report's JSON item. + +use crate::compile_command::_helpers::invocation::report::diagnostic_json; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// One diagnostic as the report's JSON item. +pub fn diagnostic_item<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { + diagnostic_json(diagnostic) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_error_json.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_error_json.rs new file mode 100644 index 0000000000..2d323a1b45 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_error_json.rs @@ -0,0 +1,66 @@ +//! A compile that stopped on a raised error, as one JSON diagnostic report. + +use crate::compile_command::_helpers::invocation::report::ERROR_SEVERITY; +use crate::compile_command::_helpers::invocation::report::any; +use crate::compile_command::_helpers::invocation::report::diagnostic_json; +use crate::compile_command::_helpers::invocation::report::dict; +use crate::compile_command::_helpers::invocation::report::sqlbuild_version; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyList}; +use sqlbuild_python_base::compile_outputs::main::emit_orjson_report::emit_orjson_report; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// A compile that stopped on a raised error, as one JSON diagnostic report. +#[pyfunction] +#[pyo3(signature = (*, code, message, help_text, location=None))] +pub fn format_compile_error_json<'py>( + code: Bound<'py, PyAny>, + message: Bound<'py, PyAny>, + help_text: Bound<'py, PyAny>, + location: Option>, +) -> PyResult { + let py: Python<'py> = code.py(); + let location: Bound<'py, PyAny> = location.unwrap_or_else(|| py.None().into_bound(py)); + let (path, line, column) = if location.is_none() { + (location.clone(), location.clone(), location.clone()) + } else { + ( + native_field(&location, "path")?, + native_field(&location, "line")?, + native_field(&location, "column")?, + ) + }; + let diagnostic: Bound<'py, PyAny> = new_record_with( + py, + RecordId::CompilerDiagnostic, + &[ + ("phase", any(py, "compile")?), + ("severity", any(py, ERROR_SEVERITY)?), + ("code", code), + ("message", message), + ("path", path), + ("line", line), + ("column", column), + ("location", location), + ("help", help_text), + ], + )?; + let report: Bound<'py, PyDict> = dict( + py, + vec![ + ("version", sqlbuild_version(py)?), + ("command", any(py, "compile")?), + ("offline", any(py, true)?), + ("has_errors", any(py, true)?), + ("stopped", any(py, true)?), + ( + "diagnostics", + PyList::new(py, [diagnostic_json(&diagnostic)?])?.into_any(), + ), + ], + )?; + emit_orjson_report(report.as_any()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_json_with_notice.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_json_with_notice.rs new file mode 100644 index 0000000000..fc045aa78f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_json_with_notice.rs @@ -0,0 +1,17 @@ +//! `format_compile_json`, first handing the selected models' partial-check notice to `notice`. + +use crate::compile_command::_helpers::invocation::report::compile_json; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyString; + +/// `format_compile_json`, first handing the selected models' partial-check notice to `notice`. +pub fn format_compile_json_with_notice<'py>( + py: Python<'py>, + options: Option>, + notice: impl FnOnce(Bound<'py, PyString>) -> PyResult<()>, +) -> PyResult { + compile_json(py, options, Some(notice)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_text.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_text.rs new file mode 100644 index 0000000000..4c604b6686 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/format_compile_text.rs @@ -0,0 +1,17 @@ +//! The human-readable compile report. + +use crate::compile_command::_helpers::invocation::report; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyString; + +/// The human-readable compile report. +#[inline] +pub fn format_compile_text<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + report::format_compile_text(py, options) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/interpreter_values.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/interpreter_values.rs new file mode 100644 index 0000000000..086f99b778 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/interpreter_values.rs @@ -0,0 +1,18 @@ +//! `sys.version`, `sys.executable`, `sys.platform` and `str(sys.implementation.cache_tag)`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `sys.version`, `sys.executable`, `sys.platform` and `str(sys.implementation.cache_tag)`. +pub fn interpreter_values(py: Python<'_>) -> PyResult<[String; 4]> { + let sys: Bound<'_, PyModule> = py.import("sys")?; + Ok([ + native_field(&sys, "version")?.extract()?, + native_field(&sys, "executable")?.extract()?, + native_field(&sys, "platform")?.extract()?, + native_field(&sys, "implementation")? + .getattr("cache_tag")? + .str()? + .extract()?, + ]) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/mod.rs new file mode 100644 index 0000000000..3a5356ebd4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/mod.rs @@ -0,0 +1,16 @@ +pub mod compile_reuse_timings; +pub mod diagnostic_item; +pub mod format_compile_error_json; +pub mod format_compile_json_with_notice; +pub mod format_compile_text; +pub mod interpreter_values; +pub mod new_artifact_recorder; +pub mod print_to; +pub mod register_report; +pub mod register_reuse_attempt; +pub mod register_reuse_recording; +pub mod register_runtime_identity; +pub mod register_startup_inputs; +pub mod reuse_debug; +pub mod run_compile; +pub mod write_reusable_compile; diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/new_artifact_recorder.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/new_artifact_recorder.rs new file mode 100644 index 0000000000..672b957bff --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/new_artifact_recorder.rs @@ -0,0 +1,16 @@ +//! A new, empty recorder of one compile's artifact writes. + +use crate::compile_command::models::CompileArtifactWrites; +use pyo3::prelude::{Py, PyResult, Python}; +use std::collections::HashMap; +use std::sync::Mutex; + +/// A new, empty recorder of one compile's artifact writes. +pub fn new_artifact_recorder(py: Python<'_>) -> PyResult> { + Py::new( + py, + CompileArtifactWrites { + artifacts: Mutex::new(HashMap::new()), + }, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/print_to.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/print_to.rs new file mode 100644 index 0000000000..31fd7101d4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/print_to.rs @@ -0,0 +1,20 @@ +//! `print(*values, file=sys.)`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::keywords::keywords; + +/// `print(*values, file=sys.)`. +pub fn print_to<'py>( + py: Python<'py>, + stream: &str, + text: impl pyo3::IntoPyObject<'py>, +) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = + keywords(py, &[("file", py.import("sys")?.getattr(stream)?)])?; + let _ = py + .import("builtins")? + .getattr("print")? + .call((text,), Some(&keywords))?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs new file mode 100644 index 0000000000..a05eb2bdcb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs @@ -0,0 +1,14 @@ +//! Register the report functions and classes. + +use crate::compile_command::_helpers::invocation::report::format_compile_json; +use crate::compile_command::_helpers::invocation::report::format_compile_text; +use crate::compile_command::main::invocation::format_compile_error_json::format_compile_error_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the report functions and classes. +pub fn register_report(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_compile_text, module)?)?; + module.add_function(wrap_pyfunction!(format_compile_json, module)?)?; + module.add_function(wrap_pyfunction!(format_compile_error_json, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_attempt.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_attempt.rs new file mode 100644 index 0000000000..ea911d36b4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_attempt.rs @@ -0,0 +1,23 @@ +//! Register the reuse attempt functions and classes. + +use crate::compile_command::_helpers::invocation::reuse_attempt::attempt_compile_reuse; +use crate::compile_command::constants::EXCLUDED_ROOT_ATTRIBUTE; +use crate::compile_command::constants::EXCLUDED_ROOT_DIRECTORIES; +use crate::compile_command::constants::RACY_WINDOW_ATTRIBUTE; +use crate::compile_command::constants::RACY_WINDOW_NS; +use crate::compile_command::main::invocation::run_compile::run_compile; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use pyo3::wrap_pyfunction; + +/// Register the reuse attempt functions and classes. +pub fn register_reuse_attempt(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + module.add(RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS)?; + module.add( + EXCLUDED_ROOT_ATTRIBUTE, + PyFrozenSet::new(py, EXCLUDED_ROOT_DIRECTORIES)?, + )?; + module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?)?; + module.add_function(wrap_pyfunction!(run_compile, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_recording.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_recording.rs new file mode 100644 index 0000000000..2e03d9b886 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_reuse_recording.rs @@ -0,0 +1,14 @@ +//! Register the reuse recording functions and classes. + +use crate::compile_command::main::invocation::write_reusable_compile::write_reusable_compile; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::RecordedCompileOutput; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the reuse recording functions and classes. +pub fn register_reuse_recording(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(write_reusable_compile, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_runtime_identity.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_runtime_identity.rs new file mode 100644 index 0000000000..c931215beb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_runtime_identity.rs @@ -0,0 +1,10 @@ +//! Register the runtime identity functions and classes. + +use crate::compile_command::_helpers::invocation::runtime_identity::interpreter_identity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the runtime identity functions and classes. +pub fn register_runtime_identity(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(interpreter_identity, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_startup_inputs.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_startup_inputs.rs new file mode 100644 index 0000000000..a19540c06f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_startup_inputs.rs @@ -0,0 +1,11 @@ +//! Register the startup inputs functions and classes. + +use crate::compile_command::_helpers::invocation::startup_inputs::startup_input_stamps; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the startup inputs functions and classes. +pub fn register_startup_inputs(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(startup_input_stamps, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/reuse_debug.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/reuse_debug.rs new file mode 100644 index 0000000000..ac82007bfe --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/reuse_debug.rs @@ -0,0 +1,10 @@ +//! Log one message at debug level on the compile reuse logger. + +use crate::compile_command::_helpers::invocation::reuse_attempt::LOG_TEXT; +use crate::compile_command::_helpers::invocation::reuse_attempt::reuse_debug_with; +use pyo3::prelude::{PyResult, Python}; + +/// Log one message at debug level on the compile reuse logger. +pub fn reuse_debug(py: Python<'_>, message: &str) -> PyResult<()> { + reuse_debug_with(py, LOG_TEXT, message) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs new file mode 100644 index 0000000000..db7e598930 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs @@ -0,0 +1,35 @@ +//! Run one compile command request, replaying the previous compile when nothing changed. + +use crate::compile_command::_helpers::invocation::reuse_attempt::HIT; +use crate::compile_command::_helpers::invocation::reuse_attempt::MISS; +use crate::compile_command::_helpers::invocation::reuse_attempt::attempt_compile_reuse; +use crate::compile_command::_helpers::invocation::reuse_attempt::compiled_with_delta; +use crate::compile_command::_helpers::invocation::reuse_attempt::outcome; +use crate::compile_command::_helpers::invocation::reuse_attempt::reuse_request; +use crate::compile_delta::main::driver::delta_enabled::delta_enabled; +use crate::compile_delta::main::verify::disarm::disarm; +use crate::compile_driver::main::stages::run_compile_command::run_compile_command; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::compile_context::main::delta::reset::reset; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Run one compile command request, replaying the previous compile when nothing changed. +#[pyfunction] +pub fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let attempt: Bound<'py, PyAny> = attempt_compile_reuse(&reuse_request(request)?)?; + let exit_code: Bound<'py, PyAny> = native_field(&attempt, "exit_code")?; + if native_field(&attempt, "outcome")?.eq(outcome(py, HIT)?)? && !exit_code.is_none() { + return Ok(exit_code); + } + let missed: bool = native_field(&attempt, "outcome")?.eq(outcome(py, MISS)?)?; + let code: PyResult = if missed && delta_enabled() { + compiled_with_delta(request, &attempt) + } else { + run_compile_command(py, request.clone(), Some(attempt)) + }; + reset(); + disarm(); + code.and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs new file mode 100644 index 0000000000..29e986472d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs @@ -0,0 +1,91 @@ +//! Store a reusable full compile, or drop the stored one when it cannot be reused. + +use crate::compile_command::_helpers::invocation::reuse_attempt::write_launch_record; +use crate::compile_command::_helpers::invocation::reuse_recording::MISS_OUTCOME; +use crate::compile_command::_helpers::invocation::reuse_recording::record; +use crate::compile_command::_helpers::invocation::reuse_recording::reuse_logger; +use crate::compile_command::models::CompileFacts; +use pyo3::exceptions::{PyException, PyOSError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Store a reusable full compile, or drop the stored one when it cannot be reused. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn write_reusable_compile(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_reusable_compile", + options, + ( + &[ + "attempt", + "output", + "exit_code", + "input_reads", + "artifact_writes", + "compile_cache_enabled", + "artifacts_written", + "dag_artifact_path", + "json_output", + ], + &[], + ), + )?; + let attempt: Bound<'_, PyAny> = arguments.value("attempt")?; + let entry_path: Bound<'_, PyAny> = native_field(&attempt, "entry_path")?; + let native: Bound<'_, PyAny> = native_field(&attempt, "native")?; + if !native_field(&attempt, "outcome")?.eq(MISS_OUTCOME)? + || entry_path.is_none() + || native.is_none() + { + return Ok(()); + } + let facts: CompileFacts<'_> = CompileFacts { + native, + output: arguments.value("output")?, + exit_code: arguments.value("exit_code")?, + input_reads: arguments.value("input_reads")?, + artifact_writes: arguments.value("artifact_writes")?, + compile_cache_enabled: arguments.flag("compile_cache_enabled")?, + artifacts_written: arguments.flag("artifacts_written")?, + dag_artifact_path: arguments.value("dag_artifact_path")?, + json_output: arguments.value("json_output")?, + }; + let stored: bool = match record(&facts) { + Ok(stored) => stored, + Err(error) if error.is_instance_of::(py) => { + let _ = reuse_logger(py)?.call_method( + "debug", + ("Compile reuse did not store this compile",), + Some(&keywords( + py, + &[("exc_info", error.value(py).clone().into_any())], + )?), + )?; + let unlinked: PyResult> = entry_path.call_method( + "unlink", + (), + Some(&keywords( + py, + &[("missing_ok", true.into_bound_py_any(py)?)], + )?), + ); + if let Err(error) = unlinked + && !error.is_instance_of::(py) + { + return Err(error); + } + false + } + Err(error) => return Err(error), + }; + if stored { + write_launch_record(&attempt)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/main/mod.rs new file mode 100644 index 0000000000..131f09fbe3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/main/mod.rs @@ -0,0 +1,2 @@ +pub mod artifacts; +pub mod invocation; diff --git a/crates/sqlbuild-python-compile/src/compile_command/mod.rs b/crates/sqlbuild-python-compile/src/compile_command/mod.rs new file mode 100644 index 0000000000..a7389b17d0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/mod.rs @@ -0,0 +1,7 @@ +//! The compile command: reuse state, prepared artifacts and the compile report. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-compile/src/compile_command/models.rs b/crates/sqlbuild-python-compile/src/compile_command/models.rs new file mode 100644 index 0000000000..156fe04f7d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/models.rs @@ -0,0 +1,382 @@ +//! Public records of compile command. + +use crate::compile_command::_helpers::artifacts::artifact_batch::PHYSICAL_WRITE_PHASE; +use crate::compile_command::_helpers::artifacts::artifact_batch::platform_lines; +use crate::compile_command::_helpers::artifacts::artifact_batch::sql_artifact; +use crate::compile_command::_helpers::invocation::reuse_recording::PrintedOutput; +use crate::compile_command::_helpers::invocation::reuse_recording::artifact_written; +use crate::compile_command::_helpers::invocation::reuse_recording::artifact_written_at; +use crate::compile_command::constants::REUSE_ATTEMPT_FIELDS; +use crate::compile_command::main::invocation::print_to::print_to; +use crate::compile_command::types::ArtifactRecorder; +use crate::compile_outputs::main::write_compiled_artifacts::write_compiled_artifacts; +use crate::compile_outputs::models::BackgroundArtifactWriter; +use crate::compile_outputs::types::ArtifactRow; +use pyo3::IntoPyObject; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyException; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use sqlbuild_cache::project_reuse::models::RecordedArtifact; +use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; +use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; +use std::sync::PoisonError; + +/// The artifacts of one write group, written together by `flush`. +#[derive(Debug)] +pub struct ArtifactBatch<'a> { + pub(crate) check_existing: bool, + pub(crate) recorder: ArtifactRecorder<'a>, + pub(crate) files: Vec<(PathBuf, Vec)>, + /// Earlier groups, still being written in the background. + pub(crate) writer: Option, +} + +impl<'a> ArtifactBatch<'a> { + pub fn new(check_existing: bool, recorder: ArtifactRecorder<'a>) -> Self { + Self { + check_existing, + recorder, + files: Vec::new(), + writer: None, + } + } + + /// Queue one artifact with platform line endings, noting its bytes for compile reuse. + pub fn queue(&mut self, path: &Bound<'_, PyAny>, contents: Vec) -> PyResult<()> { + let contents: Vec = platform_lines(contents); + artifact_written(self.recorder, path, &contents)?; + self.files.push((path.extract()?, contents)); + Ok(()) + } + + /// Queue `sql` with trailing whitespace replaced by one newline. + pub fn queue_sql(&mut self, path: &Bound<'_, PyAny>, sql: &str) -> PyResult<()> { + self.queue(path, sql_artifact(sql)) + } + + /// `queue_sql` at a native path. + pub fn queue_sql_at(&mut self, py: Python<'_>, path: PathBuf, sql: &str) -> PyResult<()> { + let contents: Vec = platform_lines(sql_artifact(sql)); + artifact_written_at(py, self.recorder, &path, &contents)?; + self.files.push((path, contents)); + Ok(()) + } + + /// Write every queued artifact natively, after any background group, and empty the queue. + pub fn flush(&mut self, py: Python<'_>) -> PyResult<()> { + self.finish_pending(py)?; + if self.files.is_empty() { + return Ok(()); + } + let files: Vec<(PathBuf, Vec)> = std::mem::take(&mut self.files); + record_compile_timing(PHYSICAL_WRITE_PHASE, || { + write_compiled_artifacts(py, files, self.check_existing).map(|_| ()) + }) + } + + /// Write the queued artifacts in the background, after every earlier background group. + pub fn flush_in_background(&mut self, py: Python<'_>) -> PyResult<()> { + if self.files.is_empty() { + return Ok(()); + } + if self.writer.is_none() { + self.writer = BackgroundArtifactWriter::start(); + } + let Some(writer) = &self.writer else { + return self.flush(py); + }; + writer.send(std::mem::take(&mut self.files), self.check_existing); + Ok(()) + } + + /// `result`, except that a background group's earlier write error wins and interrupts stop it. + pub fn settle(&mut self, py: Python<'_>, result: PyResult) -> PyResult { + let error: PyErr = match result { + Ok(value) => return Ok(value), + Err(error) => error, + }; + let Some(writer) = self.writer.take() else { + return Err(error); + }; + if !error.is_instance_of::(py) { + writer.stop(py); + return Err(error); + } + writer.finish(py)?; + Err(error) + } + + /// Wait for the background groups, raising the first write error. + pub(crate) fn finish_pending(&mut self, py: Python<'_>) -> PyResult<()> { + let Some(writer) = self.writer.take() else { + return Ok(()); + }; + let elapsed = writer.finish(py)?; + add_compile_timing( + PHYSICAL_WRITE_PHASE, + u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX), + ); + Ok(()) + } +} + +impl Drop for ArtifactBatch<'_> { + fn drop(&mut self) { + if let Some(writer) = self.writer.take() { + Python::attach(|py| writer.stop(py)); + } + } +} + +/// Result of checking for a reusable compile, carried into a full compile on a miss. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct CompileReuseAttempt { + /// `hit`, `miss` or `bypass`. + pub outcome: Py, + pub project_dir: Py, + pub started: Py, + pub check_ms: Py, + pub exit_code: Py, + pub entry_path: Py, + /// The natively checked attempt, recorded natively after a miss. + pub native: Py, + pub selected_target: Py, + pub json_output: Py, +} + +#[pymethods] +impl CompileReuseAttempt { + #[new] + #[pyo3(signature = (**options))] + pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { + let fields: KeywordArguments<'_> = + KeywordArguments::read(py, "CompileReuseAttempt", options, REUSE_ATTEMPT_FIELDS)?; + let field = |name: &str| -> PyResult> { Ok(fields.value(name)?.unbind()) }; + Ok(Self { + outcome: field("outcome")?, + project_dir: field("project_dir")?, + started: field("started")?, + check_ms: if fields.passed("check_ms")? { + field("check_ms")? + } else { + 0_i64.into_py_any(py)? + }, + exit_code: field("exit_code")?, + entry_path: field("entry_path")?, + native: field("native")?, + selected_target: field("selected_target")?, + json_output: fields.value_or_false("json_output")?.unbind(), + }) + } +} + +/// Artifacts one compile wrote or kept, by absolute path, following staged moves. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct CompileArtifactWrites { + pub(crate) artifacts: Mutex>, +} + +impl CompileArtifactWrites { + pub(crate) fn locked(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.artifacts + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) fn record(&self, path: String, artifact: RecordedArtifact) { + let _ = self.locked().insert(path, artifact); + } + + pub(crate) fn move_one(&self, source: &str, destination: String) { + let mut artifacts = self.locked(); + if let Some(artifact) = artifacts.remove(source) { + let _ = artifacts.insert(destination, artifact); + } + } + + pub(crate) fn move_tree(&self, source: &str, destination: &str) { + let prefix: String = format!("{source}{}", std::path::MAIN_SEPARATOR); + let mut artifacts = self.locked(); + let moved: Vec<(String, RecordedArtifact)> = artifacts + .iter() + .filter(|(path, _)| path.starts_with(&prefix)) + .map(|(path, artifact)| { + ( + format!("{destination}{}", &path[source.len()..]), + artifact.clone(), + ) + }) + .collect(); + artifacts.retain(|path, _| !path.starts_with(&prefix)); + artifacts.extend(moved); + } +} + +impl CompileArtifactWrites { + /// Every recorded artifact's absolute path, with its digest when this compile wrote it. + pub fn written_digests(&self) -> HashMap> { + self.locked() + .iter() + .map(|(path, artifact)| { + let digest: Option = match artifact { + RecordedArtifact::Written { digest } => Some(digest.clone()), + RecordedArtifact::Kept { .. } => None, + }; + (path.clone(), digest) + }) + .collect() + } + + /// The Python-facing artifact rows, built natively for recording or the public view. + pub(crate) fn artifact_rows(&self) -> HashMap { + self.locked() + .iter() + .map(|(path, artifact)| { + let row = match artifact { + RecordedArtifact::Written { digest } => (Some(digest.clone()), None, None), + RecordedArtifact::Kept { size, mtime_ns } => { + (None, Some(*size), Some(*mtime_ns)) + } + }; + (path.clone(), row) + }) + .collect() + } +} + +#[pymethods] +impl CompileArtifactWrites { + /// Every recorded artifact by absolute path, as `(digest, size, mtime_ns)`. + #[getter] + pub fn artifacts<'py>(&self, py: Python<'py>) -> PyResult> { + self.artifact_rows().into_pyobject(py) + } +} + +/// Compile output printed at once (notes) or after storing (the report), keeping the text. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct RecordedCompileOutput { + pub(crate) output: Mutex, +} + +impl RecordedCompileOutput { + pub(crate) fn new() -> Self { + Self { + output: Mutex::new((Vec::new(), None)), + } + } + + pub(crate) fn locked(&self) -> std::sync::MutexGuard<'_, PrintedOutput> { + self.output.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Output already printed or recorded elsewhere: its stderr lines and its report. + pub fn recorded(py: Python<'_>, stderr_lines: &[String], stdout: &str) -> Self { + let lines: Vec> = stderr_lines + .iter() + .map(|line| PyString::new(py, line).unbind()) + .collect(); + Self { + output: Mutex::new((lines, Some(PyString::new(py, stdout).unbind()))), + } + } + + /// The recorded stderr lines and report, as text. + pub fn texts(&self, py: Python<'_>) -> PyResult<(Vec, Option)> { + let output = self.locked(); + let lines: Vec = output + .0 + .iter() + .map(|line| Ok(line.bind(py).to_str()?.to_owned())) + .collect::>()?; + let stdout: Option = match &output.1 { + Some(text) => Some(text.bind(py).to_str()?.to_owned()), + None => None, + }; + Ok((lines, stdout)) + } +} + +#[pymethods] +impl RecordedCompileOutput { + /// Print one stderr result line, such as a note, and record it; a delta only records it. + pub fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { + if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() { + print_to(line.py(), "stderr", &line)?; + } + self.locked().0.push(line.unbind()); + Ok(()) + } + + /// Record the compile report; it is printed once the compile has been stored. + pub fn record_stdout(&self, text: &Bound<'_, PyString>) -> PyResult<()> { + let recorded: Bound<'_, PyString> = text.add("\n")?.downcast_into()?; + self.locked().1 = Some(recorded.unbind()); + Ok(()) + } + + /// Print the recorded compile report, if any, exactly as it is stored. + pub fn print_stdout(&self, py: Python<'_>) -> PyResult<()> { + let Some(text) = self.locked().1.as_ref().map(|text| text.clone_ref(py)) else { + return Ok(()); + }; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + let _ = stdout.call_method1("write", (text,))?; + let _ = stdout.call_method0("flush")?; + Ok(()) + } + + #[getter] + pub fn stderr_lines(&self, py: Python<'_>) -> Vec> { + self.locked() + .0 + .iter() + .map(|line| line.clone_ref(py)) + .collect() + } + + #[getter] + pub fn stdout(&self, py: Python<'_>) -> Option> { + self.locked().1.as_ref().map(|text| text.clone_ref(py)) + } +} + +/// What `write_reusable_compile` records about one full compile. +#[derive(Debug)] +pub struct CompileFacts<'py> { + pub(crate) native: Bound<'py, PyAny>, + pub(crate) output: Bound<'py, PyAny>, + pub(crate) exit_code: Bound<'py, PyAny>, + pub(crate) input_reads: Bound<'py, PyAny>, + pub(crate) artifact_writes: Bound<'py, PyAny>, + pub(crate) compile_cache_enabled: bool, + pub(crate) artifacts_written: bool, + pub(crate) dag_artifact_path: Bound<'py, PyAny>, + pub(crate) json_output: Bound<'py, PyAny>, +} + +/// SQL tests planned ahead of the write phase, and where the compile notes its artifact writes. +#[derive(Debug, Clone, Copy, Default)] +pub struct Prepared<'a, 'py> { + /// `planned_tests()` hands over the background plan, when one was started. + pub planned_tests: Option<&'a Bound<'py, PyAny>>, + pub recorder: ArtifactRecorder<'a>, +} diff --git a/crates/sqlbuild-python-compile/src/compile_command/types.rs b/crates/sqlbuild-python-compile/src/compile_command/types.rs new file mode 100644 index 0000000000..808d45c5dc --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_command/types.rs @@ -0,0 +1,7 @@ +//! Public traits and type aliases of compile command. + +use crate::compile_command::models::CompileArtifactWrites; +use pyo3::prelude::Py; + +/// Where a compile notes the artifacts it writes; `None` outside a recorded compile. +pub type ArtifactRecorder<'a> = Option<&'a Py>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/driver.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs rename to crates/sqlbuild-python-compile/src/compile_delta/_helpers/driver.rs index 2d54546bf4..9490c6e738 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/driver.rs @@ -1,31 +1,32 @@ //! Compile only changed models and SQL tests against the stored compile, or fall back. +use crate::compile_delta::constants::DELTA_VERIFY_ENV_VAR; +use crate::compile_delta::main::driver::env_enabled::env_enabled; +use crate::compile_delta::models::DeltaOutcome; +use crate::compile_delta::models::Finished; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; -use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; use sqlbuild_cache::project_reuse::types::ImageSections; -use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; -use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; -use crate::bindings::_helpers::compile_command::reuse_recording::{ - RecordedCompileOutput, print_to, -}; -use crate::bindings::_helpers::compile_delta::image::{ - ImageGate, ImageRelations, ImageRules, ProjectImage, -}; -use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; -use crate::bindings::_helpers::compile_driver::command::{ - PhasesOutcome, resolved_request, unrecorded_compile, -}; -use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; -use crate::bindings::_helpers::rules::custom_evaluation::stored_custom_rule_refusal; +use crate::compile_command::main::invocation::reuse_debug::reuse_debug; +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_delta::_helpers::splice::spliced_report; +use crate::compile_delta::models::ImageGate; +use crate::compile_delta::models::ImageRelations; +use crate::compile_delta::models::ImageRules; +use crate::compile_delta::models::ProjectImage; +use crate::compile_delta::models::SpliceScope; +use crate::compile_driver::main::stages::unrecorded_compile::unrecorded_compile; +use crate::compile_driver::types::PhasesOutcome; +use crate::compile_outputs::models::NativeReuseAttempt; +use crate::rules::main::custom::stored_custom_rule_refusal::stored_custom_rule_refusal; use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; use sqlbuild_discovery::header_changes::main::header_change::header_change; use sqlbuild_discovery::header_changes::models::HeaderChange; @@ -46,11 +47,7 @@ use sqlbuild_python_base::compile_context::models::PendingRules; use sqlbuild_python_base::compile_context::models::RecordMerge; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// Set to `1` to always compile the whole project. -pub(crate) const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; -/// Set to `1` to run each delta compile beside a full compile and fail on any difference. -pub(crate) const DELTA_VERIFY_ENV_VAR: &str = "SQLBUILD_VERIFY_DELTA_COMPILE"; -const ENABLED_VALUE: &str = "1"; +pub(crate) const ENABLED_VALUE: &str = "1"; /// Rerunning more models than this, or this share of the project, compiles the whole project. const MAX_RERUN_MODELS: usize = 500; const MAX_RERUN_PERCENT: usize = 5; @@ -68,7 +65,7 @@ const MODEL_KIND: &str = "model"; /// The custom-rule fact listing every project source. const PROJECT_SOURCES_FACT: &str = "project.sources"; const TESTS_ROOT: &str = "tests/"; -const STDERR: &str = "stderr"; +pub(crate) const STDERR: &str = "stderr"; const SQL_SUFFIX: &str = ".sql"; /// How generated SQL test CTE names mark their prefix, as in `__ref__orders`. const CTE_PREFIX_MARK: &str = "__"; @@ -81,7 +78,7 @@ const CANDIDATES_LOGGED: usize = 5; const UNREADABLE_IMAGE: &str = "the project image cannot be read"; const TOO_MANY_MODELS: &str = "too many models would compile again"; /// A compile whose Rules failed writes no SQL tests, functions or audits, so the next compile is full. -const AFTER_FAILED_RULES: &str = "the previous compile failed its Rules"; +pub(crate) const AFTER_FAILED_RULES: &str = "the previous compile failed its Rules"; /// The attribute holding a Python enum member's value. const ENUM_VALUE: &str = "value"; /// The lineage mode a delta compile reproduces; richer modes compile in full. @@ -90,42 +87,12 @@ const FAST_LINEAGE: &str = "fast"; const VERIFY_REPORT_DIRECTORY: &str = "target/cache/compiler-native-v1"; const VERIFY_REPORT_FILE: &str = "delta-report.out"; -/// Whether the environment turns `name` on. -pub(crate) fn env_enabled(name: &str) -> bool { - std::env::var(name).is_ok_and(|value| value == ENABLED_VALUE) -} - -/// Whether delta compiles, and so project images, are on for this process. -pub(crate) fn delta_enabled() -> bool { - !env_enabled(DELTA_DISABLE_ENV_VAR) -} - -/// A delta compile that finished, ready to be stored and printed or compared. -pub(crate) struct Finished<'py> { - pub(crate) stdout: String, - pub(crate) stderr_lines: Vec, - pub(crate) phases: PhasesOutcome<'py>, - pub(crate) image: Option, - pub(crate) merge: RecordMerge, - pub(crate) changed_files: usize, - pub(crate) rerun_models: usize, -} - -/// What one delta attempt came to. -pub(crate) enum DeltaOutcome<'py> { - /// The delta cannot reproduce this compile; the reason is for diagnostics only. - Fallback(String), - /// The compile stopped on an error from a changed file, exactly as a full compile would. - Raised(PyErr), - Finished(Box>), -} - -fn fallback<'py>(reason: impl Into) -> PyResult> { +pub(crate) fn fallback<'py>(reason: impl Into) -> PyResult> { Ok(DeltaOutcome::Fallback(reason.into())) } /// Whether the invocation asks for anything a delta compile does not reproduce. -fn ineligible(request: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn ineligible(request: &Bound<'_, PyAny>) -> PyResult> { let empty = |name: &str| -> PyResult { let value: Bound<'_, PyAny> = native_field(request, name)?; Ok(value.is_none() || !value.is_truthy()?) @@ -168,7 +135,7 @@ fn resource_sql(path: &str, root: &str) -> bool { } /// The stored compile's inputs and image; its report is read only after the delta runs. -struct Base { +pub(crate) struct Base { store_path: PathBuf, stored: DeltaBase, image: ProjectImage, @@ -194,7 +161,7 @@ fn changes_of( /// The rerun set: models and SQL tests, with every path and name the delta recompiles. #[derive(Clone, Debug, Default)] -struct Rerun { +pub(crate) struct Rerun { changed: BTreeSet, /// The sources the rerun models read, the only ones discovery keeps; `None` keeps all. sources: Option>, @@ -403,7 +370,7 @@ fn merged_rows( } /// One restricted compile of `rerun`, with what it captured. -struct Round<'py> { +pub(crate) struct Round<'py> { stdout: Option, stderr_lines: Vec, phases: PhasesOutcome<'py>, @@ -504,51 +471,14 @@ fn ran_round<'py>( } /// A delta compile ready to run its rounds: the stored compile, the changes and the first rerun. -struct Prepared { +pub(crate) struct Prepared { base: Base, - changed_files: usize, + pub(crate) changed_files: usize, rerun: Rerun, changed_tests: BTreeSet, limit: usize, } -/// Compile only the changed files against the stored compile, or say why it cannot. -pub(crate) fn run<'py>( - request: &Bound<'py, PyAny>, - attempt: &Bound<'py, PyAny>, -) -> PyResult> { - if !delta_enabled() { - return fallback("delta compiles are off"); - } - if let Some(reason) = ineligible(request)? { - return fallback(reason); - } - let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; - if project_dir - .join(TARGET_DIRECTORY_NAME) - .join(FAILED_COMPILED_DIRECTORY_NAME) - .is_dir() - { - return fallback(AFTER_FAILED_RULES); - } - let py: Python<'py> = request.py(); - let started: std::time::Instant = std::time::Instant::now(); - let prepared: Prepared = match prepared(request, attempt)? { - Ok(prepared) => prepared, - Err(reason) => return fallback(reason), - }; - trace(py, "delta base read", started)?; - let changed_files: usize = prepared.changed_files; - let (ran, rerun, base): (Round, Rerun, Base) = match rounds(request, attempt, prepared)? { - Ok(found) => found, - Err(outcome) => return Ok(outcome), - }; - trace(py, "delta rounds", started)?; - let outcome: PyResult> = finished(ran, base, &rerun, (py, changed_files)); - trace(py, "delta report", started)?; - outcome -} - /// The first `CANDIDATES_LOGGED` of `paths`. fn first_paths(paths: &[String]) -> &[String] { &paths[..paths.len().min(CANDIDATES_LOGGED)] @@ -570,7 +500,7 @@ fn changes_ruled_out(changes: &ProjectFileChanges) -> Option { } /// The stored compile, its image and the first rerun set, or why the gate rules a delta out. -fn prepared( +pub(crate) fn prepared( request: &Bound<'_, PyAny>, attempt: &Bound<'_, PyAny>, ) -> PyResult> { @@ -642,12 +572,12 @@ fn prepared( } /// A fallback that stops the rounds. -fn stopped<'py, T>(reason: impl Into) -> PyResult>> { +pub(crate) fn stopped<'py, T>(reason: impl Into) -> PyResult>> { Ok(Err(DeltaOutcome::Fallback(reason.into()))) } /// Run rounds until no rerun model changes the shape a model outside the rerun set reads. -fn rounds<'py>( +pub(crate) fn rounds<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, prepared: Prepared, @@ -1021,15 +951,6 @@ fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { || diagnostic.paths.is_empty() && diagnostic.resource_name.is_none() } -/// Log how long the delta has run, for profiling. -pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { - let milliseconds: u128 = started.elapsed().as_millis(); - reuse_debug( - py, - &format!("Delta compile {phase} after {milliseconds} ms"), - ) -} - /// Every diagnostic not owned by the rerun set, which the delta must leave as it was. fn unowned<'a>(rerun: &Rerun, diagnostics: &'a [CapturedDiagnostic]) -> Vec<&'a str> { diagnostics @@ -1039,7 +960,7 @@ fn unowned<'a>(rerun: &Rerun, diagnostics: &'a [CapturedDiagnostic]) -> Vec<&'a .collect() } -fn finished<'py>( +pub(crate) fn finished<'py>( ran: Round<'py>, base: Base, rerun: &Rerun, @@ -1227,38 +1148,3 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr image.lint_file_keys = capture.lint_file_keys; image } - -/// Store and print a finished delta compile; returns its exit code. -pub(crate) fn commit( - finished: Finished<'_>, - (request, attempt): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), -) -> PyResult { - let py: Python<'_> = attempt.py(); - let request: Bound<'_, PyAny> = resolved_request(request)?; - let Finished { - stdout, - stderr_lines, - phases, - image, - merge, - .. - } = finished; - let report: &str = stdout.strip_suffix('\n').unwrap_or(&stdout); - let output: Bound<'_, PyAny> = Bound::new( - py, - RecordedCompileOutput::recorded(py, &stderr_lines, &format!("{report}\n")), - )? - .into_any(); - for line in &stderr_lines { - print_to(py, STDERR, line.as_str())?; - } - sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(image); - sqlbuild_python_base::compile_context::main::delta::set_record_merge::set_record_merge(Some( - merge, - )); - crate::bindings::_helpers::compile_driver::command::store_and_print( - (&request, attempt), - &output, - phases, - ) -} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/_helpers/image.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/image.rs new file mode 100644 index 0000000000..26a25cb27d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/image.rs @@ -0,0 +1,391 @@ +//! The project image a full or delta compile stores beside its reuse slot. + +use crate::compile_delta::models::ImageRelations; +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::assembly::analysis_session::models::{DynamicFamily, RelationTables}; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_rules::models::RetainedRows; + +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::Decoder; +use sqlbuild_python_base::compile_context::models::Encoder; +use sqlbuild_python_base::compile_context::models::ModelCoverage; + +/// The upstream kind of a model another model reads. +const MODEL_KIND: &str = "model"; + +pub(crate) const RULES_SECTION: &str = "rules"; + +pub(crate) const RULE_ROWS_SECTION: &str = "rule-rows"; + +/// The value `bytes` hold as JSON, or `None` when they are malformed. +fn from_json(bytes: &[u8]) -> Option { + let Ok(value) = serde_json::from_slice(bytes) else { + return None; + }; + Some(value) +} + +pub(crate) fn decoded_rules( + request_json: Vec, + rows: Vec, + digests: &[u8], +) -> Option { + let retained: RetainedRows = RetainedRows { + request_json, + rows, + digests: decoded_strings(digests)?, + }; + BaseRules::decoded(&retained) +} + +/// Each table's dynamic column families, in the order the analysis request listed them. +pub(crate) type FamiliesByTable = Vec<(String, Vec)>; + +impl From for RelationTables { + fn from(relations: ImageRelations) -> Self { + Self { + types: relations.types, + nullability: relations.nullability, + complete: relations.complete, + schemas: relations.schemas, + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models, + families: relations.families, + } + } +} + +/// Rows of `table` that differ from the same relation's `types` row; `None` where equal. +pub(crate) fn differing(table: &Shapes, types: &Shapes) -> StoredShapes { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + table + .iter() + .map(|(name, shape)| { + let same: bool = types + .get(name.as_str()) + .is_some_and(|typed| *typed == shape); + (name.clone(), (!same).then(|| shape.clone())) + }) + .collect() +} + +/// `differing`'s rows restored against `types`. +pub(crate) fn restored(rows: StoredShapes, types: &Shapes) -> Option { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + rows.into_iter() + .map(|(name, shape)| { + let shape: Vec<(String, String)> = match shape { + Some(shape) => shape, + None => (*types.get(name.as_str())?).clone(), + }; + Some((name, shape)) + }) + .collect() +} + +/// A shape table as stored: `None` where a relation's row equals its `types` row. +type StoredShapes = Vec<(String, Option>)>; + +/// The rest of the image's facts as stored; shapes equal to a relation's types are stored once. +pub(crate) struct StoredFacts { + pub(crate) usages: BTreeMap, + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) complete: StoredShapes, + pub(crate) schemas: StoredShapes, + pub(crate) dependency_ordered: bool, + pub(crate) cyclic: bool, + pub(crate) legacy_models: Vec, + pub(crate) families: FamiliesByTable, + pub(crate) model_subjects: Option)>>, + pub(crate) diagnostics: Vec, + pub(crate) text_models: Option>, + pub(crate) environment_names: Vec, + pub(crate) pinned_sources: Vec, + pub(crate) test_sources: BTreeMap>, + pub(crate) model_shapes: StoredShapes, + pub(crate) coverage: Option>, + pub(crate) lint_model_keys: Option>, + pub(crate) lint_file_keys: Option)>>, + pub(crate) audit_references: BTreeMap>, +} + +pub(crate) fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { + sections + .iter() + .find(|(section, _)| section == name) + .map(|(_, bytes)| bytes.as_slice()) +} + +/// The diagnostics and report lines an image keeps as one JSON array, being small and varied. +struct StoredMisc { + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + coverage: Option>, +} + +impl<'de> serde::Deserialize<'de> for StoredMisc { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_tuple(4, StoredMiscVisitor) + } +} + +/// Reads [`StoredMisc`] from its four-element array. +struct StoredMiscVisitor; + +impl<'de> serde::de::Visitor<'de> for StoredMiscVisitor { + type Value = StoredMisc; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a tuple of size 4") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let missing = |index: usize| serde::de::Error::invalid_length(index, &self); + Ok(StoredMisc { + diagnostics: seq.next_element()?.ok_or_else(|| missing(0))?, + text_models: seq.next_element()?.ok_or_else(|| missing(1))?, + environment_names: seq.next_element()?.ok_or_else(|| missing(2))?, + coverage: seq.next_element()?.ok_or_else(|| missing(3))?, + }) + } +} + +fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.pairs(shape); + } +} + +fn put_stored_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a StoredShapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.flag(shape.is_some()); + if let Some(shape) = shape { + encoder.pairs(shape); + } + } +} + +pub(crate) fn shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| Some((decoder.string()?, decoder.pairs()?))) +} + +fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let name: String = decoder.string()?; + let shape: Option> = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + Some((name, shape)) + }) +} + +fn put_families<'a>(encoder: &mut Encoder<'a>, families: &'a FamiliesByTable) { + encoder.count(families.len()); + for (table, declared) in families { + encoder.string(table); + encoder.count(declared.len()); + for family in declared { + encoder.string(&family.name); + encoder.string(&family.pivot_column); + encoder.string(&family.value_column); + encoder.string(&family.aggregate); + encoder.string(&family.data_type); + encoder.optional_string(family.name_pattern.as_deref()); + } + } +} + +pub(crate) fn families(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let table: String = decoder.string()?; + let declared: Vec = decoder.list(|decoder| { + Some(DynamicFamily { + name: decoder.string()?, + pivot_column: decoder.string()?, + value_column: decoder.string()?, + aggregate: decoder.string()?, + data_type: decoder.string()?, + name_pattern: decoder.optional_string()?, + }) + })?; + Some((table, declared)) + }) +} + +impl StoredFacts { + pub(crate) fn encode(&self) -> Result, String> { + let misc: Vec = serde_json::to_vec(&( + &self.diagnostics, + &self.text_models, + &self.environment_names, + &self.coverage, + )) + .map_err(|error| error.to_string())?; + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.count(self.usages.len()); + for (path, digest) in &self.usages { + encoder.string(path); + encoder.string(digest); + } + put_shapes(&mut encoder, &self.types); + put_shapes(&mut encoder, &self.nullability); + put_stored_shapes(&mut encoder, &self.complete); + put_stored_shapes(&mut encoder, &self.schemas); + put_stored_shapes(&mut encoder, &self.model_shapes); + encoder.flag(self.dependency_ordered); + encoder.flag(self.cyclic); + encoder.strings(self.legacy_models.iter().map(String::as_str)); + put_families(&mut encoder, &self.families); + encoder.flag(self.model_subjects.is_some()); + if let Some(subjects) = &self.model_subjects { + encoder.count(subjects.len()); + for (path, name, materialized) in subjects { + encoder.string(path); + encoder.string(name); + encoder.optional_string(materialized.as_deref()); + } + } + encoder.flag(self.lint_model_keys.is_some()); + if let Some(keys) = &self.lint_model_keys { + encoder.pairs(keys); + } + encoder.flag(self.lint_file_keys.is_some()); + if let Some(keys) = &self.lint_file_keys { + encoder.count(keys.len()); + for (path, key) in keys { + encoder.string(path); + encoder.optional_string(key.as_deref()); + } + } + encoder.strings(self.pinned_sources.iter().map(String::as_str)); + encoder.count(self.test_sources.len()); + for (path, names) in &self.test_sources { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } + encoder.count(self.audit_references.len()); + for (path, names) in &self.audit_references { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } + encoder.bytes(&misc); + Ok(encoder.finish()) + } + + pub(crate) fn decode(section: &[u8]) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let usages: BTreeMap = decoder + .list(|decoder| Some((decoder.string()?, decoder.string()?)))? + .into_iter() + .collect(); + let types: Shapes = shapes(&mut decoder)?; + let nullability: Shapes = shapes(&mut decoder)?; + let complete: StoredShapes = stored_shapes(&mut decoder)?; + let schemas: StoredShapes = stored_shapes(&mut decoder)?; + let model_shapes: StoredShapes = stored_shapes(&mut decoder)?; + let dependency_ordered: bool = decoder.flag()?; + let cyclic: bool = decoder.flag()?; + let legacy_models: Vec = decoder.strings()?; + let families: FamiliesByTable = families(&mut decoder)?; + let model_subjects = if decoder.flag()? { + Some(decoder.list(|decoder| { + Some(( + decoder.string()?, + decoder.string()?, + decoder.optional_string()?, + )) + })?) + } else { + None + }; + let lint_model_keys = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + let lint_file_keys = if decoder.flag()? { + Some(decoder.list(|decoder| Some((decoder.string()?, decoder.optional_string()?)))?) + } else { + None + }; + let pinned_sources: Vec = decoder.strings()?; + let test_sources: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); + let audit_references: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); + let StoredMisc { + diagnostics, + text_models, + environment_names, + coverage, + }: StoredMisc = from_json(decoder.bytes()?)?; + decoder.finish()?; + Some(Self { + usages, + types, + nullability, + complete, + schemas, + dependency_ordered, + cyclic, + legacy_models, + families, + model_subjects, + diagnostics, + text_models, + environment_names, + pinned_sources, + test_sources, + model_shapes, + coverage, + lint_model_keys, + lint_file_keys, + audit_references, + }) + } +} + +pub(crate) fn encoded_strings(values: &[String]) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.strings(values.iter().map(String::as_str)); + encoder.finish() +} + +fn decoded_strings(section: &[u8]) -> Option> { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let values: Vec = decoder.strings()?; + decoder.finish()?; + Some(values) +} + +/// Whether `model` reads any of the models `names`. +pub(crate) fn reads_any_model(model: &CapturedModel, names: &BTreeSet) -> bool { + model + .upstream + .iter() + .any(|(kind, name)| kind == MODEL_KIND && names.contains(name)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/mod.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs rename to crates/sqlbuild-python-compile/src/compile_delta/_helpers/mod.rs index 0865da1a64..3ff2d27cfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod driver; pub(crate) mod image; -pub(crate) mod lint_hooks; pub(crate) mod report_hooks; pub(crate) mod rules_rows; pub(crate) mod splice; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/report_hooks.rs similarity index 57% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs rename to crates/sqlbuild-python-compile/src/compile_delta/_helpers/report_hooks.rs index d98c4c3381..5dc4bbe149 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/report_hooks.rs @@ -1,16 +1,11 @@ //! Where the compile's report and output stages leave image facts or take a delta's injection. -use std::collections::HashMap; - use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethods}; -use sqlbuild_core::cli_text::models::ModelLine; +use pyo3::types::{PyStringMethods, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::compile_command::report::diagnostic_item; +use crate::compile_command::main::invocation::diagnostic_item::diagnostic_item; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; -use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; -use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; use sqlbuild_python_base::compile_context::models::CapturedModel; use sqlbuild_python_base::compile_context::models::CapturedTest; @@ -21,41 +16,11 @@ use sqlbuild_python_project::graph::models::NativeProjectGraph; /// The graph key kind of a model. const MODEL_KIND: &str = "model"; -/// The text report's model lines, captured or, in a delta, patched for the rerun models. -pub(crate) fn text_model_lines(models: Vec) -> Vec { - let models: Vec = match injection() { - None => models, - Some(injection) => { - let mut fresh: HashMap = models - .into_iter() - .map(|line| (line.name.clone(), line)) - .collect(); - injection - .text_models - .iter() - .map(|(name, failed, column_count)| { - fresh.remove(name).unwrap_or_else(|| ModelLine { - name: name.clone(), - failed: *failed, - column_count: *column_count, - }) - }) - .collect() - } - }; - capture(|capture| { - capture.text_models = Some( - models - .iter() - .map(|line| (line.name.clone(), line.failed, line.column_count)) - .collect(), - ); - }); - models -} - /// Record which models each SQL test's compiled artifact is keyed by. -fn record_test_chains_inner(tests: &Bound<'_, PyAny>, chains: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn record_test_chains_inner( + tests: &Bound<'_, PyAny>, + chains: &Bound<'_, PyAny>, +) -> PyResult<()> { if !capturing() { return Ok(()); } @@ -81,7 +46,7 @@ fn record_test_chains_inner(tests: &Bound<'_, PyAny>, chains: &Bound<'_, PyAny>) } /// Record the compiled models with their references, and the report's diagnostics. -fn record_compile_facts_inner( +pub(crate) fn record_compile_facts_inner( graph: &Bound<'_, PyAny>, diagnostics: &Bound<'_, PyTuple>, ) -> PyResult<()> { @@ -150,53 +115,3 @@ pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult, - chains: &Bound<'_, PyAny>, -) -> PyResult<()> { - guarded(|| record_test_chains_inner(tests, chains)) -} - -/// Record the compiled models with their references, and the report's diagnostics. -pub(crate) fn record_compile_facts( - graph: &Bound<'_, PyAny>, - diagnostics: &Bound<'_, PyTuple>, -) -> PyResult<()> { - guarded(|| record_compile_facts_inner(graph, diagnostics)) -} - -/// In a delta, add the text of files that stored diagnostics point into but were not rerun. -pub(crate) fn complete_source_texts( - project: &Bound<'_, PyAny>, - diagnostics: &[Bound<'_, PyAny>], - texts: &Bound<'_, PyDict>, -) -> PyResult<()> { - let Some(injection) = injection() else { - return Ok(()); - }; - let _ = project; - let project_dir: &std::path::Path = &injection.project_dir; - let mut locations: Vec> = Vec::new(); - for diagnostic in diagnostics { - locations.push(native_field(diagnostic, "location")?); - for item in native_field(diagnostic, "related_locations")?.try_iter()? { - locations.push(native_field(&item?, "location")?); - } - } - for location in locations.into_iter().filter(|location| !location.is_none()) { - let path: Bound<'_, PyAny> = native_field(&location, "path")?; - if texts.contains(&path)? { - continue; - } - let relative: String = path.str()?.to_str()?.replace('\\', "/"); - if !injection.source_paths.contains(&relative) { - continue; - } - if let Ok(text) = std::fs::read_to_string(project_dir.join(&relative)) { - texts.set_item(&path, text)?; - } - } - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/_helpers/rules_rows.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/rules_rows.rs new file mode 100644 index 0000000000..c97ee9286e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/rules_rows.rs @@ -0,0 +1,55 @@ +//! A delta compile's built-in rules request: the previous compile's, with the rerun rows fresh. + +use std::collections::HashMap; + +use pyo3::PyResult; +use sqlbuild_rules::models::{EvaluateRequest, SqlTestFact}; + +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::models::BaseRules; + +/// The retained request with each row's payload digest restored. +pub(crate) fn restored(base: BaseRules) -> EvaluateRequest { + let mut request: EvaluateRequest = base.request; + for (model, digest) in request.models.iter_mut().zip(base.digests) { + model.payload_digest = Some(digest); + model.stripped = true; + } + request +} + +/// `base` with the facts of every file `fresh` covers replaced, in place and in order. +pub(crate) fn replaced_tests( + base: &[SqlTestFact], + fresh: Vec, +) -> PyResult> { + let mut by_file: HashMap> = HashMap::new(); + let mut grouped: HashMap> = HashMap::new(); + for fact in fresh { + grouped + .entry(fact.source_path.clone()) + .or_default() + .push(fact); + } + let mut counts: HashMap<&str, usize> = HashMap::new(); + for fact in base { + *counts.entry(fact.source_path.as_str()).or_default() += 1; + } + for (path, facts) in grouped { + if counts.get(path.as_str()).copied().unwrap_or_default() != facts.len() { + return Err(delta_unsupported( + "a rerun SQL test file changed its test count", + )); + } + by_file.insert(path, facts.into_iter()); + } + Ok(base + .iter() + .map(|fact| { + by_file + .get_mut(&fact.source_path) + .and_then(Iterator::next) + .unwrap_or_else(|| fact.clone()) + }) + .collect()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/splice.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs rename to crates/sqlbuild-python-compile/src/compile_delta/_helpers/splice.rs index 2085227242..c0f57d13ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/splice.rs @@ -1,6 +1,7 @@ //! Splice a delta compile's JSON report into the stored report it patches. -use std::collections::{BTreeSet, HashMap}; +use crate::compile_delta::models::SpliceScope; +use std::collections::HashMap; use serde_json::Value; @@ -157,14 +158,6 @@ fn member<'a>(found: &'a [(String, Span)], key: &str) -> Option<&'a Span> { .map(|(_, span)| span) } -/// What the splice replaces: the rerun models, their audits and the rerun SQL tests. -pub(crate) struct SpliceScope<'a> { - pub(crate) models: &'a BTreeSet, - pub(crate) tests: &'a BTreeSet, - /// The sources the delta discovered, when it discovered only some; `None` when all. - pub(crate) sources: Option<&'a BTreeSet>, -} - /// The string at `path` in the item at `span`, items keeping own keys before nested objects. fn item_field(text: &str, span: Span, path: &[&str]) -> Option { let bytes: &[u8] = text.as_bytes(); diff --git a/crates/sqlbuild-python-compile/src/compile_delta/_helpers/verify.rs b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/verify.rs new file mode 100644 index 0000000000..32ed55cb1c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/_helpers/verify.rs @@ -0,0 +1,83 @@ +//! The hidden check that each delta compile matches the full compile it replaces. + +use crate::compile_delta::models::Expected; +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +pub(crate) const MISMATCH_FILE: &str = "delta-mismatch.json"; + +/// What the full compile produced. +#[derive(Debug, Default)] +pub(crate) struct Actual { + pub(crate) stdout: Option, + pub(crate) exit_code: Option, + pub(crate) artifacts_written: bool, + pub(crate) written: HashMap>, +} + +pub(crate) static EXPECTED: Mutex> = Mutex::new(None); +pub(crate) static ACTUAL: Mutex> = Mutex::new(None); + +pub(crate) fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Every file under `directory`, depth first. +pub(crate) fn files_under(directory: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut found: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + match entry.file_type() { + Ok(kind) if kind.is_dir() => found.extend(files_under(&path)), + Ok(kind) if kind.is_file() => found.push(path), + _ => {} + } + } + found +} + +/// The report with its `compile_timings` value blanked. +pub(crate) fn without_timings(stdout: &str) -> String { + let Some(start) = stdout.find("\n \"compile_timings\": {") else { + return stdout.to_owned(); + }; + let Some(length) = stdout[start..].find("\n },\n") else { + return stdout.to_owned(); + }; + format!("{}{}", &stdout[..start], &stdout[start + length + 5..]) +} + +/// Where the files the full compile wrote under `root` differ from what the delta left there. +pub(crate) fn tree_differences( + (root, expected): (&str, &BTreeMap), + written: &HashMap>, +) -> Vec { + let mut found: Vec = Vec::new(); + let prefix: String = format!("{root}{}", std::path::MAIN_SEPARATOR); + let full_paths: BTreeMap<&str, &Option> = written + .iter() + .filter(|(path, _)| path.starts_with(prefix.as_str())) + .map(|(path, digest)| (path.as_str(), digest)) + .collect(); + for (path, digest) in &full_paths { + match (expected.get(*path), digest) { + (None, _) => found.push(format!("the delta left no artifact at {path}")), + (Some(left), Some(right)) if left != right => { + found.push(format!("artifact differs: {path}")); + } + _ => {} + } + } + for path in expected.keys() { + if !full_paths.contains_key(path.as_str()) { + found.push(format!( + "the delta left an artifact the full compile does not: {path}" + )); + } + } + found +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/constants.rs b/crates/sqlbuild-python-compile/src/compile_delta/constants.rs new file mode 100644 index 0000000000..9b2f3a0f5f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/constants.rs @@ -0,0 +1,19 @@ +//! Public constants of compile delta. + +/// Set to `1` to always compile the whole project. +pub const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; + +/// Set to `1` to run each delta compile beside a full compile and fail on any difference. +pub const DELTA_VERIFY_ENV_VAR: &str = "SQLBUILD_VERIFY_DELTA_COMPILE"; + +/// Changes whenever the image's meaning changes, so an older image is never read. +pub const IMAGE_FORMAT: &str = "sqlbuild-project-image-v9"; + +pub const GATE_SECTION: &str = "gate"; + +pub const FACTS_SECTION: &str = "facts"; + +pub const RULE_DIGESTS_SECTION: &str = "rule-digests"; + +/// The exit code of a compile whose delta differed from its full compile. +pub const MISMATCH_EXIT_CODE: i64 = 70; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/commit.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/commit.rs new file mode 100644 index 0000000000..bedc3c4568 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/commit.rs @@ -0,0 +1,43 @@ +//! Store and print a finished delta compile; returns its exit code. + +use crate::compile_command::main::invocation::print_to::print_to; +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_delta::_helpers::driver::STDERR; +use crate::compile_delta::models::Finished; +use crate::compile_driver::main::stages::resolved_request::resolved_request; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// Store and print a finished delta compile; returns its exit code. +pub fn commit( + finished: Finished<'_>, + (request, attempt): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = attempt.py(); + let request: Bound<'_, PyAny> = resolved_request(request)?; + let Finished { + stdout, + stderr_lines, + phases, + image, + merge, + .. + } = finished; + let report: &str = stdout.strip_suffix('\n').unwrap_or(&stdout); + let output: Bound<'_, PyAny> = Bound::new( + py, + RecordedCompileOutput::recorded(py, &stderr_lines, &format!("{report}\n")), + )? + .into_any(); + for line in &stderr_lines { + print_to(py, STDERR, line.as_str())?; + } + sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(image); + sqlbuild_python_base::compile_context::main::delta::set_record_merge::set_record_merge(Some( + merge, + )); + crate::compile_driver::main::stages::store_and_print::store_and_print( + (&request, attempt), + &output, + phases, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/delta_enabled.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/delta_enabled.rs new file mode 100644 index 0000000000..1c789397cf --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/delta_enabled.rs @@ -0,0 +1,9 @@ +//! Whether delta compiles, and so project images, are on for this process. + +use crate::compile_delta::constants::DELTA_DISABLE_ENV_VAR; +use crate::compile_delta::main::driver::env_enabled::env_enabled; + +/// Whether delta compiles, and so project images, are on for this process. +pub fn delta_enabled() -> bool { + !env_enabled(DELTA_DISABLE_ENV_VAR) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/env_enabled.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/env_enabled.rs new file mode 100644 index 0000000000..7e78524cb1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/env_enabled.rs @@ -0,0 +1,8 @@ +//! Whether the environment turns `name` on. + +use crate::compile_delta::_helpers::driver::ENABLED_VALUE; + +/// Whether the environment turns `name` on. +pub fn env_enabled(name: &str) -> bool { + std::env::var(name).is_ok_and(|value| value == ENABLED_VALUE) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/mod.rs new file mode 100644 index 0000000000..1c09d29bd2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/mod.rs @@ -0,0 +1,5 @@ +pub mod commit; +pub mod delta_enabled; +pub mod env_enabled; +pub mod run; +pub mod trace; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/run.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/run.rs new file mode 100644 index 0000000000..c830aae56a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/run.rs @@ -0,0 +1,57 @@ +//! Compile only the changed files against the stored compile, or say why it cannot. + +use crate::compile_command::constants::TARGET_DIRECTORY_NAME; +use crate::compile_delta::_helpers::driver::AFTER_FAILED_RULES; +use crate::compile_delta::_helpers::driver::Base; +use crate::compile_delta::_helpers::driver::Prepared; +use crate::compile_delta::_helpers::driver::Rerun; +use crate::compile_delta::_helpers::driver::Round; +use crate::compile_delta::_helpers::driver::fallback; +use crate::compile_delta::_helpers::driver::finished; +use crate::compile_delta::_helpers::driver::ineligible; +use crate::compile_delta::_helpers::driver::prepared; +use crate::compile_delta::_helpers::driver::rounds; +use crate::compile_delta::main::driver::delta_enabled::delta_enabled; +use crate::compile_delta::main::driver::trace::trace; +use crate::compile_delta::models::DeltaOutcome; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::path::PathBuf; + +/// Compile only the changed files against the stored compile, or say why it cannot. +pub fn run<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult> { + if !delta_enabled() { + return fallback("delta compiles are off"); + } + if let Some(reason) = ineligible(request)? { + return fallback(reason); + } + let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + if project_dir + .join(TARGET_DIRECTORY_NAME) + .join(FAILED_COMPILED_DIRECTORY_NAME) + .is_dir() + { + return fallback(AFTER_FAILED_RULES); + } + let py: Python<'py> = request.py(); + let started: std::time::Instant = std::time::Instant::now(); + let prepared: Prepared = match prepared(request, attempt)? { + Ok(prepared) => prepared, + Err(reason) => return fallback(reason), + }; + trace(py, "delta base read", started)?; + let changed_files: usize = prepared.changed_files; + let (ran, rerun, base): (Round, Rerun, Base) = match rounds(request, attempt, prepared)? { + Ok(found) => found, + Err(outcome) => return Ok(outcome), + }; + trace(py, "delta rounds", started)?; + let outcome: PyResult> = finished(ran, base, &rerun, (py, changed_files)); + trace(py, "delta report", started)?; + outcome +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/driver/trace.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/trace.rs new file mode 100644 index 0000000000..42b2de3bd7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/driver/trace.rs @@ -0,0 +1,13 @@ +//! Log how long the delta has run, for profiling. + +use crate::compile_command::main::invocation::reuse_debug::reuse_debug; +use pyo3::prelude::{PyResult, Python}; + +/// Log how long the delta has run, for profiling. +pub fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { + let milliseconds: u128 = started.elapsed().as_millis(); + reuse_debug( + py, + &format!("Delta compile {phase} after {milliseconds} ms"), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/changed_files.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/changed_files.rs new file mode 100644 index 0000000000..fbfde5414d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/changed_files.rs @@ -0,0 +1,28 @@ +//! The changed files a delta compile lints, read as a full lint walk reads them. + +use crate::rules::models::LintFile; +use pyo3::PyResult; +use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use std::path::{Path, PathBuf}; + +/// The changed files a delta compile lints, read as a full lint walk reads them. +pub fn changed_files(project_dir: &Path) -> PyResult> { + let Some(injection) = injection() else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + for path in &injection.changed_paths { + let linted: bool = path + .split_once('/') + .is_some_and(|(root, _)| LINT_DIRECTORY_NAMES.contains(&root)); + if !linted { + continue; + } + let path: PathBuf = project_dir.join(path); + let contents: String = std::fs::read_to_string(&path) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + files.push(LintFile { path, contents }); + } + Ok(files) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/file_keys.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/file_keys.rs new file mode 100644 index 0000000000..22c77f2342 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/file_keys.rs @@ -0,0 +1,50 @@ +//! Each non-model file's lint key in walk order, recorded or, in a delta, patched. + +use pyo3::PyResult; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Each non-model file's lint key in walk order, recorded or, in a delta, patched. +pub fn file_keys( + project_dir: &Path, + fresh: Vec<(PathBuf, String, Option)>, +) -> PyResult)>> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, Option)> = fresh + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_file_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the project files were not recorded", + )); + }; + let mut fresh: HashMap)> = + fresh.into_iter().map(|row| (row.1.clone(), row)).collect(); + let whole: Vec<(PathBuf, String, Option)> = stored + .iter() + .map(|(path, key)| { + fresh + .remove(path) + .unwrap_or_else(|| (project_dir.join(path), path.clone(), key.clone())) + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported("a changed file has no recorded lint key")); + } + let keys: Vec<(String, Option)> = whole + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + Ok(whole) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/mod.rs new file mode 100644 index 0000000000..a4c0500e2c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/mod.rs @@ -0,0 +1,3 @@ +pub mod changed_files; +pub mod file_keys; +pub mod model_identities; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/model_identities.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/model_identities.rs new file mode 100644 index 0000000000..938986e841 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/lint_hooks/model_identities.rs @@ -0,0 +1,39 @@ +//! Each model file's lint key, recorded for the image or, in a delta, patched for reruns. + +use pyo3::PyResult; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; +use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use std::collections::HashMap; + +/// Each model file's lint key, recorded for the image or, in a delta, patched for reruns. +pub fn model_identities(fresh: Vec<(String, String)>) -> PyResult> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, String)> = fresh.clone(); + capture(|capture| capture.lint_model_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_model_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the models were not recorded", + )); + }; + let mut fresh: HashMap = fresh.into_iter().collect(); + let whole: Vec<(String, String)> = stored + .iter() + .map(|(path, key)| match fresh.remove(path) { + Some(key) => (path.clone(), key), + None => (path.clone(), key.clone()), + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported( + "a recompiled model has no recorded lint key", + )); + } + capture(|capture| capture.lint_model_keys = Some(whole.clone())); + Ok(whole) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/mod.rs new file mode 100644 index 0000000000..e837bab0c2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/mod.rs @@ -0,0 +1,6 @@ +pub mod driver; +pub mod lint_hooks; +pub mod report_hooks; +pub mod rules_rows; +pub mod splice; +pub mod verify; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/complete_source_texts.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/complete_source_texts.rs new file mode 100644 index 0000000000..4e7c61f959 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/complete_source_texts.rs @@ -0,0 +1,40 @@ +//! In a delta, add the text of files that stored diagnostics point into but were not rerun. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// In a delta, add the text of files that stored diagnostics point into but were not rerun. +pub fn complete_source_texts( + project: &Bound<'_, PyAny>, + diagnostics: &[Bound<'_, PyAny>], + texts: &Bound<'_, PyDict>, +) -> PyResult<()> { + let Some(injection) = injection() else { + return Ok(()); + }; + let _ = project; + let project_dir: &std::path::Path = &injection.project_dir; + let mut locations: Vec> = Vec::new(); + for diagnostic in diagnostics { + locations.push(native_field(diagnostic, "location")?); + for item in native_field(diagnostic, "related_locations")?.try_iter()? { + locations.push(native_field(&item?, "location")?); + } + } + for location in locations.into_iter().filter(|location| !location.is_none()) { + let path: Bound<'_, PyAny> = native_field(&location, "path")?; + if texts.contains(&path)? { + continue; + } + let relative: String = path.str()?.to_str()?.replace('\\', "/"); + if !injection.source_paths.contains(&relative) { + continue; + } + if let Ok(text) = std::fs::read_to_string(project_dir.join(&relative)) { + texts.set_item(&path, text)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/mod.rs new file mode 100644 index 0000000000..98278dd744 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/mod.rs @@ -0,0 +1,4 @@ +pub mod complete_source_texts; +pub mod record_compile_facts; +pub mod record_test_chains; +pub mod text_model_lines; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_compile_facts.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_compile_facts.rs new file mode 100644 index 0000000000..f822278431 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_compile_facts.rs @@ -0,0 +1,14 @@ +//! Record the compiled models with their references, and the report's diagnostics. + +use crate::compile_delta::_helpers::report_hooks::record_compile_facts_inner; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; + +/// Record the compiled models with their references, and the report's diagnostics. +pub fn record_compile_facts( + graph: &Bound<'_, PyAny>, + diagnostics: &Bound<'_, PyTuple>, +) -> PyResult<()> { + guarded(|| record_compile_facts_inner(graph, diagnostics)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_test_chains.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_test_chains.rs new file mode 100644 index 0000000000..63db182c45 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/record_test_chains.rs @@ -0,0 +1,10 @@ +//! Record which models each SQL test's compiled artifact is keyed by. + +use crate::compile_delta::_helpers::report_hooks::record_test_chains_inner; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; + +/// Record which models each SQL test's compiled artifact is keyed by. +pub fn record_test_chains(tests: &Bound<'_, PyAny>, chains: &Bound<'_, PyAny>) -> PyResult<()> { + guarded(|| record_test_chains_inner(tests, chains)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/text_model_lines.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/text_model_lines.rs new file mode 100644 index 0000000000..94f071c68f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/report_hooks/text_model_lines.rs @@ -0,0 +1,39 @@ +//! The text report's model lines, captured or, in a delta, patched for the rerun models. + +use sqlbuild_core::cli_text::models::ModelLine; +use sqlbuild_python_base::compile_context::main::delta::capture::capture; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use std::collections::HashMap; + +/// The text report's model lines, captured or, in a delta, patched for the rerun models. +pub fn text_model_lines(models: Vec) -> Vec { + let models: Vec = match injection() { + None => models, + Some(injection) => { + let mut fresh: HashMap = models + .into_iter() + .map(|line| (line.name.clone(), line)) + .collect(); + injection + .text_models + .iter() + .map(|(name, failed, column_count)| { + fresh.remove(name).unwrap_or_else(|| ModelLine { + name: name.clone(), + failed: *failed, + column_count: *column_count, + }) + }) + .collect() + } + }; + capture(|capture| { + capture.text_models = Some( + models + .iter() + .map(|line| (line.name.clone(), line.failed, line.column_count)) + .collect(), + ); + }); + models +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/mod.rs new file mode 100644 index 0000000000..b318f834fc --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/mod.rs @@ -0,0 +1 @@ +pub mod whole_project_request; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/whole_project_request.rs similarity index 51% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs rename to crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/whole_project_request.rs index ad430a127e..8ba7ed53db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/rules_rows/whole_project_request.rs @@ -1,18 +1,18 @@ -//! A delta compile's built-in rules request: the previous compile's, with the rerun rows fresh. - -use std::collections::HashMap; +//! In a delta, the whole project's rules request with the rerun rows taken from `request`. +use crate::compile_delta::_helpers::rules_rows::replaced_tests; +use crate::compile_delta::_helpers::rules_rows::restored; use pyo3::PyResult; -use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains; -use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; - use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::BaseRules; use sqlbuild_python_base::compile_context::models::PendingRules; +use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains; +use sqlbuild_rules::models::{EvaluateRequest, Model}; +use std::collections::HashMap; /// In a delta, the whole project's rules request with the rerun rows taken from `request`. -pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult { +pub fn whole_project_request(request: EvaluateRequest) -> PyResult { let Some(injection) = injection() else { return Ok(request); }; @@ -56,46 +56,3 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult EvaluateRequest { - let mut request: EvaluateRequest = base.request; - for (model, digest) in request.models.iter_mut().zip(base.digests) { - model.payload_digest = Some(digest); - model.stripped = true; - } - request -} - -/// `base` with the facts of every file `fresh` covers replaced, in place and in order. -fn replaced_tests(base: &[SqlTestFact], fresh: Vec) -> PyResult> { - let mut by_file: HashMap> = HashMap::new(); - let mut grouped: HashMap> = HashMap::new(); - for fact in fresh { - grouped - .entry(fact.source_path.clone()) - .or_default() - .push(fact); - } - let mut counts: HashMap<&str, usize> = HashMap::new(); - for fact in base { - *counts.entry(fact.source_path.as_str()).or_default() += 1; - } - for (path, facts) in grouped { - if counts.get(path.as_str()).copied().unwrap_or_default() != facts.len() { - return Err(delta_unsupported( - "a rerun SQL test file changed its test count", - )); - } - by_file.insert(path, facts.into_iter()); - } - Ok(base - .iter() - .map(|fact| { - by_file - .get_mut(&fact.source_path) - .and_then(Iterator::next) - .unwrap_or_else(|| fact.clone()) - }) - .collect()) -} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/splice/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/splice/mod.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/splice/mod.rs @@ -0,0 +1 @@ + diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/compiled_digests.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/compiled_digests.rs new file mode 100644 index 0000000000..cedbd20acf --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/compiled_digests.rs @@ -0,0 +1,27 @@ +//! Every file under `root` with its native digest, keyed by absolute path text. + +use crate::compile_delta::_helpers::verify::files_under; +use sqlbuild_cache::digest::main::digest_files::digest_files; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::project_reuse::constants::NATIVE_DIGEST_PREFIX; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +/// Every file under `root` with its native digest, keyed by absolute path text. +pub fn compiled_digests(root: &Path) -> BTreeMap { + let paths: Vec = files_under(root); + let digests = digest_files(&paths); + paths + .into_iter() + .zip(digests) + .filter_map(|(path, digest)| { + let Ok(digest) = digest else { + return None; + }; + Some(( + path.to_string_lossy().into_owned(), + format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest)), + )) + }) + .collect() +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/differences.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/differences.rs new file mode 100644 index 0000000000..bee3617465 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/differences.rs @@ -0,0 +1,74 @@ +//! Compare the held delta result with the full compile's; every difference found. + +use crate::compile_delta::_helpers::verify::ACTUAL; +use crate::compile_delta::_helpers::verify::Actual; +use crate::compile_delta::_helpers::verify::EXPECTED; +use crate::compile_delta::_helpers::verify::locked; +use crate::compile_delta::_helpers::verify::tree_differences; +use crate::compile_delta::_helpers::verify::without_timings; + +/// Compare the held delta result with the full compile's; every difference found. +pub fn differences(raised: Option) -> Vec { + let Some(expected) = locked(&EXPECTED).take() else { + return Vec::new(); + }; + let actual: Actual = locked(&ACTUAL).take().unwrap_or_default(); + let mut found: Vec = Vec::new(); + if expected.raised != raised { + found.push(format!( + "raised error differs: delta {:?}, full {:?}", + expected.raised, raised + )); + } + if expected.raised.is_some() || raised.is_some() { + return found; + } + if expected.exit_code != actual.exit_code { + found.push(format!( + "exit code differs: delta {:?}, full {:?}", + expected.exit_code, actual.exit_code + )); + } + let delta_stdout: Option = expected.stdout.as_deref().map(without_timings); + let full_stdout: Option = actual.stdout.as_deref().map(without_timings); + if delta_stdout != full_stdout { + let (delta, full) = ( + delta_stdout.unwrap_or_default(), + full_stdout.unwrap_or_default(), + ); + let common: usize = delta + .bytes() + .zip(full.bytes()) + .take_while(|(left, right)| left == right) + .count(); + let start: usize = common.saturating_sub(200); + let excerpt = |text: &str| -> String { + text.get(start..(common + 300).min(text.len())) + .unwrap_or_default() + .to_owned() + }; + found.push(format!( + "report differs at byte {common}:\n--- delta\n{}\n--- full\n{}", + excerpt(&delta), + excerpt(&full) + )); + } + if expected.artifacts_written != actual.artifacts_written { + found.push(format!( + "artifact writing differs: delta {}, full {}", + expected.artifacts_written, actual.artifacts_written + )); + } + found.extend(tree_differences( + (&expected.failed_root, &expected.failed), + &actual.written, + )); + if actual.artifacts_written { + found.extend(tree_differences( + (&expected.compiled_root, &expected.compiled), + &actual.written, + )); + } + found.truncate(50); + found +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/disarm.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/disarm.rs new file mode 100644 index 0000000000..bb888104e9 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/disarm.rs @@ -0,0 +1,11 @@ +//! Forget any held result. + +use crate::compile_delta::_helpers::verify::ACTUAL; +use crate::compile_delta::_helpers::verify::EXPECTED; +use crate::compile_delta::_helpers::verify::locked; + +/// Forget any held result. +pub fn disarm() { + *locked(&EXPECTED) = None; + *locked(&ACTUAL) = None; +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/full_compile_finished.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/full_compile_finished.rs new file mode 100644 index 0000000000..e009556c6d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/full_compile_finished.rs @@ -0,0 +1,36 @@ +//! Note the full compile's report, exit code and artifact writes, when a check is armed. + +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_delta::_helpers::verify::ACTUAL; +use crate::compile_delta::_helpers::verify::Actual; +use crate::compile_delta::_helpers::verify::EXPECTED; +use crate::compile_delta::_helpers::verify::locked; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use std::collections::HashMap; + +/// Note the full compile's report, exit code and artifact writes, when a check is armed. +pub fn full_compile_finished( + output: &Bound<'_, PyAny>, + (exit_code, artifacts_written): (i64, bool), + artifact_writes: &Bound<'_, PyAny>, +) -> PyResult<()> { + if locked(&EXPECTED).is_none() { + return Ok(()); + } + let (_, stdout): (Vec, Option) = output + .cast::()? + .get() + .texts(output.py())?; + let written: HashMap> = artifact_writes + .cast::() + .map(|writes| writes.get().written_digests()) + .unwrap_or_default(); + *locked(&ACTUAL) = Some(Actual { + stdout, + exit_code: Some(exit_code), + artifacts_written, + written, + }); + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/hold_expected.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/hold_expected.rs new file mode 100644 index 0000000000..79b4925e67 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/hold_expected.rs @@ -0,0 +1,12 @@ +//! Hold the delta's result for comparison once the full compile finishes. + +use crate::compile_delta::_helpers::verify::ACTUAL; +use crate::compile_delta::_helpers::verify::EXPECTED; +use crate::compile_delta::_helpers::verify::locked; +use crate::compile_delta::models::Expected; + +/// Hold the delta's result for comparison once the full compile finishes. +pub fn hold_expected(expected: Expected) { + *locked(&EXPECTED) = Some(expected); + *locked(&ACTUAL) = None; +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/mod.rs new file mode 100644 index 0000000000..1674cddbb5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/mod.rs @@ -0,0 +1,6 @@ +pub mod compiled_digests; +pub mod differences; +pub mod disarm; +pub mod full_compile_finished; +pub mod hold_expected; +pub mod write_mismatch; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/main/verify/write_mismatch.rs b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/write_mismatch.rs new file mode 100644 index 0000000000..6e8f8cc8a8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/main/verify/write_mismatch.rs @@ -0,0 +1,11 @@ +//! Write the differences beside the compiler cache for inspection. + +use crate::compile_delta::_helpers::verify::MISMATCH_FILE; +use std::path::Path; + +/// Write the differences beside the compiler cache for inspection. +pub fn write_mismatch(cache_dir: &Path, differences: &[String]) { + let _ = std::fs::create_dir_all(cache_dir); + let text: String = serde_json::to_string_pretty(differences).unwrap_or_default(); + let _ = std::fs::write(cache_dir.join(MISMATCH_FILE), text); +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/mod.rs b/crates/sqlbuild-python-compile/src/compile_delta/mod.rs new file mode 100644 index 0000000000..56cacb3165 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/mod.rs @@ -0,0 +1,9 @@ +//! Delta compiles: the stored project image, the changed models and SQL tests, and the fallback to a full compile. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/models.rs b/crates/sqlbuild-python-compile/src/compile_delta/models.rs new file mode 100644 index 0000000000..6b9471f073 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/models.rs @@ -0,0 +1,421 @@ +//! Public records of compile delta. + +use crate::compile_delta::_helpers::image::FamiliesByTable; +use crate::compile_delta::_helpers::image::RULE_ROWS_SECTION; +use crate::compile_delta::_helpers::image::RULES_SECTION; +use crate::compile_delta::_helpers::image::StoredFacts; +use crate::compile_delta::_helpers::image::decoded_rules; +use crate::compile_delta::_helpers::image::differing; +use crate::compile_delta::_helpers::image::encoded_strings; +use crate::compile_delta::_helpers::image::reads_any_model; +use crate::compile_delta::_helpers::image::restored; +use crate::compile_delta::_helpers::image::section; +use crate::compile_delta::constants::FACTS_SECTION; +use crate::compile_delta::constants::GATE_SECTION; +use crate::compile_delta::constants::IMAGE_FORMAT; +use crate::compile_delta::constants::RULE_DIGESTS_SECTION; +use crate::compile_driver::types::PhasesOutcome; +use pyo3::prelude::PyErr; +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_discovery::header_changes::types::HeaderKeys; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_context::models::Decoder; +use sqlbuild_python_base::compile_context::models::Encoder; +use sqlbuild_python_base::compile_context::models::ImageCapture; +use sqlbuild_python_base::compile_context::models::ModelCoverage; +use sqlbuild_python_base::compile_context::models::PendingRules; +use sqlbuild_python_base::compile_context::models::RecordMerge; +use sqlbuild_python_base::compile_context::types::ModelShape; +use sqlbuild_rules::models::RetainedRows; +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +/// A delta compile that finished, ready to be stored and printed or compared. +#[derive(Debug)] +pub struct Finished<'py> { + pub stdout: String, + pub stderr_lines: Vec, + pub phases: PhasesOutcome<'py>, + pub image: Option, + pub merge: RecordMerge, + pub changed_files: usize, + pub rerun_models: usize, +} + +/// What one delta attempt came to. +#[derive(Debug)] +pub enum DeltaOutcome<'py> { + /// The delta cannot reproduce this compile; the reason is for diagnostics only. + Fallback(String), + /// The compile stopped on an error from a changed file, exactly as a full compile would. + Raised(PyErr), + Finished(Box>), +} + +/// Everything a delta compile reads of the compile it patches. +#[derive(Debug, Default)] +pub struct ProjectImage { + pub format: String, + pub json_output: bool, + pub models: Vec, + pub usages: BTreeMap, + pub tests: Vec, + pub relations: ImageRelations, + pub fact_digests: Vec<(Vec, String)>, + pub model_subjects: Option)>>, + pub diagnostics: Vec, + pub text_models: Option>, + pub environment_names: Vec, + pub pinned_sources: Vec, + pub test_sources: BTreeMap>, + pub model_shapes: Vec, + pub coverage: Option>, + pub lint_model_keys: Option>, + pub lint_file_keys: Option)>>, + pub header_keys: BTreeMap, + pub audit_references: BTreeMap>, + pub rules: Option, +} + +/// An image's rules request: decoding from a stored image, or as a compile just retained it. +#[derive(Debug)] +pub enum ImageRules { + Decoding(PendingRules), + Retained(RetainedRows), +} + +impl ImageRules { + /// The rules request still decoding from a stored image, if this is one. + pub fn pending(self) -> Option { + match self { + Self::Decoding(rules) => Some(rules), + Self::Retained(_) => None, + } + } + + /// The rows a compile just retained, if this is them. + pub fn retained(self) -> Option { + match self { + Self::Retained(rules) => Some(rules), + Self::Decoding(_) => None, + } + } +} + +/// The analysis session's final relation facts, as stored. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ImageRelations { + pub types: Shapes, + pub nullability: Shapes, + pub complete: Shapes, + pub schemas: Shapes, + pub dependency_ordered: bool, + pub cyclic: bool, + pub legacy_models: Vec, + pub families: FamiliesByTable, +} + +impl From for ImageRelations { + fn from(tables: RelationTables) -> Self { + Self { + types: tables.types, + nullability: tables.nullability, + complete: tables.complete, + schemas: tables.schemas, + dependency_ordered: tables.dependency_ordered, + cyclic: tables.cyclic, + legacy_models: tables.legacy_models, + families: tables.families, + } + } +} + +/// What a delta compile reads first to decide, cheaply, whether it can patch the image. +#[derive(Clone, Debug, Default)] +pub struct ImageGate { + pub format: String, + pub json_output: bool, + /// Whether the stored compile reported any error diagnostic. + pub has_errors: bool, + /// Whether custom rules recorded which models they select. + pub has_model_subjects: bool, + pub models: Vec, + pub tests: Vec, + pub fact_digests: Vec<(Vec, String)>, + pub header_keys: BTreeMap, +} + +impl ImageGate { + pub(crate) fn encode(&self) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.string(&self.format); + encoder.flag(self.json_output); + encoder.flag(self.has_errors); + encoder.flag(self.has_model_subjects); + encoder.count(self.models.len()); + for model in &self.models { + encoder.string(&model.name); + encoder.string(&model.relative_path); + encoder.pairs(&model.upstream); + } + encoder.count(self.tests.len()); + for test in &self.tests { + encoder.string(&test.key); + encoder.string(&test.relative_path); + encoder.strings(test.chain.iter().map(String::as_str)); + } + encoder.count(self.fact_digests.len()); + for (key, digest) in &self.fact_digests { + encoder.strings(key.iter().map(String::as_str)); + encoder.string(digest); + } + encoder.count(self.header_keys.len()); + for (path, keys) in &self.header_keys { + encoder.string(path); + encoder.pairs(keys); + } + encoder.finish() + } + + /// The gate stored in `sections`, or `None` when it is unreadable or of another format. + pub fn decode(sections: &ImageSections) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(section(sections, GATE_SECTION)?)?; + let format: String = decoder.string()?; + if format != IMAGE_FORMAT { + return None; + } + let gate: Self = Self { + format, + json_output: decoder.flag()?, + has_errors: decoder.flag()?, + has_model_subjects: decoder.flag()?, + models: decoder.list(|decoder| { + Some(CapturedModel { + name: decoder.string()?, + relative_path: decoder.string()?, + upstream: decoder.pairs()?, + }) + })?, + tests: decoder.list(|decoder| { + Some(CapturedTest { + key: decoder.string()?, + relative_path: decoder.string()?, + chain: decoder.strings()?, + }) + })?, + fact_digests: decoder.list(|decoder| Some((decoder.strings()?, decoder.string()?)))?, + header_keys: decoder + .list(|decoder| Some((decoder.string()?, decoder.pairs()?)))? + .into_iter() + .collect(), + }; + decoder.finish()?; + Some(gate) + } +} + +impl ProjectImage { + /// The image of a finished compile's capture, or why it cannot be a delta base. + pub fn from_capture(capture: ImageCapture, json_output: bool) -> Result { + if let Some(reason) = capture.unsupported { + return Err(reason); + } + if capture.stdlib_json { + return Err("the report fell back to the standard library serializer".to_owned()); + } + let Some(relations) = capture.relations else { + return Err("no model analysis session ran".to_owned()); + }; + if !json_output && capture.text_models.is_none() { + return Err("no text report was rendered".to_owned()); + } + let Some(model_shapes) = capture.model_shapes else { + return Err("no semantic shapes were published".to_owned()); + }; + let mut environment_names: Vec = capture.environment_names; + environment_names.sort(); + environment_names.dedup(); + Ok(Self { + format: IMAGE_FORMAT.to_owned(), + json_output, + models: capture.models, + usages: capture.usages, + tests: capture.test_chains, + relations: relations.into(), + fact_digests: capture.fact_digests.into_iter().collect(), + model_subjects: capture.model_subjects, + diagnostics: capture.diagnostics, + text_models: capture.text_models, + environment_names, + pinned_sources: capture.pinned_sources.into_iter().collect(), + test_sources: capture + .test_sources + .into_iter() + .map(|(path, names)| (path, names.into_iter().collect())) + .collect(), + model_shapes, + coverage: capture.coverage, + lint_model_keys: capture.lint_model_keys, + lint_file_keys: capture.lint_file_keys, + header_keys: capture.header_keys, + audit_references: capture.audit_references, + rules: capture + .rules + .map(|rules| ImageRules::Retained(rules.retained)), + }) + } + + /// The image as the sections of its file. + pub fn encode(self) -> Result { + let gate: ImageGate = ImageGate { + format: self.format, + json_output: self.json_output, + has_errors: self.diagnostics.iter().any(|diagnostic| diagnostic.error), + has_model_subjects: self.model_subjects.is_some(), + models: self.models, + tests: self.tests, + fact_digests: self.fact_digests, + header_keys: self.header_keys, + }; + let relations: ImageRelations = self.relations; + let types: &Shapes = &relations.types; + let facts: StoredFacts = StoredFacts { + usages: self.usages, + complete: differing(&relations.complete, types), + schemas: differing(&relations.schemas, types), + model_shapes: differing(&self.model_shapes, types), + types: relations.types, + nullability: relations.nullability, + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models, + families: relations.families, + model_subjects: self.model_subjects, + diagnostics: self.diagnostics, + text_models: self.text_models, + environment_names: self.environment_names, + coverage: self.coverage, + pinned_sources: self.pinned_sources, + test_sources: self.test_sources, + lint_model_keys: self.lint_model_keys, + lint_file_keys: self.lint_file_keys, + audit_references: self.audit_references, + }; + let rules: Option = self.rules.and_then(ImageRules::retained); + let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); + let rules = rules.map(|rules| { + ( + rules.request_json, + rules.rows, + encoded_strings(&rules.digests), + ) + }); + let mut sections: ImageSections = vec![ + (GATE_SECTION.to_owned(), gate), + (FACTS_SECTION.to_owned(), facts?), + ]; + if let Some((request, rows, digests)) = rules { + sections.push((RULES_SECTION.to_owned(), request)); + sections.push((RULE_ROWS_SECTION.to_owned(), rows)); + sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); + } + Ok(sections) + } + + /// The image behind `gate`, its rules request decoding on its own thread; `None` if unreadable. + pub fn decode(mut sections: ImageSections, gate: ImageGate) -> Option { + let mut taken = |name: &str| { + sections + .iter_mut() + .find(|(section, _)| section == name) + .map(|(_, bytes)| std::mem::take(bytes)) + }; + let stored_rules = ( + taken(RULES_SECTION), + taken(RULE_ROWS_SECTION), + taken(RULE_DIGESTS_SECTION), + ); + let rules: Option = match stored_rules { + (Some(request), Some(rows), Some(digests)) => { + Some(ImageRules::Decoding(PendingRules::Decoding( + std::thread::spawn(move || decoded_rules(request, rows, &digests)), + ))) + } + (None, None, None) => None, + _ => return None, + }; + let facts: StoredFacts = StoredFacts::decode(section(§ions, FACTS_SECTION)?)?; + let types: Shapes = facts.types; + let relations: ImageRelations = ImageRelations { + complete: restored(facts.complete, &types)?, + schemas: restored(facts.schemas, &types)?, + nullability: facts.nullability, + dependency_ordered: facts.dependency_ordered, + cyclic: facts.cyclic, + legacy_models: facts.legacy_models, + families: facts.families, + types: Shapes::new(), + }; + let model_shapes: Shapes = restored(facts.model_shapes, &types)?; + Some(Self { + format: gate.format, + json_output: gate.json_output, + models: gate.models, + usages: facts.usages, + tests: gate.tests, + relations: ImageRelations { types, ..relations }, + fact_digests: gate.fact_digests, + model_subjects: facts.model_subjects, + diagnostics: facts.diagnostics, + text_models: facts.text_models, + environment_names: facts.environment_names, + pinned_sources: facts.pinned_sources, + test_sources: facts.test_sources, + model_shapes, + coverage: facts.coverage, + lint_model_keys: facts.lint_model_keys, + lint_file_keys: facts.lint_file_keys, + header_keys: gate.header_keys, + audit_references: facts.audit_references, + rules, + }) + } + + /// Every model whose reads include a relation of `names`, outside `names`. + pub fn dependents(&self, names: &BTreeSet) -> BTreeSet { + self.models + .iter() + .filter(|model| !names.contains(&model.name)) + .filter(|model| reads_any_model(model, names)) + .map(|model| model.name.clone()) + .collect() + } +} + +/// What the splice replaces: the rerun models, their audits and the rerun SQL tests. +#[derive(Debug)] +pub struct SpliceScope<'a> { + pub models: &'a BTreeSet, + pub tests: &'a BTreeSet, + /// The sources the delta discovered, when it discovered only some; `None` when all. + pub sources: Option<&'a BTreeSet>, +} + +/// What the delta compile produced, held until the full compile finishes. +#[derive(Debug, Default)] +pub struct Expected { + pub stdout: Option, + pub exit_code: Option, + pub raised: Option, + /// Every file under the compiled root after the delta, with its native digest. + pub compiled: BTreeMap, + pub compiled_root: String, + pub artifacts_written: bool, + /// Every file under `target/compiled-failed/` after the delta, with its native digest. + pub failed: BTreeMap, + pub failed_root: String, +} diff --git a/crates/sqlbuild-python-compile/src/compile_delta/tests.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests.rs new file mode 100644 index 0000000000..e73f7237fe --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/test_coverage.rs"] +mod coverage; +#[path = "tests/test_delta_facts.rs"] +mod delta_facts; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_image.rs"] +mod image; +#[path = "tests/test_splice.rs"] +mod splice; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/tests/helpers.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/helpers.rs new file mode 100644 index 0000000000..6ba7133e99 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/helpers.rs @@ -0,0 +1,263 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; +use sqlbuild_cache::project_reuse::types::ImageSections; + +use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; + +use crate::compile_delta::constants::FACTS_SECTION; +use crate::compile_delta::constants::GATE_SECTION; +use crate::compile_delta::constants::IMAGE_FORMAT; +use crate::compile_delta::constants::RULE_DIGESTS_SECTION; +use crate::compile_delta::models::ImageRelations; +use crate::compile_delta::models::ImageRules; +use crate::compile_delta::models::ProjectImage; +use sqlbuild_python_base::compile_context::models::BaseRules; +use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; +use sqlbuild_python_base::compile_context::models::CapturedModel; +use sqlbuild_python_base::compile_context::models::CapturedTest; +use sqlbuild_python_base::compile_context::models::ModelCoverage; +use sqlbuild_python_base::compile_context::models::PendingRules; + +/// The rows of a request without models: a zero model count. +const RULE_ROWS: [u8; 8] = [0; 8]; +const OTHER_FORMAT: &str = "sqlbuild-project-image-v0"; + +pub(crate) fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { + values + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +pub(crate) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// An image holding at least one value of every kind it stores. +pub(crate) fn image() -> ProjectImage { + let order_types: Vec<(String, String)> = pairs(&[("order_id", "BIGINT"), ("amount", "DOUBLE")]); + let customer_types: Vec<(String, String)> = pairs(&[("customer_id", "BIGINT")]); + let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(); + ProjectImage { + format: String::new(), + json_output: true, + models: vec![ + CapturedModel { + name: "orders".to_owned(), + relative_path: "models/orders.sql".to_owned(), + upstream: pairs(&[("source", "raw_orders"), ("model", "customers")]), + }, + CapturedModel { + name: "customers".to_owned(), + relative_path: "models/customers.sql".to_owned(), + upstream: Vec::new(), + }, + ], + usages: BTreeMap::from([("models/orders.sql".to_owned(), "digest-1".to_owned())]), + tests: vec![CapturedTest { + key: "orders_test".to_owned(), + relative_path: "tests/unit/orders.sql".to_owned(), + chain: strings(&["orders", "customers"]), + }], + relations: ImageRelations { + types: vec![ + ("orders".to_owned(), order_types.clone()), + ("customers".to_owned(), customer_types.clone()), + ], + nullability: vec![("orders".to_owned(), pairs(&[("order_id", "never")]))], + complete: vec![("orders".to_owned(), order_types.clone())], + schemas: vec![("customers".to_owned(), pairs(&[("customer_id", "INTEGER")]))], + dependency_ordered: true, + cyclic: false, + legacy_models: strings(&["customers"]), + families: vec![( + "order_pivot".to_owned(), + vec![DynamicFamily { + name: "category_amounts".to_owned(), + pivot_column: "category".to_owned(), + value_column: "amount".to_owned(), + aggregate: "MAX".to_owned(), + data_type: "DOUBLE".to_owned(), + name_pattern: Some("^amount_".to_owned()), + }], + )], + }, + fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], + model_subjects: Some(vec![ + ( + "models/orders.sql".to_owned(), + "orders".to_owned(), + Some("table".to_owned()), + ), + ( + "models/customers.sql".to_owned(), + "customers".to_owned(), + None, + ), + ]), + diagnostics: vec![CapturedDiagnostic { + json: "{\"code\": \"W1\"}".to_owned(), + resource_name: Some("orders".to_owned()), + paths: strings(&["models/orders.sql"]), + error: false, + }], + text_models: Some(vec![("orders".to_owned(), true, 3)]), + environment_names: strings(&["SQLBUILD_TARGET"]), + pinned_sources: strings(&["raw_customers"]), + test_sources: BTreeMap::from([( + "tests/unit/orders.sql".to_owned(), + strings(&["raw_orders"]), + )]), + model_shapes: vec![ + ("orders".to_owned(), order_types), + ("customers".to_owned(), pairs(&[("customer_id", "VARCHAR")])), + ], + coverage: Some(vec![ModelCoverage { + name: "orders".to_owned(), + disabled: false, + blocked: true, + partial: strings(&["open source: raw_orders"]), + }]), + lint_model_keys: Some(pairs(&[("models/orders.sql", "lint-key")])), + lint_file_keys: Some(vec![ + ("models/orders.sql".to_owned(), Some("file-key".to_owned())), + ("models/customers.sql".to_owned(), None), + ]), + header_keys: BTreeMap::from([( + "models/orders.sql".to_owned(), + pairs(&[("description", "0123")]), + )]), + audit_references: BTreeMap::from([( + "models/orders.sql".to_owned(), + strings(&["order_quality"]), + )]), + rules: Some(ImageRules::Retained(RetainedRows { + request_json: request, + rows: RULE_ROWS.to_vec(), + digests: Vec::new(), + })), + } +} + +/// Every stored fact of `image` but its rules, in a comparable form. +pub(crate) fn facts(image: &ProjectImage) -> String { + format!( + "{:?}", + ( + ( + &image.json_output, + &image.models, + &image.usages, + &image.tests, + &image.relations, + &image.fact_digests, + &image.model_subjects, + ), + ( + &image.diagnostics, + &image.text_models, + &image.environment_names, + &image.pinned_sources, + &image.test_sources, + &image.model_shapes, + &image.coverage, + &image.lint_model_keys, + &image.lint_file_keys, + &image.header_keys, + &image.audit_references, + ), + ) + ) +} + +pub(crate) fn current_format() -> String { + IMAGE_FORMAT.to_owned() +} + +pub(crate) fn encoded(image: ProjectImage) -> ImageSections { + let stored: ProjectImage = ProjectImage { + format: current_format(), + ..image + }; + stored.encode().unwrap_or_default() +} + +/// The image `image()` without any of its optional facts. +pub(crate) fn sparse_image() -> ProjectImage { + ProjectImage { + model_subjects: None, + text_models: None, + lint_model_keys: None, + lint_file_keys: None, + rules: None, + ..image() + } +} + +/// The rules request a decoded image holds, resolved. +pub(crate) fn decoded_rules(image: ProjectImage) -> Option { + image + .rules + .and_then(ImageRules::pending) + .and_then(PendingRules::resolved) +} + +/// `sections` with the gate of an image of another format. +pub(crate) fn with_other_format(sections: &mut ImageSections) { + let other: ProjectImage = ProjectImage { + format: OTHER_FORMAT.to_owned(), + ..ProjectImage::default() + }; + let gate: Vec = other + .encode() + .unwrap_or_default() + .into_iter() + .find(|(name, _)| name == GATE_SECTION) + .map(|(_, bytes)| bytes) + .unwrap_or_default(); + sections + .iter_mut() + .filter(|(name, _)| name == GATE_SECTION) + .for_each(|section| section.1 = gate.clone()); +} + +/// `sections` with their facts cut in half. +pub(crate) fn with_truncated_facts(sections: &mut ImageSections) { + sections + .iter_mut() + .filter(|(name, _)| name == FACTS_SECTION) + .for_each(|(_, bytes)| bytes.truncate(bytes.len() / 2)); +} + +/// `sections` without the rules request's digests. +pub(crate) fn without_rule_digests(sections: &mut ImageSections) { + sections.retain(|(name, _)| name != RULE_DIGESTS_SECTION); +} + +/// The names in `values`, as a set. +pub(crate) fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// One model's semantic coverage facts. +pub(crate) fn coverage( + name: &str, + (disabled, blocked): (bool, bool), + partial: &[&str], +) -> ModelCoverage { + ModelCoverage { + name: name.to_owned(), + disabled, + blocked, + partial: strings(partial), + } +} + +/// Models paired with their upstream names, or with their reasons. +pub(crate) fn order(models: &[(&str, &[&str])]) -> Vec<(String, Vec)> { + models + .iter() + .map(|(name, upstream)| ((*name).to_owned(), strings(upstream))) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_coverage.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/tests/test_coverage.rs rename to crates/sqlbuild-python-compile/src/compile_delta/tests/test_coverage.rs index b3d4fcc24f..950d3900a2 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_coverage.rs @@ -1,8 +1,8 @@ use std::collections::BTreeSet; -use crate::bindings::_helpers::compile_delta::driver::names_uncompiled_model; -use crate::bindings::tests::helpers::{coverage, names, order, strings}; -use crate::bindings::tests::test_types::{CoverageTestCase, UncompiledModelTestCase}; +use crate::compile_delta::_helpers::driver::names_uncompiled_model; +use crate::compile_delta::tests::helpers::{coverage, names, order, strings}; +use crate::compile_delta::tests::test_types::{CoverageTestCase, UncompiledModelTestCase}; use sqlbuild_python_base::compile_context::main::coverage::coverage_reasons::coverage_reasons; use sqlbuild_python_base::compile_context::main::coverage::merged_coverage::merged_coverage; use sqlbuild_python_base::compile_context::main::coverage::merged_order::merged_order; diff --git a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_delta_facts.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs rename to crates/sqlbuild-python-compile/src/compile_delta/tests/test_delta_facts.rs index 4b582b8eff..45839fa62a 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_delta_facts.rs @@ -1,8 +1,8 @@ use std::collections::BTreeSet; -use crate::bindings::_helpers::rules::fact_replay::delta_safe_fact; -use crate::bindings::tests::helpers::{names, strings}; -use crate::bindings::tests::test_types::DeltaFactTestCase; +use crate::compile_delta::tests::helpers::{names, strings}; +use crate::compile_delta::tests::test_types::DeltaFactTestCase; +use crate::rules::_helpers::fact::fact_replay::delta_safe_fact; #[test] fn given_custom_rule_fact_when_a_delta_compiles_some_models_then_only_facts_it_holds_are_safe() { diff --git a/crates/sqlbuild-python/src/bindings/tests/test_image.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_image.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/tests/test_image.rs rename to crates/sqlbuild-python-compile/src/compile_delta/tests/test_image.rs index 16c9cbbe23..1f76d4b8d6 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_image.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_image.rs @@ -1,11 +1,12 @@ use sqlbuild_cache::project_reuse::types::ImageSections; -use crate::bindings::_helpers::compile_delta::image::{ImageGate, ProjectImage}; -use crate::bindings::tests::helpers::{ +use crate::compile_delta::models::ImageGate; +use crate::compile_delta::models::ProjectImage; +use crate::compile_delta::tests::helpers::{ current_format, decoded_rules, encoded, facts, image, sparse_image, with_other_format, with_truncated_facts, without_rule_digests, }; -use crate::bindings::tests::test_types::{DamagedImageTestCase, ImageRoundTripTestCase}; +use crate::compile_delta::tests::test_types::{DamagedImageTestCase, ImageRoundTripTestCase}; #[test] fn given_project_image_when_encoded_and_decoded_then_every_fact_and_the_rules_survive() { diff --git a/crates/sqlbuild-python/src/bindings/tests/test_splice.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_splice.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/tests/test_splice.rs rename to crates/sqlbuild-python-compile/src/compile_delta/tests/test_splice.rs index 705202a2fe..5a20d3ab1e 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_splice.rs +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_splice.rs @@ -1,8 +1,9 @@ use std::collections::BTreeSet; -use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; -use crate::bindings::tests::helpers::names; -use crate::bindings::tests::test_types::SpliceTestCase; +use crate::compile_delta::_helpers::splice::spliced_report; +use crate::compile_delta::models::SpliceScope; +use crate::compile_delta::tests::helpers::names; +use crate::compile_delta::tests::test_types::SpliceTestCase; const BASE: &str = r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [], "compile_timings": {"total_ms": 9}}"#; diff --git a/crates/sqlbuild-python-compile/src/compile_delta/tests/test_types.rs b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_types.rs new file mode 100644 index 0000000000..a616381a42 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_delta/tests/test_types.rs @@ -0,0 +1,58 @@ +use sqlbuild_cache::project_reuse::types::ImageSections; + +use crate::compile_delta::models::ProjectImage; +use sqlbuild_python_base::compile_context::models::ModelCoverage; + +/// A stored report, a delta report and what splicing the delta into the stored one gives. +pub(crate) struct SpliceTestCase { + pub(crate) description: &'static str, + pub(crate) base: &'static str, + pub(crate) delta: &'static str, + pub(crate) models: &'static [&'static str], + pub(crate) tests: &'static [&'static str], + pub(crate) sources: Option<&'static [&'static str]>, + /// The spliced report, or `None` when the reports must not line up. + pub(crate) expected_report: Option<&'static str>, +} + +/// Stored image bytes damaged before they are read back. +pub(crate) struct DamagedImageTestCase { + pub(crate) description: &'static str, + pub(crate) damage: fn(&mut ImageSections), + pub(crate) expected_gate: bool, + pub(crate) expected_image: bool, +} + +/// An image encoded and decoded again. +pub(crate) struct ImageRoundTripTestCase { + pub(crate) description: &'static str, + pub(crate) image: fn() -> ProjectImage, + pub(crate) expected_model_subjects: bool, + /// The retained rules request's model and digest count, or `None` without one. + pub(crate) expected_rule_models: Option, +} + +/// Stored and fresh semantic coverage facts, and the whole project's partial-check reasons. +pub(crate) struct CoverageTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) fresh: Vec, + pub(crate) stored_order: Vec<(String, Vec)>, + pub(crate) fresh_order: Vec<(String, Vec)>, + pub(crate) expected_reasons: Vec<(String, Vec)>, +} + +/// An error message a delta round reported. +pub(crate) struct UncompiledModelTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_uncompiled: bool, +} + +/// A custom-rule fact key a delta compile holding some models may or may not read in full. +pub(crate) struct DeltaFactTestCase { + pub(crate) description: &'static str, + pub(crate) key: &'static [&'static str], + pub(crate) all_sources: bool, + pub(crate) expected_safe: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs index 774472d267..acbf1c23a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs @@ -13,14 +13,15 @@ use sqlbuild_host::compile_context::main::record_compile_cpu_timing::record_comp use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis_session::model_analyses::{ - ModelAnalyses, ModelAnalysisRequest, analyze_models, -}; -use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; -use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; -use crate::bindings::_helpers::project_assembly::declared_relations::{ - DeclaredRelations, DeclaredShapes, declared_relations, -}; +use crate::analysis_session::main::analyze_models::analyze_models; +use crate::analysis_session::models::ModelAnalyses; +use crate::analysis_session::models::ModelAnalysisRequest; +use crate::compile_driver::_helpers::stages::analysis_cache::analysis_cache_context; +use crate::compile_driver::_helpers::stages::assembly::AssemblyRequest; +use crate::compile_driver::_helpers::stages::assembly::AssemblyStage; +use crate::project_assembly::main::declared_relations::declared_relations; +use crate::project_assembly::models::DeclaredRelations; +use crate::project_assembly::models::DeclaredShapes; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::compile_context::models::PythonProcessClock; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_checks.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_checks.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_facts.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_facts.rs index 22b0b991b9..50e172d767 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_facts.rs @@ -4,11 +4,12 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; -use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; -use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; -use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; -use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_rows; +use crate::compile_driver::_helpers::assembly::assembly_analysis::AnalysisContext; +use crate::project_assembly::main::assemble_native_project_resources::assemble_native_project_resources; +use crate::project_assembly::main::model_syntax_checks::model_syntax_checks; +use crate::project_assembly::main::standalone_pivot_rows::standalone_pivot_rows; +use crate::project_assembly::models::NativeModelFacts; +use crate::project_assembly::models::NativeProjectFacts; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::compile_driver::constants::PIVOT_PROOFS; use sqlbuild_python_base::compile_driver::constants::SESSION_PIVOT_PROOFS; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs index 4b396dea53..39afe0b4ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs @@ -7,15 +7,15 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, P use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; use sqlbuild_analysis::analysis_route::models::AnalysisRoute; -use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; -use crate::bindings::_helpers::compile_driver::assembly_analysis::{ - AnalysisContext, model_stem, selected_for_analysis, -}; -use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; -use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; -use crate::bindings::_helpers::project_assembly::syntax_validation::{ - ParseOptions, validate_hook_sql_syntax, validate_sql_syntax, -}; +use crate::analysis_session::models::ModelAnalyses; +use crate::compile_driver::_helpers::assembly::assembly_analysis::AnalysisContext; +use crate::compile_driver::_helpers::assembly::assembly_analysis::model_stem; +use crate::compile_driver::_helpers::assembly::assembly_analysis::selected_for_analysis; +use crate::hand_built::main::compiled_model::compiled_model as hand_built_model; +use crate::project_assembly::main::model_placeholders::model_placeholders; +use crate::project_assembly::main::validate_hook_sql_syntax::validate_hook_sql_syntax; +use crate::project_assembly::main::validate_sql_syntax::validate_sql_syntax; +use crate::project_assembly::types::ParseOptions; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs index c5811e1da4..b05f339fc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs @@ -9,24 +9,22 @@ use sqlbuild_analysis::assembly::project::main::configured_namespace::configured use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compile_attachment::target::{ - connection_is_default_target, project_location, -}; -use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; -use crate::bindings::_helpers::compile_driver::assembly_analysis::{ - AnalysisContext, analysis_context, and_not, -}; -use crate::bindings::_helpers::compile_driver::assembly_checks::compiled_sql_scenarios; -use crate::bindings::_helpers::compile_driver::assembly_facts::{ - native_project_facts, pivot_tables, -}; -use crate::bindings::_helpers::compile_driver::assembly_models::{ - binding_rows, compiled_models, project_binding_diagnostics, -}; -use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::semantic_checks::stage::{ - SemanticCompletion, completed_semantic_project, -}; +use crate::compile_attachment::main::connection_is_default_target::connection_is_default_target; +use crate::compile_attachment::main::project_location::project_location; +use crate::compile_driver::_helpers::assembly::assembly_analysis::AnalysisContext; +use crate::compile_driver::_helpers::assembly::assembly_analysis::analysis_context; +use crate::compile_driver::_helpers::assembly::assembly_analysis::and_not; +use crate::compile_driver::_helpers::assembly::assembly_checks::compiled_sql_scenarios; +use crate::compile_driver::_helpers::assembly::assembly_facts::native_project_facts; +use crate::compile_driver::_helpers::assembly::assembly_facts::pivot_tables; +use crate::compile_driver::_helpers::assembly::assembly_models::binding_rows; +use crate::compile_driver::_helpers::assembly::assembly_models::compiled_models; +use crate::compile_driver::_helpers::assembly::assembly_models::project_binding_diagnostics; +use crate::compile_driver::_helpers::assembly::assembly_sql_tests::assembled_sql_tests; +use crate::compile_driver::_helpers::stages::assembly::AssemblyRequest; +use crate::compile_driver::_helpers::stages::assembly::AssemblyStage; +use crate::semantic_checks::main::completed_semantic_project::completed_semantic_project; +use crate::semantic_checks::models::SemanticCompletion; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::compile_driver::constants::SEMANTIC_COMPLETION; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_sql_tests.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_sql_tests.rs index fa8b6fc2d4..4250d05e8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_sql_tests.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyList, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; -use crate::bindings::_helpers::sql_test_glue::compiled_tests::assemble_native_sql_tests; +use crate::compile_driver::_helpers::assembly::assembly_checks::test_model_query_overrides; +use crate::sql_test_glue::main::assemble_native_sql_tests::assemble_native_sql_tests; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/mod.rs new file mode 100644 index 0000000000..88999fb4fd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod assembly_analysis; +pub(crate) mod assembly_checks; +pub(crate) mod assembly_facts; +pub(crate) mod assembly_models; +pub(crate) mod assembly_project; +pub(crate) mod assembly_sql_tests; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs new file mode 100644 index 0000000000..4b77187d1e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod assembly; +pub(crate) mod setup; +pub(crate) mod stages; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs index 16dbc2b370..afe4871bc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs @@ -1,33 +1,33 @@ //! The first attached compile input snapshot: target, vars, macros, scopes, models and resources. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::compile_driver::main::setup::attached_inputs::attached_inputs; +use crate::compile_driver::models::InputsRequest; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; -use sqlbuild_python_front::compile_attachment::main::build_effective_connection::build_effective_connection; -use sqlbuild_python_front::compile_attachment::main::build_effective_settings::build_effective_settings; -use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; -use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; -use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; -use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_views::main::project::compile_inputs_view::compile_inputs_view; -use sqlbuild_python_project::compiled_views::main::project::inputs_with::inputs_with; -use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; -use crate::bindings::_helpers::input_loops::attachment::{attach_checks, attach_resources}; -use sqlbuild_python_front::model_config::main::config::run_id::run_id; -use sqlbuild_python_front::render::main::declarations::public_declaration_indexes::public_declaration_indexes; -use sqlbuild_python_front::render::main::declarations::public_model_schema_index::public_model_schema_index; +use crate::compile_attachment::main::build_declaration_scope::build_declaration_scope; +use crate::compile_attachment::main::build_model_inputs::build_model_inputs; +use crate::input_loops::main::attach_checks::attach_checks; +use crate::input_loops::main::attach_resources::attach_resources; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection::build_effective_connection; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings::build_effective_settings; +use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::model_config::main::config::run_id::run_id; +use sqlbuild_python_front::render::main::declarations::public_declaration_indexes::public_declaration_indexes; +use sqlbuild_python_front::render::main::declarations::public_model_schema_index::public_model_schema_index; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::project::compile_inputs_view::compile_inputs_view; const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; const OPTIONAL: [&str; 8] = [ @@ -41,42 +41,6 @@ const OPTIONAL: [&str; 8] = [ "no_cache", ]; -/// The arguments of one compile input attachment; unset flags are `False`, other options `None`. -pub(crate) struct InputsRequest { - pub(crate) discovered_inputs: Py, - pub(crate) adapter_context: Py, - pub(crate) selected_target: Py, - pub(crate) cli_vars: Py, - pub(crate) run_id: Py, - pub(crate) no_sql_validation: Py, - pub(crate) defer_model_sql_validation: Py, - pub(crate) external_sql_reference_resolver: Py, - pub(crate) resolved_connection: Py, - pub(crate) no_cache: Py, -} - -impl InputsRequest { - /// The request `build_compile_inputs` was called with. - fn read(arguments: &KeywordArguments<'_>) -> PyResult { - Ok(Self { - discovered_inputs: arguments.value("discovered_inputs")?.unbind(), - adapter_context: arguments.value("adapter_context")?.unbind(), - selected_target: arguments.value("selected_target")?.unbind(), - cli_vars: arguments.value("cli_vars")?.unbind(), - run_id: arguments.value("run_id")?.unbind(), - no_sql_validation: arguments.value_or_false("no_sql_validation")?.unbind(), - defer_model_sql_validation: arguments - .value_or_false("defer_model_sql_validation")? - .unbind(), - external_sql_reference_resolver: arguments - .value("external_sql_reference_resolver")? - .unbind(), - resolved_connection: arguments.value("resolved_connection")?.unbind(), - no_cache: arguments.value_or_false("no_cache")?.unbind(), - }) - } -} - /// The effective target, vars, settings and macro context every later stage shares. struct TargetContext<'py> { target_name: Bound<'py, PyAny>, @@ -107,7 +71,7 @@ struct AttachedResources<'py> { /// Attach discovered metadata into the first compile input snapshot. #[pyfunction] #[pyo3(signature = (**options))] -fn build_compile_inputs<'py>( +pub(crate) fn build_compile_inputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -116,35 +80,11 @@ fn build_compile_inputs<'py>( attached_inputs(py, &InputsRequest::read(&arguments)?) } -/// The first compile input snapshot, with diagnostics collected while attaching appended. -pub(crate) fn attached_inputs<'py>( +/// The undecorated `build_compile_inputs`. +pub(crate) fn compile_inputs<'py>( py: Python<'py>, request: &InputsRequest, ) -> PyResult> { - let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); - let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = - within(&collector, |violations| { - Ok((compile_inputs(py, request)?, violations.clone())) - })?; - let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; - if !collected.is_truthy()? { - return Ok(built); - } - let diagnostics: Bound<'py, PyTuple> = PyTuple::new( - py, - native_field(&built, "diagnostics")? - .try_iter()? - .chain(collected.try_iter()?) - .collect::>>()?, - )?; - inputs_with( - &built, - Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), - ) -} - -/// The undecorated `build_compile_inputs`. -fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult> { let discovered: Bound<'py, PyAny> = request.discovered_inputs.bind(py).clone(); let adapter_context: Bound<'py, PyAny> = request.adapter_context.bind(py).clone(); let target: TargetContext<'py> = target_context(request, &discovered, &adapter_context)?; @@ -484,7 +424,3 @@ fn effective_connection<'py>( &target.vars.clone(), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_compile_inputs, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/integration_loaders.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/integration_loaders.rs diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs new file mode 100644 index 0000000000..ba9e864c07 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod inputs; +pub(crate) mod integration_loaders; +pub(crate) mod pipeline; +pub(crate) mod pipeline_entries; +pub(crate) mod project; +pub(crate) mod python_plan_entries; +pub(crate) mod relation_targets; +pub(crate) mod target_defaults; +pub(crate) mod targets; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline.rs new file mode 100644 index 0000000000..9e585ae375 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline.rs @@ -0,0 +1,71 @@ +//! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. + +use crate::compile_driver::main::setup::progress::progress; +use pyo3::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods}; + +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; + +/// Raise one `CompileInputError` listing every compile error the command must stop on. +pub(crate) fn reject_compile_errors( + project: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, + on_progress: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = project.py(); + let plan_sql_tests: bool = native_field(options, "plan_sql_tests")?.is_truthy()?; + let sql_test: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "SQL_TEST")?; + let mut errors: Vec> = Vec::new(); + for diagnostic in native_field(project, "diagnostics")?.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + if native_field(&diagnostic, "is_error")?.is_truthy()? + && (plan_sql_tests || native_field(&diagnostic, "resource_type")?.ne(&sql_test)?) + { + errors.push(diagnostic); + } + } + let Some(first) = errors.first() else { + return Ok(()); + }; + progress(on_progress, "Project compilation failed.")?; + let mut lines: Vec = Vec::with_capacity(errors.len()); + let helps: Bound<'_, PySet> = PySet::empty(py)?; + for diagnostic in &errors { + let or_one = |name: &str| -> PyResult { + let value: Bound<'_, PyAny> = native_field(diagnostic, name)?; + Ok(if value.is_truthy()? { + value.str()?.to_string() + } else { + "1".to_owned() + }) + }; + lines.push(format!( + "[{}] {}:{}:{}: {}", + native_field(diagnostic, "code")?.str()?, + native_field(diagnostic, "path")?.str()?, + or_one("line")?, + or_one("column")?, + native_field(diagnostic, "message")?.str()?, + )); + helps.add(native_field(diagnostic, "help")?)?; + } + let help: Bound<'_, PyAny> = if helps.len() == 1 { + helps.pop().unwrap_or_else(|| py.None().into_bound(py)) + } else { + py.None().into_bound(py) + }; + let error: Bound<'_, PyAny> = error_class(py, ErrorId::CompileInput)?.call( + (lines.join("\n"),), + Some(&keywords( + py, + &[("code", native_field(first, "code")?), ("help", help)], + )?), + )?; + Err(PyErr::from_value(error)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs new file mode 100644 index 0000000000..05d6928b9f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs @@ -0,0 +1,86 @@ +//! Small pipeline entry points: analysis selection, project graphs and pipeline results. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// Discovered Python node functions whose selectors need the Python graph. +const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; +/// Plan entry lists that are runtime work. +pub(crate) const PLAN_WORK_ENTRIES: [&str; 6] = [ + "model_entries", + "seed_entries", + "source_load_entries", + "function_entries", + "audit_entries", + "test_entries", +]; + +/// Limit analysis when selectors cannot resolve through Python graph nodes. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn resolve_compile_analysis_selection<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_compile_analysis_selection", + options, + (&["options", "discovered_inputs"], &[]), + )?; + compile_analysis_selection( + &arguments.value("options")?, + &arguments.value("discovered_inputs")?, + ) +} + +/// `resolve_compile_analysis_selection` with its arguments. +pub(crate) fn compile_analysis_selection<'py>( + options: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = options.py(); + let mut has_python_nodes: bool = false; + for name in PYTHON_NODE_FUNCTIONS { + if native_field(discovered, name)?.is_truthy()? { + has_python_nodes = true; + break; + } + } + let no_cache: Bound<'py, PyAny> = native_field(options, "no_cache")?; + if (native_field(options, "resolve_python_run_selectors")?.is_truthy()? && has_python_nodes) + || native_field(options, "sql_test_selection")? + .getattr("names")? + .is_truthy()? + { + return new_record_with( + py, + RecordId::CompileAnalysisSelection, + &[("no_cache", no_cache)], + ); + } + new_record_with( + py, + RecordId::CompileAnalysisSelection, + &[ + ("select", native_field(options, "select")?), + ("exclude", native_field(options, "exclude")?), + ( + "auto_load_sources", + native_field(options, "auto_load_sources")?, + ), + ("no_cache", no_cache), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs similarity index 73% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 421fab6a16..4190c20d49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -1,51 +1,53 @@ //! Compiled project and project graph construction: frontier stages, target defaults and checks. +use crate::compile_driver::main::setup::attached_inputs::attached_inputs; +use crate::compile_driver::main::setup::compile_adapter_context::compile_adapter_context; +use crate::compile_driver::main::setup::preserved_loader_schemas::preserved_loader_schemas; +use crate::compile_driver::main::setup::project_graph::project_graph; +use crate::compile_driver::main::stages::monotonic::monotonic; use std::collections::HashSet; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; -use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; -use crate::bindings::_helpers::compile_driver::phases::monotonic; -use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; -use crate::bindings::_helpers::compile_driver::target_defaults::{ - TargetDefaults, apply_target_defaults, -}; -use crate::bindings::_helpers::compile_driver::targets::{ - managed_loader_target_isolation, managed_source_loaders, managed_write_schemas, - named_target_schema_strategy, preserved_loader_schemas, project_targets, - requires_explicit_targets, -}; -use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; +use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; +use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; +use crate::compile_driver::_helpers::setup::targets::managed_loader_target_isolation; +use crate::compile_driver::_helpers::setup::targets::managed_source_loaders; +use crate::compile_driver::_helpers::setup::targets::managed_write_schemas; +use crate::compile_driver::_helpers::setup::targets::named_target_schema_strategy; +use crate::compile_driver::_helpers::setup::targets::project_targets; +use crate::compile_driver::_helpers::setup::targets::requires_explicit_targets; +use crate::compile_driver::models::InputsRequest; +use crate::frontier::main::bridged::bridged; +use crate::frontier::main::produced::produced; +use crate::frontier::models::FrontierStage; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; use sqlbuild_python_front::sqltext::main::syntax_record::syntax_record; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compile_driver::assembly::assemble_project; -use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; +use crate::compile_driver::_helpers::stages::assembly::assemble_project; +use crate::compile_driver::_helpers::stages::selection_phases::resolve_analysis_model_names; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::compile_driver::constants::PROJECT_GRAPHS; use sqlbuild_python_base::compile_driver::constants::PROMOTION_CONFLICTS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; -use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; use sqlbuild_python_project::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Optional keywords of the project graph builders. -const GRAPH_OPTIONAL: [&str; 7] = [ +pub(crate) const GRAPH_OPTIONAL: [&str; 7] = [ "selected_target", "no_sql_validation", "skip_column_inference", @@ -56,7 +58,7 @@ const GRAPH_OPTIONAL: [&str; 7] = [ ]; /// The keyword arguments of one compiled-project build. -struct ProjectRequest { +pub(crate) struct ProjectRequest { discovered_inputs: Py, adapter: Py, selected_target: Py, @@ -81,42 +83,13 @@ pub(crate) fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult, value: bool) -> Bound<'_, PyAny> { +pub(crate) fn py_bool(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() } -/// Build one compiled project with adapter defaults and target validation applied. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_compiled_project<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_compiled_project", - options, - ( - &["discovered_inputs", "adapter"], - &[ - "selected_target", - "no_sql_validation", - "skip_column_inference", - "column_lineage_mode", - "cli_vars", - "external_sql_reference_resolver", - "resolved_connection", - "analysis_selection", - "source_lexical_syntax", - ], - ), - )?; - compiled_project(py, ProjectRequest::read(&arguments)?) -} - impl ProjectRequest { /// The request named by keyword `arguments`. - fn read(arguments: &KeywordArguments<'_>) -> PyResult { + pub(crate) fn read(arguments: &KeywordArguments<'_>) -> PyResult { Ok(Self { discovered_inputs: arguments.value("discovered_inputs")?.unbind(), adapter: arguments.value("adapter")?.unbind(), @@ -148,7 +121,10 @@ impl ProjectRequest { } /// The compiled project produced through the `COMPILED_PROJECT` frontier. -fn compiled_project(py: Python<'_>, request: ProjectRequest) -> PyResult> { +pub(crate) fn compiled_project( + py: Python<'_>, + request: ProjectRequest, +) -> PyResult> { produced( FrontierStage::CompiledProject, python_compiled_project(py, &request)?, @@ -289,37 +265,6 @@ fn targeted_project<'py>( ) } -/// The adapter behaviour compile inputs need, from the discovered project and its adapter. -pub(crate) fn compile_adapter_context<'py>( - discovered: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - syntax: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = discovered.py(); - let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( - &native_field(discovered, "project_config")?, - &py.None().into_bound(py), - ) - .and_then(|value| value.into_bound_py_any(py))?; - new_record_with( - py, - RecordId::CompileAdapterContext, - &[ - ("value_renderer", adapter.clone()), - ("collection_rendering", collection_rendering), - ( - "python_functions_inherit_default_namespace", - adapter.call_method0("python_functions_inherit_default_namespace")?, - ), - ("sql_lexical_syntax", syntax.clone()), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ], - ) -} - /// The compile inputs produced through the `COMPILE_PROJECT_INPUTS` frontier. fn frontier_compile_inputs<'py>( py: Python<'py>, @@ -375,32 +320,8 @@ pub(crate) fn build_project_graph_with_analysis_selection<'py>( selected_graph(&arguments, arguments.value("analysis_selection")?) } -/// Build the static dependency graph for a compiled project. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_project_graph<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - count_stage(py, PROJECT_GRAPHS, 1)?; - let mut optional: Vec<&str> = GRAPH_OPTIONAL.to_vec(); - optional.push("no_cache"); - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_project_graph", - options, - (&["discovered_inputs", "adapter"], &optional), - )?; - let selection: Bound<'py, PyAny> = new_record_with( - py, - RecordId::CompileAnalysisSelection, - &[("no_cache", py_bool(py, arguments.flag("no_cache")?))], - )?; - selected_graph(&arguments, selection) -} - /// `build_project_graph_with_analysis_selection` with its `analysis_selection`. -fn selected_graph<'py>( +pub(crate) fn selected_graph<'py>( arguments: &KeywordArguments<'py>, analysis_selection: Bound<'py, PyAny>, ) -> PyResult> { @@ -419,12 +340,3 @@ fn selected_graph<'py>( } project_graph(&project) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_compiled_project, module)?)?; - module.add_function(wrap_pyfunction!( - build_project_graph_with_analysis_selection, - module - )?)?; - module.add_function(wrap_pyfunction!(build_project_graph, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs index cdd6b92def..10ee6aaf6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs @@ -5,19 +5,15 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::replaced::replaced; -use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::ordered_python_names::ordered_python_names; use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -28,13 +24,13 @@ use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::pytho const SKIPPED_UPSTREAM_CODE: &str = "P501"; /// The Python node kinds and plan phases entries are built from. -struct NodeKinds<'py> { +pub(crate) struct NodeKinds<'py> { task: Bound<'py, PyAny>, asset: Bound<'py, PyAny>, } impl<'py> NodeKinds<'py> { - fn load(py: Python<'py>) -> PyResult { + pub(crate) fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host_entry(py, UserCodeHost::PythonNodeKind)?; Ok(Self { task: native_field(&kinds, "TASK")?, @@ -67,124 +63,14 @@ fn upstream_names(graph: &Bound<'_, PyAny>) -> PyResult(discovered: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn node_graph<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { python_node_graph(discovered) } -/// Return sources declared by the selected Python nodes and their Python upstreams. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn python_read_source_names_for<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "python_read_source_names_for", - options, - (&["discovered_inputs", "python_node_names"], &[]), - )?; - let frozenset: Bound<'py, PyAny> = imported(py, "builtins", "frozenset")?; - let selected: Bound<'py, PyAny> = arguments.value("python_node_names")?; - if !selected.is_truthy()? { - return frozenset.call0(); - } - let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; - let source: Bound<'py, PyAny> = - host_entry(py, UserCodeHost::SqlResourceRefKind)?.getattr("SOURCE")?; - let mut names: Vec> = Vec::new(); - for reference in call_with( - &native_field(&graph, "selected_sql_refs")?, - &[("selected_names", selected)], - )? - .try_iter()? - { - let reference: Bound<'py, PyAny> = reference?; - if native_field(&reference, "kind")?.eq(&source)? { - names.push(native_field(&reference, "name")?); - } - } - frozenset.call1((PyTuple::new(py, names)?,)) -} - -/// Attach python-aware selection warnings and plan entries to the plan output. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_python_run_plan_outputs<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_python_run_plan_outputs", - options, - ( - &[ - "discovered_inputs", - "plan_output", - "run_selection", - "selected_python_node_names", - ], - &[], - ), - )?; - let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; - let run_selection: Bound<'py, PyAny> = arguments.value("run_selection")?; - let selected: Bound<'py, PyAny> = arguments.value("selected_python_node_names")?; - if run_selection.is_none() { - return new_record_with( - py, - RecordId::PythonRunPlanOutputs, - &[ - ("plan_output", plan_output), - ("python_plan_entries", PyTuple::empty(py).into_any()), - ("selected_python_node_names", selected), - ], - ); - } - let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; - let kinds: NodeKinds<'py> = NodeKinds::load(py)?; - let mut warnings: Vec> = native_field(&plan_output, "warnings")? - .try_iter()? - .collect::>()?; - warnings.extend(skipped_ingress_warnings( - &plan_output, - &run_selection, - (&graph, &kinds), - )?); - let plan_output: Bound<'py, PyAny> = replaced( - &plan_output, - &[("warnings", PyTuple::new(py, warnings)?.into_any())], - )?; - let selection: Bound<'py, PyAny> = new_record_with( - py, - RecordId::PythonSqlRunSelection, - &[ - ("sql_keys", native_field(&plan_output, "selected_keys")?), - ("python_node_names", selected.clone()), - ], - )?; - let lifecycle: Bound<'py, PyAny> = build_python_sql_run_lifecycle(&selection, &graph)?; - let entries: Bound<'py, PyTuple> = plan_entries( - &lifecycle, - (&graph, &kinds), - &native_field(&plan_output, "python_identity_fingerprints")?, - )?; - new_record_with( - py, - RecordId::PythonRunPlanOutputs, - &[ - ("plan_output", plan_output), - ("python_plan_entries", entries.into_any()), - ("selected_python_node_names", selected), - ], - ) -} - /// Return task/asset plan entries ordered by lifecycle dependency readiness. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_plan_entries<'py>( +pub(crate) fn build_python_plan_entries<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -211,7 +97,7 @@ fn build_python_plan_entries<'py>( } /// The entries of the lifecycle's ingress and then read-side tasks and assets. -fn plan_entries<'py>( +pub(crate) fn plan_entries<'py>( lifecycle: &Bound<'py, PyAny>, (graph, kinds): (&Bound<'py, PyAny>, &NodeKinds<'py>), previous_identities: &Bound<'py, PyAny>, @@ -316,7 +202,7 @@ fn identity_status<'py>( } /// Warnings for skipped task/asset deps of planned source loaders. -fn skipped_ingress_warnings<'py>( +pub(crate) fn skipped_ingress_warnings<'py>( plan_output: &Bound<'py, PyAny>, run_selection: &Bound<'py, PyAny>, (graph, kinds): (&Bound<'py, PyAny>, &NodeKinds<'py>), @@ -368,9 +254,3 @@ fn skipped_ingress_warnings<'py>( } Ok(warnings) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(python_read_source_names_for, module)?)?; - module.add_function(wrap_pyfunction!(build_python_run_plan_outputs, module)?)?; - module.add_function(wrap_pyfunction!(build_python_plan_entries, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs index b137970c1b..4ab615e5b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs @@ -1,15 +1,11 @@ //! Runtime SQL relation maps for Python node contexts and hard-coded name guards. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyString}; use sqlbuild_host::entries::models::{AdapterHost, UserCodeHost}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -18,14 +14,14 @@ use sqlbuild_python_project::references::main::render_source_relation::render_so const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; /// The `SqlResourceRefKind` members a relation map distinguishes. -struct RefKinds<'py> { +pub(crate) struct RefKinds<'py> { model: Bound<'py, PyAny>, seed: Bound<'py, PyAny>, source: Bound<'py, PyAny>, } impl<'py> RefKinds<'py> { - fn load(py: Python<'py>) -> PyResult { + pub(crate) fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host_entry(py, UserCodeHost::SqlResourceRefKind)?; Ok(Self { model: native_field(&kinds, "MODEL")?, @@ -45,72 +41,8 @@ fn shared_error(py: Python<'_>, message: String) -> PyErr { } } -/// Return adapter-qualified runtime relations required by selected Python nodes. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_python_relation_targets<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_python_relation_targets", - options, - (&["adapter", "project", "plan_output"], &["required_refs"]), - )?; - let kinds: RefKinds<'py> = RefKinds::load(py)?; - let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let project: Bound<'py, PyAny> = arguments.value("project")?; - let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; - let required: Bound<'py, PyAny> = arguments.value("required_refs")?; - let refs: Bound<'py, PyAny> = if required.is_none() { - planned_relation_refs(&plan_output, &kinds)? - } else { - required - }; - let targets: Bound<'py, PyDict> = PyDict::new(py); - for reference in refs.try_iter()? { - let reference: Bound<'py, PyAny> = reference?; - let relation: Bound<'py, PyAny> = - resolved_relation((&adapter, &project, &plan_output), &reference, &kinds)?; - targets.set_item(reference, relation)?; - } - Ok(targets) -} - -/// Return every planned project relation for hard-coded name warnings, or None when off. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_project_relation_targets<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult>> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_project_relation_targets", - options, - (&["adapter", "plan_output"], &[]), - )?; - let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; - if !native_field(&plan_output, "enforce_explicit_references")?.is_truthy()? { - return Ok(None); - } - let kinds: RefKinds<'py> = RefKinds::load(py)?; - let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let targets: Bound<'py, PyDict> = PyDict::new(py); - for reference in planned_relation_refs(&plan_output, &kinds)?.try_iter()? { - let reference: Bound<'py, PyAny> = reference?; - let relation: Bound<'py, PyAny> = - planned_relation(&adapter, &plan_output, &reference, &kinds)?; - if !relation.is_none() { - targets.set_item(reference, relation)?; - } - } - Ok(Some(targets)) -} - /// The model, seed and Python source refs of a plan, as the frozenset Python builds. -fn planned_relation_refs<'py>( +pub(crate) fn planned_relation_refs<'py>( plan_output: &Bound<'py, PyAny>, kinds: &RefKinds<'py>, ) -> PyResult> { @@ -133,7 +65,7 @@ fn planned_relation_refs<'py>( } /// `_resolve_relation`: the planned relation, or the compiled location's qualified name. -fn resolved_relation<'py>( +pub(crate) fn resolved_relation<'py>( (adapter, project, plan_output): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), reference: &Bound<'py, PyAny>, kinds: &RefKinds<'py>, @@ -195,7 +127,7 @@ fn qualified_name<'py>( } /// `_planned_relation`: the relation the plan resolved for `reference`, or `None`. -fn planned_relation<'py>( +pub(crate) fn planned_relation<'py>( adapter: &Bound<'py, PyAny>, plan_output: &Bound<'py, PyAny>, reference: &Bound<'py, PyAny>, @@ -224,8 +156,3 @@ fn planned_relation<'py>( } qualified_name(adapter, &location) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_python_relation_targets, module)?)?; - module.add_function(wrap_pyfunction!(build_project_relation_targets, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/target_defaults.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/target_defaults.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs index f797a8508e..371073c49b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs @@ -1,17 +1,17 @@ //! Adapter-aware target namespace validation of a compile. +use crate::compile_driver::main::setup::expanded_loader_namespace::expanded_loader_namespace; +use crate::compile_driver::main::setup::managed_write_schema::managed_write_schema; +use crate::compile_driver::models::TargetSettingsValues; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString, PyStringMethods}; use sqlbuild_analysis::assembly::target_checks::constants::{ - LOADER_COLLISION_CODE, MANAGED_WRITE_SCHEMA_CODE, MISSING_TARGET_PARTS_CODE, - NAMED_TARGET_SCHEMA_CODE, PRESERVE_TARGET_VALUE, + LOADER_COLLISION_CODE, MISSING_TARGET_PARTS_CODE, NAMED_TARGET_SCHEMA_CODE, }; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; @@ -21,11 +21,8 @@ use sqlbuild_analysis::assembly::target_checks::main::target_error_message::targ use sqlbuild_analysis::assembly::target_checks::models::{LoaderCollision, MissingParts}; use sqlbuild_core::text::main::python_strip::python_strip; -use sqlbuild_model_config::templates::models::TemplateOptions; - use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ @@ -39,17 +36,15 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ ]; /// Adapters whose execution needs every model and seed database and schema resolved. const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; -const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; -const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; +pub(crate) const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; +pub(crate) const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; -use crate::bindings::_helpers::compile_attachment::target::resolved_target; +use crate::compile_attachment::main::resolved_target::resolved_target; use pyo3::IntoPyObjectExt; -use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -58,7 +53,7 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_connection_ use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::discovery::main::failures::setting_help::setting_help; -const DEFAULTS_SECTION: &str = "defaults"; +pub(crate) const DEFAULTS_SECTION: &str = "defaults"; /// One optional `(database, schema)` namespace. type Namespace = (Option, Option); @@ -66,7 +61,12 @@ type Namespace = (Option, Option); type TargetNamespaces = (Vec<(String, String)>, Vec); /// A `PlannerInputError` with `message`, `code` and optional `help`. -fn planner_error(py: Python<'_>, message: String, code: &str, help: Option) -> PyErr { +pub(crate) fn planner_error( + py: Python<'_>, + message: String, + code: &str, + help: Option, +) -> PyErr { let raised: PyResult> = (|| { error_class(py, ErrorId::PlannerInput)?.call( (message,), @@ -100,7 +100,7 @@ fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { } /// `value` when it is a `str`, else `None`. -fn text_or_none(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn text_or_none(value: &Bound<'_, PyAny>) -> PyResult> { match value.downcast::() { Ok(text) => Ok(Some(text.to_cow()?.into_owned())), Err(_) => Ok(None), @@ -119,7 +119,7 @@ fn ignored_connection_value(project: &Bound<'_, PyAny>, part: &str) -> PyResult< } /// The sentence saying the connection's `part` is unused. -fn ignored_connection_note(part: &str, value: &str) -> String { +pub(crate) fn ignored_connection_note(part: &str, value: &str) -> String { format!( " The connection's {part} '{value}' is not used because the project defines named \ connections or targets." @@ -127,7 +127,7 @@ fn ignored_connection_note(part: &str, value: &str) -> String { } /// `setting_help` for adding `key = "value"` under `[section]` of the project file. -fn project_setting_help( +pub(crate) fn project_setting_help( py: Python<'_>, (purpose, section): (&str, &str), (key, value): (&str, &str), @@ -155,7 +155,7 @@ fn project_setting_help( /// Require shared-warehouse named targets to choose a schema strategy. #[pyfunction] #[pyo3(signature = (*, discovered_inputs, adapter, selected_target, cli_vars = None))] -fn validate_named_target_schema_strategy<'py>( +pub(crate) fn validate_named_target_schema_strategy<'py>( discovered_inputs: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, selected_target: &Bound<'py, PyAny>, @@ -215,7 +215,7 @@ pub(crate) fn named_target_schema_strategy( /// Reject managed loader write namespaces shared by multiple targets. #[pyfunction] #[pyo3(signature = (**options))] -fn validate_managed_loader_target_isolation<'py>( +pub(crate) fn validate_managed_loader_target_isolation<'py>( py: Python<'py>, options: Option>, ) -> Result<(), PyErr> { @@ -416,53 +416,6 @@ fn managed_source_namespace<'py>( Ok((part("database")?, part("schema")?)) } -/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. -#[derive(Clone)] -pub(crate) struct TargetSettingsValues { - name: Option, - database: Option, - schema: Option, - loader_schema: Option, -} - -impl TargetSettingsValues { - /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. - pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { - let located = |value: &Option| -> PyResult> { - value - .as_deref() - .filter(|value| *value != PRESERVE_TARGET_VALUE) - .into_bound_py_any(py) - }; - let preserved = |value: &Option| -> Bound<'py, PyAny> { - PyBool::new(py, value.as_deref() == Some(PRESERVE_TARGET_VALUE)) - .to_owned() - .into_any() - }; - new_record_with( - py, - RecordId::TargetSettings, - &[ - ("name", self.name.clone().into_bound_py_any(py)?), - ("database", located(&self.database)?), - ("schema", located(&self.schema)?), - ("loader_schema", located(&self.loader_schema)?), - ("database_preserved", preserved(&self.database)), - ("schema_preserved", preserved(&self.schema)), - ], - ) - } - - /// The `ctx` a provider's `setup` receives in a command with these settings. - pub(crate) fn setup_context<'py>(&self, py: Python<'py>) -> PyResult> { - new_record_with( - py, - RecordId::ProviderSetupContext, - &[("target_config", self.record(py)?)], - ) - } -} - /// The target `target_name` with its resolved locations; `loader_schema` is `None` when it resolves only inside a model, such as a `${CTX:...}` schema. pub(crate) fn target_settings<'py>( (discovered, target_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -502,73 +455,10 @@ pub(crate) fn target_settings<'py>( }) } -/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. -pub(crate) fn selected_target_settings<'py>( - (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter_name: &str, - (connection, effective_vars, adapter): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), -) -> PyResult { - let namespace: Bound<'py, PyTuple> = effective_target_namespace( - (discovered, selected_target), - (adapter_name, connection, adapter), - )?; - target_settings( - (discovered, &namespace.get_item(0)?), - ( - namespace.get_item(2)?.extract()?, - namespace.get_item(3)?.extract()?, - ), - (connection, effective_vars, adapter), - ) -} - -/// `target_settings` of a compiled project's target. -pub(crate) fn project_target_settings<'py>( - discovered: &Bound<'py, PyAny>, - project: &Bound<'py, PyAny>, - (connection, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult { - target_settings( - (discovered, &native_field(project, "effective_target_name")?), - ( - native_field(project, "effective_target_database")?.extract()?, - native_field(project, "effective_target_schema")?.extract()?, - ), - ( - connection, - &native_field(project, "effective_vars")?, - adapter, - ), - ) -} - -/// A managed source's `database` and `schema` with project vars expanded and no context templates. -pub(crate) fn expanded_loader_namespace<'py>( - value: &Bound<'py, PyAny>, - effective_vars: &Bound<'py, PyAny>, -) -> PyResult> { - expanded_template_data( - value, - (effective_vars.downcast()?, &PyDict::new(value.py())), - ( - TemplateOptions { - allow_context: false, - preserve_context_tokens: false, - preserve_unknown_context: false, - }, - LOADER_NAMESPACE_LABEL, - ), - ) -} - /// Validate compiled model and seed locations for the effective adapter. #[pyfunction] #[pyo3(signature = (**options))] -fn validate_project_targets<'py>( +pub(crate) fn validate_project_targets<'py>( py: Python<'py>, options: Option>, ) -> Result<(), PyErr> { @@ -676,7 +566,7 @@ fn required_target_parts( /// Require an explicitly resolved physical schema for every managed warehouse write. #[pyfunction] #[pyo3(signature = (**options))] -fn validate_managed_write_schemas<'py>( +pub(crate) fn validate_managed_write_schemas<'py>( py: Python<'py>, options: Option>, ) -> Result<(), PyErr> { @@ -772,80 +662,3 @@ pub(crate) fn managed_source_loaders(project: &Bound<'_, PyAny>) -> PyResult, - selected_target: &Bound<'_, PyAny>, - loaders: &HashSet, -) -> PyResult<()> { - let py: Python<'_> = discovered.py(); - let (target_name, target_config) = resolved_target(discovered, selected_target)?; - if target_name.is_none() { - return Ok(()); - } - if !text_or_none(&native_field(&target_config, "schema")?)? - .is_some_and(|schema| python_strip(&schema) == PRESERVE_TARGET_VALUE) - { - return Ok(()); - } - for loader in native_field(discovered, "loader_functions")?.try_iter()? { - let loader: Bound<'_, PyAny> = loader?; - let name: String = native_field(&loader, "name")?.extract()?; - let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; - if !loaders.contains(&name) || destination.as_deref().is_some_and(|d| d.contains('.')) { - continue; - } - let table: String = destination.unwrap_or_else(|| format!("__loader__{name}")); - return Err(planner_error( - py, - target_error_message(&TargetError::PreservedLoaderSchema(name)), - MANAGED_WRITE_SCHEMA_CODE, - Some(format!( - "to choose where it writes, set its destination in its @loader decorator:\n \ - @loader(destination=\"my_schema.{table}\")" - )), - )); - } - Ok(()) -} - -/// Raise when neither the resource nor the project names a physical write schema. -pub(crate) fn managed_write_schema( - resource_label: &str, - schema: &Bound<'_, PyAny>, - (project_schema, ignored): (Option<&str>, Option<&str>), -) -> PyResult<()> { - if explicit_write_schema(text_or_none(schema)?.as_deref()).is_some() || project_schema.is_some() - { - return Ok(()); - } - let py: Python<'_> = schema.py(); - let mut message: String = - target_error_message(&TargetError::ManagedWriteSchema(resource_label.to_owned())); - let help: Option = match ignored { - Some(value) => { - message.push_str(&ignored_connection_note("schema", value)); - Some(project_setting_help( - py, - (DEFAULTS_HELP_PURPOSE, DEFAULTS_SECTION), - ("schema", value), - )?) - } - None => None, - }; - Err(planner_error(py, message, MANAGED_WRITE_SCHEMA_CODE, help)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - validate_named_target_schema_strategy, - module - )?)?; - module.add_function(wrap_pyfunction!( - validate_managed_loader_target_isolation, - module - )?)?; - module.add_function(wrap_pyfunction!(validate_project_targets, module)?)?; - module.add_function(wrap_pyfunction!(validate_managed_write_schemas, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs index b7ce0c6d93..8ab1d28a76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs @@ -1,22 +1,23 @@ //! The compile analysis sequence: discovery, graph, lineage, contracts, selection and rules. +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use std::time::Instant; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; -use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; -use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; -use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, EarlyRulesSettings}; -use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; -use crate::bindings::_helpers::rules::results::rules_progress; -use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; +use crate::compile_driver::_helpers::setup::project::build_project_graph_with_analysis_selection; +use crate::lineage::main::build_project_column_lineage::build_project_column_lineage; +use crate::rules::constants::RULE_FIX_AVAILABLE_NOTE; +use crate::rules::main::checking::rules_progress::rules_progress; +use crate::rules::main::configuration::load_rules_config_with_cache::load_rules_config_with_cache; +use crate::rules::main::rule::evaluate_rules::evaluate_rules; +use crate::rules::models::EarlyRulesScope; +use crate::rules::models::EarlyRulesSettings; +use crate::rules::models::EarlySqlLint; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -140,16 +141,6 @@ fn any_error(diagnostics: &Bound<'_, PyAny>) -> PyResult { Ok(false) } -/// Discover, compile and validate the project into one `CompileAnalysis`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn analyze_compile_project<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - analyze(py, &options.unwrap_or_else(|| PyDict::new(py))) -} - /// `analyze_compile_project` with its keyword options. pub(crate) fn analyze<'py>( py: Python<'py>, @@ -603,7 +594,3 @@ fn with_rule_diagnostics<'py>( } PyTuple::new(py, diagnostics) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(analyze_compile_project, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis_cache.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis_cache.rs index 5d054f0092..e4d7d09671 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis_cache.rs @@ -1,13 +1,10 @@ //! The analysis cache context and the model analysis cache metrics. use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PySlice, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; -use sqlbuild_host::compile_context::main::record_compile_metric::record_compile_metric; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -99,7 +96,7 @@ pub(crate) fn analysis_cache_context<'py>( /// Build a reusable cache context, or bypass when profile identity is unstable. #[pyfunction] #[pyo3(signature = (**options))] -fn build_analysis_cache_context<'py>( +pub(crate) fn build_analysis_cache_context<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { @@ -122,33 +119,6 @@ fn build_analysis_cache_context<'py>( ) } -/// Semantic analysis cache outcome counts for one invocation. -pub(crate) struct AnalysisCacheCounts { - pub(crate) batch_hits: usize, - pub(crate) entry_hits: usize, - pub(crate) misses: usize, - pub(crate) bypasses: usize, -} - -/// Record semantic analysis cache outcomes for one invocation. -pub(crate) fn record_analysis_cache_metrics( - AnalysisCacheCounts { - batch_hits, - entry_hits, - misses, - bypasses, - }: AnalysisCacheCounts, -) { - for (metric, value) in [ - ("analysis_batch_cache_hits", batch_hits), - ("analysis_entry_cache_hits", entry_hits), - ("analysis_cache_misses", misses), - ("analysis_cache_bypasses", bypasses), - ] { - record_compile_metric(metric, i64::try_from(value).unwrap_or(i64::MAX)); - } -} - /// Whether `error` is the `TypeError` or `ValueError` an unstable payload raises. fn json_failure(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) || error.is_instance_of::(py) @@ -242,7 +212,3 @@ fn package_version(py: Python<'_>, package: &str) -> PyResult { Err(error) => Err(error), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_analysis_cache_context, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs index 78d31418c8..3b87e0baa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs @@ -3,14 +3,14 @@ use std::sync::{Arc, Mutex, PoisonError}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyCFunction, PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; -use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; -use crate::bindings::_helpers::compile_driver::project::lineage_mode; +use crate::compile_driver::_helpers::assembly::assembly_project::assembled_project; +use crate::compile_driver::_helpers::setup::project::lineage_mode; use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; use sqlbuild_python_project::references::main::explicit_reference_report::explicit_reference_report; @@ -250,8 +250,3 @@ fn with_native_project<'py>( Some(keywords(py, &[("native_project", native_project.clone())])?), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_project, module)?)?; - module.add_function(wrap_pyfunction!(assembly_stage_inputs, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs index d76f3ae913..c4021e6625 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs @@ -1,11 +1,15 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::compile_driver::main::stages::elapsed_ms::elapsed_ms; +use crate::compile_driver::main::stages::store_and_print::store_and_print; +use crate::compile_driver::models::CommandOutcome; +use crate::compile_driver::types::PhasesOutcome; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyString, PyTuple, }; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction}; use sqlbuild_host::compile_context::main::collect_compile_timings::collect_compile_timings; use sqlbuild_host::compile_context::main::timing_milliseconds::timing_milliseconds; use sqlbuild_host::compile_context::types::CompileTimingsHandle; @@ -15,24 +19,23 @@ use sqlbuild_rules::custom_host_pool::constants::{ use std::path::PathBuf; use std::rc::Rc; -use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; -use crate::bindings::_helpers::compile_command::report::{ - format_compile_json_with_notice, format_compile_text, -}; -use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_compile; -use crate::bindings::_helpers::compile_command::reuse_recording::{ - CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, -}; -use crate::bindings::_helpers::compile_command::target_writer::{ - remove_failed_compile, write_rule_failed_models, -}; -use crate::bindings::_helpers::compile_driver::analysis::analyze; -use crate::bindings::_helpers::compile_driver::phases::{ - elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, - target_dir, write_artifacts, write_dag_artifact, -}; -use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; -use crate::bindings::_helpers::rules::rules_config::rules_config_of; +use crate::compile_command::main::artifacts::remove_failed_compile::remove_failed_compile; +use crate::compile_command::main::artifacts::write_rule_failed_models::write_rule_failed_models; +use crate::compile_command::main::invocation::compile_reuse_timings::compile_reuse_timings; +use crate::compile_command::main::invocation::format_compile_json_with_notice::format_compile_json_with_notice; +use crate::compile_command::main::invocation::format_compile_text::format_compile_text; +use crate::compile_command::main::invocation::new_artifact_recorder::new_artifact_recorder; +use crate::compile_command::models::CompileArtifactWrites; +use crate::compile_command::models::CompileReuseAttempt; +use crate::compile_driver::_helpers::stages::analysis::analyze; +use crate::compile_driver::_helpers::stages::phases::manifest_payload; +use crate::compile_driver::_helpers::stages::phases::resolved_dag_path; +use crate::compile_driver::_helpers::stages::phases::sql_test_planning_diagnostics; +use crate::compile_driver::_helpers::stages::phases::target_dir; +use crate::compile_driver::_helpers::stages::phases::write_artifacts; +use crate::compile_driver::_helpers::stages::phases::write_dag_artifact; +use crate::rules::main::checking::format_skipped_type_proof_rules::format_skipped_type_proof_rules; +use crate::rules::main::configuration::rules_config_of::rules_config_of; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -58,7 +61,7 @@ static RETAINED_UNTIL_EXIT: PyOnceLock> = PyOnceLock::new(); /// Let compiles in this process leave their project state for process exit to reclaim. #[pyfunction] -fn defer_compile_teardown_to_exit(py: Python<'_>) { +pub(crate) fn defer_compile_teardown_to_exit(py: Python<'_>) { let _ = RETAINED_UNTIL_EXIT.get_or_init(py, || PyList::empty(py).unbind()); } @@ -70,17 +73,6 @@ fn retain_until_exit(value: &Bound<'_, PyAny>) -> PyResult<()> { } } -/// What one full compile reports for reuse recording. -pub(crate) struct CommandOutcome<'py> { - pub(crate) exit_code: i64, - pub(crate) compile_cache_enabled: bool, - pub(crate) artifacts_written: bool, - pub(crate) dag_artifact_path: Bound<'py, PyAny>, -} - -/// One compile's outcome, input reads and artifact writes, before it is stored and printed. -pub(crate) type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); - /// The per-invocation recorders a compile runs inside. struct Recorders<'py> { detailed_timings: CompileTimingsHandle, @@ -89,69 +81,8 @@ struct Recorders<'py> { prepared_artifacts: Bound<'py, PyAny>, } -/// Run one full compile, recording it for reuse; returns the exit code. -#[pyfunction] -#[pyo3(signature = (request, reuse_attempt=None))] -pub(crate) fn run_compile_command<'py>( - py: Python<'py>, - request: Bound<'py, PyAny>, - reuse_attempt: Option>, -) -> PyResult { - let total_start: f64 = match &reuse_attempt { - Some(attempt) => native_field(attempt, "started")?.extract()?, - None => monotonic(py)?, - }; - let effective_request: Bound<'py, PyAny> = resolved_request(&request)?; - let attempt: Bound<'py, PyAny> = match reuse_attempt { - Some(attempt) => attempt, - None => bypass_attempt(py, &effective_request, total_start)?, - }; - let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); - let status: Status<'py> = Status(compile_status(&request)?); - collecting_later(py, || { - recorded_compile( - (&effective_request, &attempt), - (&output, &status), - total_start, - ) - })? -} - -/// The resolved request, output recorder and phases of a compile neither stored nor printed. -pub(crate) fn unrecorded_compile<'py>( - request: &Bound<'py, PyAny>, - attempt: &Bound<'py, PyAny>, -) -> PyResult<( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - PyResult>, -)> { - let py: Python<'py> = request.py(); - let total_start: f64 = native_field(attempt, "started")?.extract()?; - let effective_request: Bound<'py, PyAny> = resolved_request(request)?; - let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); - let status: Status<'py> = Status(py.None().into_bound(py)); - let phases: PyResult> = collecting_later(py, || { - compile_phases( - (&effective_request, attempt), - (&output, &status), - total_start, - ) - })?; - Ok((effective_request, output, phases)) -} - -/// `request` with its project directory resolved to the working directory when unset. -pub(crate) fn resolved_request<'py>(request: &Bound<'py, PyAny>) -> PyResult> { - if !native_field(request, "project_dir")?.is_none() { - return Ok(request.clone()); - } - let cwd: Bound<'py, PyAny> = imported(request.py(), "pathlib", "Path")?.call_method0("cwd")?; - replaced(&request.clone(), &[("project_dir", cwd)]) -} - /// Run `body` with Python's garbage collector paused, then resume or freeze it as compiles do. -fn collecting_later(py: Python<'_>, body: impl FnOnce() -> T) -> PyResult { +pub(crate) fn collecting_later(py: Python<'_>, body: impl FnOnce() -> T) -> PyResult { let gc: Bound<'_, PyModule> = py.import("gc")?; let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; gc.call_method0("disable")?; @@ -164,7 +95,7 @@ fn collecting_later(py: Python<'_>, body: impl FnOnce() -> T) -> PyResult } /// `CompileReuseAttempt` of a compile that did not consult the reuse record. -fn bypass_attempt<'py>( +pub(crate) fn bypass_attempt<'py>( py: Python<'py>, request: &Bound<'py, PyAny>, started: f64, @@ -187,7 +118,7 @@ fn bypass_attempt<'py>( } /// The interactive status line, shown only on a terminal without `--json`. -fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); let none: Bound<'py, PyAny> = py.None().into_bound(py); if native_field(request, "json_output")?.is_truthy()? { @@ -210,7 +141,7 @@ fn close_status(status: &Status<'_>) -> PyResult<()> { } /// Compile inside the recorders, then store the reusable compile and print its output. -fn recorded_compile<'py>( +pub(crate) fn recorded_compile<'py>( (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (output, status): (&Bound<'py, PyAny>, &Status<'py>), total_start: f64, @@ -219,7 +150,7 @@ fn recorded_compile<'py>( compile_phases((request, attempt), (output, status), total_start); close_status(status)?; let phases: PhasesOutcome<'py> = phases?; - crate::bindings::_helpers::compile_delta::verify::full_compile_finished( + crate::compile_delta::main::verify::full_compile_finished::full_compile_finished( output, (phases.0.exit_code, phases.0.artifacts_written), &phases.2, @@ -227,47 +158,6 @@ fn recorded_compile<'py>( store_and_print((request, attempt), output, phases) } -/// Store a finished compile for reuse, then print its recorded output. -pub(crate) fn store_and_print<'py>( - (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - output: &Bound<'py, PyAny>, - phases: PhasesOutcome<'py>, -) -> PyResult { - let py: Python<'py> = request.py(); - let (outcome, input_reads, artifact_writes): PhasesOutcome<'py> = phases; - let stored: PyResult<()> = write_reusable_compile( - py, - Some(keywords( - py, - &[ - ("attempt", attempt.clone()), - ("output", output.clone()), - ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), - ("input_reads", input_reads), - ("artifact_writes", artifact_writes), - ( - "compile_cache_enabled", - PyBool::new(py, outcome.compile_cache_enabled) - .to_owned() - .into_any(), - ), - ( - "artifacts_written", - PyBool::new(py, outcome.artifacts_written) - .to_owned() - .into_any(), - ), - ("dag_artifact_path", outcome.dag_artifact_path), - ("json_output", native_field(request, "json_output")?), - ], - )?), - ); - let printed: PyResult> = output.call_method0("print_stdout"); - printed?; - stored?; - Ok(outcome.exit_code) -} - /// Run the compile's phases inside its recorders, without storing or printing it. pub(crate) fn compile_phases<'py>( (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -413,7 +303,7 @@ fn compiled_with_status<'py>( .chain(withheld.try_iter()?) .collect::>>()?, )?; - crate::bindings::_helpers::compile_delta::report_hooks::record_compile_facts( + crate::compile_delta::main::report_hooks::record_compile_facts::record_compile_facts( &native_field(&analysis, "graph")?, &diagnostics, )?; @@ -618,8 +508,3 @@ fn record_output<'py>( output.call_method1("record_stdout", (rendered,))?; Ok(()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(defer_compile_teardown_to_exit, module)?)?; - module.add_function(wrap_pyfunction!(run_compile_command, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs index ec03b3a6f9..572a536448 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs @@ -1,18 +1,13 @@ //! Deferred target relation locations, and the warehouse relations they name. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; use sqlbuild_model_config::templates::models::DeferredSegment; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; @@ -25,51 +20,16 @@ const PRESERVE_TARGET_VALUE: &str = "preserve"; const DEFERRED_TARGET_CONTEXT_KEYS: [&str; 2] = ["database", "schema"]; /// The deferred target's naming and the adapter defaults its locations fall back to. -struct DeferredNaming<'py> { - target_config: Bound<'py, PyAny>, - effective_vars: Bound<'py, PyAny>, - default_schema: Bound<'py, PyAny>, - default_database: Bound<'py, PyAny>, - render_qualified_name: Bound<'py, PyAny>, -} - -/// Build physical locations for all models and seeds under a deferred target. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_deferred_locations<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_deferred_locations", - options, - ( - &[ - "project", - "deferred_target_config", - "effective_vars", - "default_schema", - "default_database", - "render_qualified_name", - ], - &[], - ), - )?; - deferred_locations( - &arguments.value("project")?, - &DeferredNaming { - target_config: arguments.value("deferred_target_config")?, - effective_vars: arguments.value("effective_vars")?, - default_schema: arguments.value("default_schema")?, - default_database: arguments.value("default_database")?, - render_qualified_name: arguments.value("render_qualified_name")?, - }, - ) +pub(crate) struct DeferredNaming<'py> { + pub(crate) target_config: Bound<'py, PyAny>, + pub(crate) effective_vars: Bound<'py, PyAny>, + pub(crate) default_schema: Bound<'py, PyAny>, + pub(crate) default_database: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, } /// `build_deferred_locations` with its arguments. -fn deferred_locations<'py>( +pub(crate) fn deferred_locations<'py>( project: &Bound<'py, PyAny>, naming: &DeferredNaming<'py>, ) -> PyResult> { @@ -179,31 +139,8 @@ fn planner_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::PlannerInput, &message) } -/// Validate and resolve the deferred target config. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn resolve_deferred_target_config<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_deferred_target_config", - options, - ( - &["discovered_inputs", "defer_to", "current_target_name"], - &[], - ), - )?; - deferred_target_config( - &arguments.value("discovered_inputs")?, - &arguments.value("defer_to")?, - &arguments.value("current_target_name")?, - ) -} - /// `resolve_deferred_target_config` with its arguments. -fn deferred_target_config<'py>( +pub(crate) fn deferred_target_config<'py>( discovered: &Bound<'py, PyAny>, defer_to: &Bound<'py, PyAny>, current_target_name: &Bound<'py, PyAny>, @@ -227,28 +164,8 @@ fn deferred_target_config<'py>( targets.get_item(defer_to) } -/// Gather existing relations from the deferred target's relation-location schemas. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn gather_deferred_relations<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "gather_deferred_relations", - options, - (&["adapter", "connection", "deferred_locations"], &[]), - )?; - deferred_relations( - &arguments.value("adapter")?, - &arguments.value("connection")?, - &arguments.value("deferred_locations")?, - ) -} - /// `gather_deferred_relations` with its arguments. -fn deferred_relations<'py>( +pub(crate) fn deferred_relations<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, locations: &Bound<'py, PyAny>, @@ -289,68 +206,3 @@ fn deferred_relations<'py>( } Ok(relations_by_name) } - -/// Resolve native target locations and warehouse relations for model deferral. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn resolve_model_deferral_inputs<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_model_deferral_inputs", - options, - ( - &[ - "project", - "discovered_inputs", - "adapter", - "connection", - "defer_to", - ], - &[], - ), - )?; - let defer_to: Bound<'py, PyAny> = arguments.value("defer_to")?; - if defer_to.is_none() { - return new_record(py, RecordId::DeferralInputs, (), None); - } - let project: Bound<'py, PyAny> = arguments.value("project")?; - let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let target_config: Bound<'py, PyAny> = deferred_target_config( - &arguments.value("discovered_inputs")?, - &defer_to, - &native_field(&project, "effective_target_name")?, - )?; - let locations: Bound<'py, PyDict> = deferred_locations( - &project, - &DeferredNaming { - target_config, - effective_vars: native_field(&project, "effective_vars")?, - default_schema: adapter.call_method0("default_schema")?, - default_database: adapter.call_method0("default_database")?, - render_qualified_name: native_field(&adapter, "render_qualified_name")?, - }, - )?; - let relations: Bound<'py, PyDict> = deferred_relations( - &adapter, - &arguments.value("connection")?, - locations.as_any(), - )?; - new_record_with( - py, - RecordId::DeferralInputs, - &[ - ("deferred_locations", locations.into_any()), - ("deferred_relations", relations.into_any()), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_deferred_locations, module)?)?; - module.add_function(wrap_pyfunction!(resolve_deferred_target_config, module)?)?; - module.add_function(wrap_pyfunction!(gather_deferred_relations, module)?)?; - module.add_function(wrap_pyfunction!(resolve_model_deferral_inputs, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs index 9ae7596cda..2526d668d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs @@ -2,17 +2,14 @@ use std::collections::HashSet; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::diff_selection::errors::SelectorError; use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_diff_selection; use sqlbuild_analysis::diff_selection::models::DiffGraph; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -131,7 +128,7 @@ fn selector_error(py: Python<'_>, error: SelectorError) -> PyErr { /// Resolve diff selectors to model names only. #[pyfunction] #[pyo3(signature = (**options))] -fn resolve_diff_model_names<'py>( +pub(crate) fn resolve_diff_model_names<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -149,7 +146,7 @@ fn resolve_diff_model_names<'py>( } /// `resolve_diff_model_names` with its arguments. -fn diff_model_names<'py>( +pub(crate) fn diff_model_names<'py>( project: &Bound<'py, PyAny>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, @@ -178,66 +175,3 @@ fn diff_model_names<'py>( } PyTuple::new(py, names) } - -/// Compile both environments and resolve selected diffable model names. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn run_diff_pipeline<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "run_diff_pipeline", - options, - ( - &[ - "discovered_inputs", - "adapter", - "from_target", - "to_target", - "resolved_connection", - "no_sql_validation", - "select", - "exclude", - ], - &["cli_vars", "external_sql_reference_resolver"], - ), - )?; - let compile = |target: &str| -> PyResult> { - build_compiled_project( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("selected_target", arguments.value(target)?), - ( - "resolved_connection", - arguments.value("resolved_connection")?, - ), - ("no_sql_validation", arguments.value("no_sql_validation")?), - ("cli_vars", arguments.value("cli_vars")?), - ( - "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, - ), - ], - )?), - ) - }; - let left: Bound<'py, PyAny> = compile("from_target")?; - let right: Bound<'py, PyAny> = compile("to_target")?; - let names: Bound<'py, PyTuple> = diff_model_names( - &right, - &arguments.value("select")?, - &arguments.value("exclude")?, - )?; - PyTuple::new(py, [left, right, names.into_any()]) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_diff_model_names, module)?)?; - module.add_function(wrap_pyfunction!(run_diff_pipeline, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs index a1ae3cec86..a55fbc50db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs @@ -1,20 +1,17 @@ //! Project discovery: configuration, declaration files and imported project Python, validated. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; -use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produced}; +use crate::compile_driver::_helpers::setup::integration_loaders::integration_loader_functions; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::compile_driver::constants::PROJECT_DISCOVERIES; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::event_exporting::main::configure_discovered_event_exporters::configure_discovered_event_exporters; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -178,7 +175,7 @@ impl Discoverer { const PYTHON_NODE_GROUPS: [&str; 5] = ["loaders", "tasks", "assets", "checks", "audit_factories"]; /// `OperationLifecycle(operation_kind="project", operation_name=name)`. -fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { +pub(crate) fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { call_with( &host_entry(py, ExecutorHost::OperationLifecycle)?, &[ @@ -189,11 +186,11 @@ fn lifecycle<'py>(py: Python<'py>, name: &str) -> PyResult> { } /// The keyword arguments of one discovery, kept for the frontier stage. -struct DiscoveryRequest { - project_dir: Py, - sql_analysis_enabled_override: Py, - extract_output_column_locations: bool, - secret_policy: SecretPolicy, +pub(crate) struct DiscoveryRequest { + pub(crate) project_dir: Py, + pub(crate) sql_analysis_enabled_override: Py, + pub(crate) extract_output_column_locations: bool, + pub(crate) secret_policy: SecretPolicy, } /// Whether `--allow-missing-secrets`, `SQLBUILD_ALLOW_MISSING_SECRETS` or the local `allow_missing_secrets` setting lets a static command run without provider secrets. @@ -216,50 +213,8 @@ fn missing_secrets_allowed(local_config: &Bound<'_, PyAny>) -> PyResult { ) } -/// Load all raw project inputs from disk before semantic resolution. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn discover_project_inputs<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "discover_project_inputs", - options, - ( - &["project_dir"], - &[ - "sql_analysis_enabled_override", - "extract_output_column_locations", - "secret_policy", - ], - ), - )?; - count_stage(py, PROJECT_DISCOVERIES, 1)?; - let secret_policy: SecretPolicy = if arguments.passed("secret_policy")? { - SecretPolicy::parse(&arguments.value("secret_policy")?.extract::()?)? - } else { - SecretPolicy::Require - }; - let request: DiscoveryRequest = DiscoveryRequest { - project_dir: arguments.value("project_dir")?.unbind(), - sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), - extract_output_column_locations: !arguments.passed("extract_output_column_locations")? - || arguments.flag("extract_output_column_locations")?, - secret_policy, - }; - within(&lifecycle(py, "project_discovery")?, |_| { - let assembled: Bound<'py, PyAny> = - within(&lifecycle(py, "discovery_project_assembly")?, |_| { - assembled_inputs(py, &request) - })?; - produced(FrontierStage::DiscoveredProjectInputs, assembled) - }) -} - /// `_assemble_discovered_project_inputs`: configuration, its checks, discovery and validation. -fn assembled_inputs<'py>( +pub(crate) fn assembled_inputs<'py>( py: Python<'py>, request: &DiscoveryRequest, ) -> PyResult> { @@ -322,7 +277,7 @@ fn assembled_inputs<'py>( /// Discover all project files and functions into one inputs bundle. #[pyfunction] #[pyo3(signature = (**options))] -fn build_discovered_project_inputs<'py>( +pub(crate) fn build_discovered_project_inputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -549,8 +504,3 @@ fn project_inputs<'py>( fields.push(("native_session", declarations.get_item("native_session")?)); new_record_with(py, RecordId::DiscoveredProjectInputs, &fields) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_project_inputs, module)?)?; - module.add_function(wrap_pyfunction!(build_discovered_project_inputs, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/mod.rs new file mode 100644 index 0000000000..f10c47b292 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod analysis; +pub(crate) mod analysis_cache; +pub(crate) mod assembly; +pub(crate) mod command; +pub(crate) mod deferral; +pub(crate) mod diff; +pub(crate) mod discovery; +pub(crate) mod phases; +pub(crate) mod selection_phases; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs index a43af45a37..08e644e2c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs @@ -1,16 +1,18 @@ //! The compile output phases: manifest, DAG artifact, compiled artifacts and withheld test errors. +use crate::compile_driver::main::stages::elapsed_ms::elapsed_ms; +use crate::compile_driver::main::stages::monotonic::monotonic; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; -use crate::bindings::_helpers::compile_command::artifact_batch::ArtifactBatch; -use crate::bindings::_helpers::compile_command::reuse_recording::ArtifactRecorder; -use crate::bindings::_helpers::compile_command::target_writer::{ - Prepared, planning_diagnostics, write_static_target, -}; -use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; -use crate::bindings::_helpers::manifest::builder::build_manifest; +use crate::compile_command::main::artifacts::planning_diagnostics::planning_diagnostics; +use crate::compile_command::main::artifacts::write_static_target::write_static_target; +use crate::compile_command::models::ArtifactBatch; +use crate::compile_command::models::Prepared; +use crate::compile_command::types::ArtifactRecorder; +use crate::dag_artifact::main::build_dag_json::build_dag_json; +use crate::manifest::main::build_manifest::build_manifest; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -34,16 +36,6 @@ fn seconds_since(py: Python<'_>, started: f64) -> PyResult { Ok(format!("{:.2}", now - started)) } -/// A `time.monotonic()` reading. -pub(crate) fn monotonic(py: Python<'_>) -> PyResult { - imported(py, "time", "monotonic")?.call0()?.extract() -} - -/// Milliseconds since a `time.monotonic()` reading, truncated as `int(... * 1000)`. -pub(crate) fn elapsed_ms(py: Python<'_>, started: f64) -> PyResult { - Ok(((monotonic(py)? - started) * 1000.0) as i64) -} - /// `project_dir / "target"`. pub(crate) fn target_dir<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,)) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs new file mode 100644 index 0000000000..44aa2d9d4c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs @@ -0,0 +1,87 @@ +//! Unit-test selector splitting, analysis and rule selection, and static command compilation. + +use std::sync::Arc; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyFrozenSet; + +use crate::compile_driver::_helpers::stages::assembly::assemble_project; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::ScopeData; + +pub(crate) const DATACLASSES_MODULE: &str = "dataclasses"; + +/// Whether a selection names nothing to select or exclude. +pub(crate) fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { + Ok(!native_field(selection, "select")?.is_truthy()? + && !native_field(selection, "exclude")?.is_truthy()?) +} + +/// The planner scopes `selection` resolves over `project`. +pub(crate) fn planner_scopes<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + resolve_planner_scopes( + project, + &new_record_with( + py, + RecordId::PlannerSelection, + &[ + ("select", native_field(selection, "select")?), + ("exclude", native_field(selection, "exclude")?), + ], + )?, + &new_record_with( + py, + RecordId::PlannerPolicies, + &[( + "auto_load_sources", + native_field(selection, "auto_load_sources")?, + )], + )?, + ) +} + +/// The native scope whose models receive deep analysis. +pub(crate) fn analysis_scope(scopes: &Bound<'_, PyAny>) -> PyResult> { + scope_data(&native_field(scopes, "stale_warning_scope")?) +} + +/// Return the models selected for deep analysis, or None to analyse every model. +#[pyfunction] +#[pyo3(signature = (*, compile_inputs, inference_profile, selection))] +pub(crate) fn resolve_analysis_model_names<'py>( + compile_inputs: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = compile_inputs.py(); + if selection.is_none() || unselective(selection)? { + return Ok(py.None().into_bound(py)); + } + let structural_project: Bound<'py, PyAny> = assemble_project( + py, + Some(keywords( + py, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ( + "skip_column_inference", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?), + )?; + let scope = analysis_scope(&planner_scopes(&structural_project, selection)?)?; + Ok(PyFrozenSet::new(py, scope.models.keys())?.into_any()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs new file mode 100644 index 0000000000..2f69896363 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs @@ -0,0 +1,2 @@ +pub mod setup; +pub mod stages; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/attached_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/attached_inputs.rs new file mode 100644 index 0000000000..755317714f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/attached_inputs.rs @@ -0,0 +1,38 @@ +//! The first compile input snapshot, with diagnostics collected while attaching appended. + +use crate::compile_driver::models::InputsRequest; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; +use sqlbuild_python_project::compiled_views::main::project::inputs_with::inputs_with; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::compile_driver::_helpers::setup::inputs::compile_inputs; + +/// The first compile input snapshot, with diagnostics collected while attaching appended. +pub fn attached_inputs<'py>( + py: Python<'py>, + request: &InputsRequest, +) -> PyResult> { + let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); + let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + within(&collector, |violations| { + Ok((compile_inputs(py, request)?, violations.clone())) + })?; + let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; + if !collected.is_truthy()? { + return Ok(built); + } + let diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + native_field(&built, "diagnostics")? + .try_iter()? + .chain(collected.try_iter()?) + .collect::>>()?, + )?; + inputs_with( + &built, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_compiled_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_compiled_project.rs new file mode 100644 index 0000000000..c0032df978 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_compiled_project.rs @@ -0,0 +1,37 @@ +//! Build one compiled project with adapter defaults and target validation applied. + +use crate::compile_driver::_helpers::setup::project::ProjectRequest; +use crate::compile_driver::_helpers::setup::project::compiled_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Build one compiled project with adapter defaults and target validation applied. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_compiled_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_compiled_project", + options, + ( + &["discovered_inputs", "adapter"], + &[ + "selected_target", + "no_sql_validation", + "skip_column_inference", + "column_lineage_mode", + "cli_vars", + "external_sql_reference_resolver", + "resolved_connection", + "analysis_selection", + "source_lexical_syntax", + ], + ), + )?; + compiled_project(py, ProjectRequest::read(&arguments)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph.rs new file mode 100644 index 0000000000..8b241f188b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph.rs @@ -0,0 +1,37 @@ +//! Build the static dependency graph for a compiled project. + +use crate::compile_driver::_helpers::setup::project::GRAPH_OPTIONAL; +use crate::compile_driver::_helpers::setup::project::py_bool; +use crate::compile_driver::_helpers::setup::project::selected_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_driver::constants::PROJECT_GRAPHS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// Build the static dependency graph for a compiled project. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_project_graph<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + count_stage(py, PROJECT_GRAPHS, 1)?; + let mut optional: Vec<&str> = GRAPH_OPTIONAL.to_vec(); + optional.push("no_cache"); + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_graph", + options, + (&["discovered_inputs", "adapter"], &optional), + )?; + let selection: Bound<'py, PyAny> = new_record_with( + py, + RecordId::CompileAnalysisSelection, + &[("no_cache", py_bool(py, arguments.flag("no_cache")?))], + )?; + selected_graph(&arguments, selection) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph_from_compiled_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph_from_compiled_project.rs new file mode 100644 index 0000000000..7cf71058ba --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_graph_from_compiled_project.rs @@ -0,0 +1,23 @@ +//! Build a static dependency graph for an already-compiled project. + +use crate::compile_driver::main::setup::project_graph::project_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Build a static dependency graph for an already-compiled project. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_project_graph_from_compiled_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_graph_from_compiled_project", + options, + (&["project"], &[]), + )?; + project_graph(&arguments.value("project")?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_relation_targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_relation_targets.rs new file mode 100644 index 0000000000..d452ba5d76 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_project_relation_targets.rs @@ -0,0 +1,41 @@ +//! Return every planned project relation for hard-coded name warnings, or None when off. + +use crate::compile_driver::_helpers::setup::relation_targets::RefKinds; +use crate::compile_driver::_helpers::setup::relation_targets::planned_relation; +use crate::compile_driver::_helpers::setup::relation_targets::planned_relation_refs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Return every planned project relation for hard-coded name warnings, or None when off. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_project_relation_targets<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_relation_targets", + options, + (&["adapter", "plan_output"], &[]), + )?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + if !native_field(&plan_output, "enforce_explicit_references")?.is_truthy()? { + return Ok(None); + } + let kinds: RefKinds<'py> = RefKinds::load(py)?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let targets: Bound<'py, PyDict> = PyDict::new(py); + for reference in planned_relation_refs(&plan_output, &kinds)?.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + let relation: Bound<'py, PyAny> = + planned_relation(&adapter, &plan_output, &reference, &kinds)?; + if !relation.is_none() { + targets.set_item(reference, relation)?; + } + } + Ok(Some(targets)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_relation_targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_relation_targets.rs new file mode 100644 index 0000000000..1e8eaa6b06 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_relation_targets.rs @@ -0,0 +1,42 @@ +//! Return adapter-qualified runtime relations required by selected Python nodes. + +use crate::compile_driver::_helpers::setup::relation_targets::RefKinds; +use crate::compile_driver::_helpers::setup::relation_targets::planned_relation_refs; +use crate::compile_driver::_helpers::setup::relation_targets::resolved_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Return adapter-qualified runtime relations required by selected Python nodes. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_python_relation_targets<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_python_relation_targets", + options, + (&["adapter", "project", "plan_output"], &["required_refs"]), + )?; + let kinds: RefKinds<'py> = RefKinds::load(py)?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let project: Bound<'py, PyAny> = arguments.value("project")?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + let required: Bound<'py, PyAny> = arguments.value("required_refs")?; + let refs: Bound<'py, PyAny> = if required.is_none() { + planned_relation_refs(&plan_output, &kinds)? + } else { + required + }; + let targets: Bound<'py, PyDict> = PyDict::new(py); + for reference in refs.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + let relation: Bound<'py, PyAny> = + resolved_relation((&adapter, &project, &plan_output), &reference, &kinds)?; + targets.set_item(reference, relation)?; + } + Ok(targets) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_run_plan_outputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_run_plan_outputs.rs new file mode 100644 index 0000000000..1ea0b41a35 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_python_run_plan_outputs.rs @@ -0,0 +1,89 @@ +//! Attach python-aware selection warnings and plan entries to the plan output. + +use crate::compile_driver::_helpers::setup::python_plan_entries::NodeKinds; +use crate::compile_driver::_helpers::setup::python_plan_entries::node_graph; +use crate::compile_driver::_helpers::setup::python_plan_entries::plan_entries; +use crate::compile_driver::_helpers::setup::python_plan_entries::skipped_ingress_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; + +/// Attach python-aware selection warnings and plan entries to the plan output. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_python_run_plan_outputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_python_run_plan_outputs", + options, + ( + &[ + "discovered_inputs", + "plan_output", + "run_selection", + "selected_python_node_names", + ], + &[], + ), + )?; + let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; + let run_selection: Bound<'py, PyAny> = arguments.value("run_selection")?; + let selected: Bound<'py, PyAny> = arguments.value("selected_python_node_names")?; + if run_selection.is_none() { + return new_record_with( + py, + RecordId::PythonRunPlanOutputs, + &[ + ("plan_output", plan_output), + ("python_plan_entries", PyTuple::empty(py).into_any()), + ("selected_python_node_names", selected), + ], + ); + } + let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; + let kinds: NodeKinds<'py> = NodeKinds::load(py)?; + let mut warnings: Vec> = native_field(&plan_output, "warnings")? + .try_iter()? + .collect::>()?; + warnings.extend(skipped_ingress_warnings( + &plan_output, + &run_selection, + (&graph, &kinds), + )?); + let plan_output: Bound<'py, PyAny> = replaced( + &plan_output, + &[("warnings", PyTuple::new(py, warnings)?.into_any())], + )?; + let selection: Bound<'py, PyAny> = new_record_with( + py, + RecordId::PythonSqlRunSelection, + &[ + ("sql_keys", native_field(&plan_output, "selected_keys")?), + ("python_node_names", selected.clone()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = build_python_sql_run_lifecycle(&selection, &graph)?; + let entries: Bound<'py, PyTuple> = plan_entries( + &lifecycle, + (&graph, &kinds), + &native_field(&plan_output, "python_identity_fingerprints")?, + )?; + new_record_with( + py, + RecordId::PythonRunPlanOutputs, + &[ + ("plan_output", plan_output), + ("python_plan_entries", entries.into_any()), + ("selected_python_node_names", selected), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_static_pipeline_result.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_static_pipeline_result.rs new file mode 100644 index 0000000000..180bec14ff --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/build_static_pipeline_result.rs @@ -0,0 +1,38 @@ +//! Wrap a focused command projection in the established pipeline envelope. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyFloat}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// Wrap a focused command projection in the established pipeline envelope. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_static_pipeline_result<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_static_pipeline_result", + options, + (&["context", "plan_output"], &[]), + )?; + let context: Bound<'py, PyAny> = arguments.value("context")?; + new_record_with( + py, + RecordId::CompilePipelineResult, + &[ + ("project", native_field(&context, "project")?), + ("plan_output", arguments.value("plan_output")?), + ( + "compile_seconds", + native_field(&context, "compile_seconds")?, + ), + ("planning_seconds", PyFloat::new(py, 0.0).into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_adapter_context.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_adapter_context.rs new file mode 100644 index 0000000000..82f1b259e4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_adapter_context.rs @@ -0,0 +1,39 @@ +//! The adapter behaviour compile inputs need, from the discovered project and its adapter. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; + +/// The adapter behaviour compile inputs need, from the discovered project and its adapter. +pub fn compile_adapter_context<'py>( + discovered: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( + &native_field(discovered, "project_config")?, + &py.None().into_bound(py), + ) + .and_then(|value| value.into_bound_py_any(py))?; + new_record_with( + py, + RecordId::CompileAdapterContext, + &[ + ("value_renderer", adapter.clone()), + ("collection_rendering", collection_rendering), + ( + "python_functions_inherit_default_namespace", + adapter.call_method0("python_functions_inherit_default_namespace")?, + ), + ("sql_lexical_syntax", syntax.clone()), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_project_phase.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_project_phase.rs new file mode 100644 index 0000000000..dbbcb9c1d1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_project_phase.rs @@ -0,0 +1,34 @@ +//! Compile a project without opening a warehouse planning connection. + +use crate::compile_driver::main::setup::compiled_phase::compiled_phase; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Compile a project without opening a warehouse planning connection. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn compile_project_phase<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "compile_project_phase", + options, + ( + &["discovered_inputs", "adapter"], + &["options", "on_progress"], + ), + )?; + compiled_phase( + ( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + ), + &arguments.value("options")?, + &arguments.value("on_progress")?, + None, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compiled_phase.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compiled_phase.rs new file mode 100644 index 0000000000..87cab0830a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/compiled_phase.rs @@ -0,0 +1,104 @@ +//! `compile_project_phase` with resolved arguments. + +use crate::compile_driver::_helpers::setup::pipeline::reject_compile_errors; +use crate::compile_driver::_helpers::setup::pipeline_entries::compile_analysis_selection; +use crate::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use crate::compile_driver::main::setup::progress::progress; +use crate::compile_driver::main::setup::timing_tracker::timing_tracker; +use crate::compile_driver::main::stages::monotonic::monotonic; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; + +/// `compile_project_phase` with resolved arguments. +pub fn compiled_phase<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + options: &Bound<'py, PyAny>, + on_progress: &Bound<'py, PyAny>, + on_inputs_ready: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let options: Bound<'py, PyAny> = if options.is_truthy()? { + options.clone() + } else { + new_record(py, RecordId::CompilePipelineOptions, (), None)? + }; + let configured: Bound<'py, PyAny> = native_field(&options, "connection_config")?; + let connection_config: Bound<'py, PyAny> = if configured.is_none() { + build_effective_connection_config( + discovered, + Some(native_field(&options, "selected_target")?), + Some(native_field(&options, "cli_vars")?), + false, + )? + } else { + configured + }; + progress(on_progress, "Compiling project...")?; + let started: f64 = monotonic(py)?; + let compiled: PyResult> = (|| { + let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; + let selection: Bound<'py, PyAny> = match on_inputs_ready { + Some(hook) => replaced(&selection, &[("on_inputs_ready", hook.clone())])?, + None => selection, + }; + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ( + "selected_target", + native_field(&options, "selected_target")?, + ), + ( + "no_sql_validation", + native_field(&options, "no_sql_validation")?, + ), + ("cli_vars", native_field(&options, "cli_vars")?), + ( + "external_sql_reference_resolver", + native_field(&options, "external_sql_reference_resolver")?, + ), + ("resolved_connection", connection_config.clone()), + ("analysis_selection", selection), + ( + "source_lexical_syntax", + native_field(&options, "source_lexical_syntax")?, + ), + ], + )?), + ) + })(); + let compile_seconds: f64 = monotonic(py)? - started; + let tracker: Bound<'py, PyAny> = timing_tracker(py)?; + if !tracker.is_none() { + tracker.setattr("compile_seconds", compile_seconds)?; + } + let project: Bound<'py, PyAny> = compiled?; + reject_compile_errors(&project, &options, on_progress)?; + progress( + on_progress, + &format!("Compiled project. ({compile_seconds:.2}s)"), + )?; + new_record_with( + py, + RecordId::CompiledProjectPhaseResult, + &[ + ("project", project), + ("connection_config", connection_config), + ( + "compile_seconds", + compile_seconds.into_pyobject(py)?.into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/expanded_loader_namespace.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/expanded_loader_namespace.rs new file mode 100644 index 0000000000..0e9faa6b6d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/expanded_loader_namespace.rs @@ -0,0 +1,26 @@ +//! A managed source's `database` and `schema` with project vars expanded and no context templates. + +use crate::compile_driver::_helpers::setup::targets::LOADER_NAMESPACE_LABEL; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_model_config::templates::models::TemplateOptions; +use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; + +/// A managed source's `database` and `schema` with project vars expanded and no context templates. +pub fn expanded_loader_namespace<'py>( + value: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + expanded_template_data( + value, + (effective_vars.downcast()?, &PyDict::new(value.py())), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + LOADER_NAMESPACE_LABEL, + ), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/managed_write_schema.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/managed_write_schema.rs new file mode 100644 index 0000000000..e61e725346 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/managed_write_schema.rs @@ -0,0 +1,40 @@ +//! Raise when neither the resource nor the project names a physical write schema. + +use crate::compile_driver::_helpers::setup::targets::DEFAULTS_HELP_PURPOSE; +use crate::compile_driver::_helpers::setup::targets::DEFAULTS_SECTION; +use crate::compile_driver::_helpers::setup::targets::ignored_connection_note; +use crate::compile_driver::_helpers::setup::targets::planner_error; +use crate::compile_driver::_helpers::setup::targets::project_setting_help; +use crate::compile_driver::_helpers::setup::targets::text_or_none; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::assembly::target_checks::constants::MANAGED_WRITE_SCHEMA_CODE; +use sqlbuild_analysis::assembly::target_checks::errors::TargetError; +use sqlbuild_analysis::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; +use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; + +/// Raise when neither the resource nor the project names a physical write schema. +pub fn managed_write_schema( + resource_label: &str, + schema: &Bound<'_, PyAny>, + (project_schema, ignored): (Option<&str>, Option<&str>), +) -> PyResult<()> { + if explicit_write_schema(text_or_none(schema)?.as_deref()).is_some() || project_schema.is_some() + { + return Ok(()); + } + let py: Python<'_> = schema.py(); + let mut message: String = + target_error_message(&TargetError::ManagedWriteSchema(resource_label.to_owned())); + let help: Option = match ignored { + Some(value) => { + message.push_str(&ignored_connection_note("schema", value)); + Some(project_setting_help( + py, + (DEFAULTS_HELP_PURPOSE, DEFAULTS_SECTION), + ("schema", value), + )?) + } + None => None, + }; + Err(planner_error(py, message, MANAGED_WRITE_SCHEMA_CODE, help)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs new file mode 100644 index 0000000000..7217f6eb02 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -0,0 +1,29 @@ +pub mod attached_inputs; +pub mod build_compiled_project; +pub mod build_project_graph; +pub mod build_project_graph_from_compiled_project; +pub mod build_project_relation_targets; +pub mod build_python_relation_targets; +pub mod build_python_run_plan_outputs; +pub mod build_static_pipeline_result; +pub mod compile_adapter_context; +pub mod compile_project_phase; +pub mod compiled_phase; +pub mod expanded_loader_namespace; +pub mod managed_write_schema; +pub mod plan_has_executable_work; +pub mod preserved_loader_schemas; +pub mod progress; +pub mod project_graph; +pub mod project_target_settings; +pub mod python_read_source_names_for; +pub mod register_inputs; +pub mod register_pipeline; +pub mod register_pipeline_entries; +pub mod register_project; +pub mod register_python_plan_entries; +pub mod register_relation_targets; +pub mod register_targets; +pub mod run_compile_only_pipeline; +pub mod selected_target_settings; +pub mod timing_tracker; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/plan_has_executable_work.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/plan_has_executable_work.rs new file mode 100644 index 0000000000..e01f8e9c9a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/plan_has_executable_work.rs @@ -0,0 +1,33 @@ +//! Return whether a planned build has runtime work to execute. + +use crate::compile_driver::_helpers::setup::pipeline_entries::PLAN_WORK_ENTRIES; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Return whether a planned build has runtime work to execute. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn plan_has_executable_work<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "plan_has_executable_work", + options, + (&["plan"], &["python_plan_entries"]), + )?; + let plan: Bound<'py, PyAny> = arguments.value("plan")?; + for name in PLAN_WORK_ENTRIES { + if native_field(&plan, name)?.is_truthy()? { + return Ok(true); + } + } + if !arguments.passed("python_plan_entries")? { + return Ok(false); + } + arguments.value("python_plan_entries")?.is_truthy() +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs new file mode 100644 index 0000000000..c02dee4ac6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -0,0 +1,51 @@ +//! Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. + +use crate::compile_attachment::main::resolved_target::resolved_target; +use crate::compile_driver::_helpers::setup::targets::planner_error; +use crate::compile_driver::_helpers::setup::targets::text_or_none; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::assembly::target_checks::constants::{ + MANAGED_WRITE_SCHEMA_CODE, PRESERVE_TARGET_VALUE, +}; +use sqlbuild_analysis::assembly::target_checks::errors::TargetError; +use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashSet; + +/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +pub fn preserved_loader_schemas( + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, + loaders: &HashSet, +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let (target_name, target_config) = resolved_target(discovered, selected_target)?; + if target_name.is_none() { + return Ok(()); + } + if !text_or_none(&native_field(&target_config, "schema")?)? + .is_some_and(|schema| python_strip(&schema) == PRESERVE_TARGET_VALUE) + { + return Ok(()); + } + for loader in native_field(discovered, "loader_functions")?.try_iter()? { + let loader: Bound<'_, PyAny> = loader?; + let name: String = native_field(&loader, "name")?.extract()?; + let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; + if !loaders.contains(&name) || destination.as_deref().is_some_and(|d| d.contains('.')) { + continue; + } + let table: String = destination.unwrap_or_else(|| format!("__loader__{name}")); + return Err(planner_error( + py, + target_error_message(&TargetError::PreservedLoaderSchema(name)), + MANAGED_WRITE_SCHEMA_CODE, + Some(format!( + "to choose where it writes, set its destination in its @loader decorator:\n \ + @loader(destination=\"my_schema.{table}\")" + )), + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/progress.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/progress.rs new file mode 100644 index 0000000000..820fb7746b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/progress.rs @@ -0,0 +1,11 @@ +//! Report `message` through `on_progress` when a callback is set. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// Report `message` through `on_progress` when a callback is set. +pub fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + if !on_progress.is_none() { + on_progress.call1((message,))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_graph.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_graph.rs new file mode 100644 index 0000000000..48aef1320a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_graph.rs @@ -0,0 +1,16 @@ +//! The project's natively held graph; dict views are built only when a boundary reads them. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::models::{CompiledProjectGraph, NativeProjectGraph}; + +/// The project's natively held graph; dict views are built only when a boundary reads them. +pub fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let native: Bound<'py, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + Ok(Bound::new( + project.py(), + CompiledProjectGraph::new(project.clone().unbind(), native.unbind()), + )? + .into_any()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_target_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_target_settings.rs new file mode 100644 index 0000000000..d6d9b84cee --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_target_settings.rs @@ -0,0 +1,26 @@ +//! `target_settings` of a compiled project's target. + +use crate::compile_driver::_helpers::setup::targets::target_settings; +use crate::compile_driver::models::TargetSettingsValues; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `target_settings` of a compiled project's target. +pub fn project_target_settings<'py>( + discovered: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + (connection, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { + target_settings( + (discovered, &native_field(project, "effective_target_name")?), + ( + native_field(project, "effective_target_database")?.extract()?, + native_field(project, "effective_target_schema")?.extract()?, + ), + ( + connection, + &native_field(project, "effective_vars")?, + adapter, + ), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs new file mode 100644 index 0000000000..2bb6e5f761 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs @@ -0,0 +1,48 @@ +//! Return sources declared by the selected Python nodes and their Python upstreams. + +use crate::compile_driver::_helpers::setup::python_plan_entries::node_graph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Return sources declared by the selected Python nodes and their Python upstreams. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn python_read_source_names_for<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "python_read_source_names_for", + options, + (&["discovered_inputs", "python_node_names"], &[]), + )?; + let frozenset: Bound<'py, PyAny> = imported(py, "builtins", "frozenset")?; + let selected: Bound<'py, PyAny> = arguments.value("python_node_names")?; + if !selected.is_truthy()? { + return frozenset.call0(); + } + let graph: Bound<'py, PyAny> = node_graph(&arguments.value("discovered_inputs")?)?; + let source: Bound<'py, PyAny> = + host_entry(py, UserCodeHost::SqlResourceRefKind)?.getattr("SOURCE")?; + let mut names: Vec> = Vec::new(); + for reference in call_with( + &native_field(&graph, "selected_sql_refs")?, + &[("selected_names", selected)], + )? + .try_iter()? + { + let reference: Bound<'py, PyAny> = reference?; + if native_field(&reference, "kind")?.eq(&source)? { + names.push(native_field(&reference, "name")?); + } + } + frozenset.call1((PyTuple::new(py, names)?,)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_inputs.rs new file mode 100644 index 0000000000..29b59b3d94 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_inputs.rs @@ -0,0 +1,10 @@ +//! Register the inputs functions and classes. + +use crate::compile_driver::_helpers::setup::inputs::build_compile_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the inputs functions and classes. +pub fn register_inputs(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_compile_inputs, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline.rs new file mode 100644 index 0000000000..fac2dcfb7b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline.rs @@ -0,0 +1,11 @@ +//! Register the pipeline functions and classes. + +use crate::compile_driver::main::setup::compile_project_phase::compile_project_phase; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the pipeline functions and classes. +pub fn register_pipeline(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_project_phase, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline_entries.rs new file mode 100644 index 0000000000..0ac8064f44 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline_entries.rs @@ -0,0 +1,24 @@ +//! Register the pipeline entries functions and classes. + +use crate::compile_driver::_helpers::setup::pipeline_entries::resolve_compile_analysis_selection; +use crate::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; +use crate::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; +use crate::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; +use crate::compile_driver::main::setup::run_compile_only_pipeline::run_compile_only_pipeline; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the pipeline entries functions and classes. +pub fn register_pipeline_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + resolve_compile_analysis_selection, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_project_graph_from_compiled_project, + module + )?)?; + module.add_function(wrap_pyfunction!(plan_has_executable_work, module)?)?; + module.add_function(wrap_pyfunction!(build_static_pipeline_result, module)?)?; + module.add_function(wrap_pyfunction!(run_compile_only_pipeline, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs new file mode 100644 index 0000000000..84dc88e991 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs @@ -0,0 +1,17 @@ +//! Register the project functions and classes. + +use crate::compile_driver::_helpers::setup::project::build_project_graph_with_analysis_selection; +use crate::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use crate::compile_driver::main::setup::build_project_graph::build_project_graph; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the project functions and classes. +pub fn register_project(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_compiled_project, module)?)?; + module.add_function(wrap_pyfunction!( + build_project_graph_with_analysis_selection, + module + )?)?; + module.add_function(wrap_pyfunction!(build_project_graph, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs new file mode 100644 index 0000000000..598e71194b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs @@ -0,0 +1,14 @@ +//! Register the python plan entries functions and classes. + +use crate::compile_driver::_helpers::setup::python_plan_entries::build_python_plan_entries; +use crate::compile_driver::main::setup::build_python_run_plan_outputs::build_python_run_plan_outputs; +use crate::compile_driver::main::setup::python_read_source_names_for::python_read_source_names_for; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the python plan entries functions and classes. +pub fn register_python_plan_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(python_read_source_names_for, module)?)?; + module.add_function(wrap_pyfunction!(build_python_run_plan_outputs, module)?)?; + module.add_function(wrap_pyfunction!(build_python_plan_entries, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_relation_targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_relation_targets.rs new file mode 100644 index 0000000000..3de097f968 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_relation_targets.rs @@ -0,0 +1,12 @@ +//! Register the relation targets functions and classes. + +use crate::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; +use crate::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the relation targets functions and classes. +pub fn register_relation_targets(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_relation_targets, module)?)?; + module.add_function(wrap_pyfunction!(build_project_relation_targets, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs new file mode 100644 index 0000000000..1f7a9c3d9a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs @@ -0,0 +1,22 @@ +//! Register the targets functions and classes. + +use crate::compile_driver::_helpers::setup::targets::validate_managed_loader_target_isolation; +use crate::compile_driver::_helpers::setup::targets::validate_managed_write_schemas; +use crate::compile_driver::_helpers::setup::targets::validate_named_target_schema_strategy; +use crate::compile_driver::_helpers::setup::targets::validate_project_targets; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the targets functions and classes. +pub fn register_targets(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + validate_named_target_schema_strategy, + module + )?)?; + module.add_function(wrap_pyfunction!( + validate_managed_loader_target_isolation, + module + )?)?; + module.add_function(wrap_pyfunction!(validate_project_targets, module)?)?; + module.add_function(wrap_pyfunction!(validate_managed_write_schemas, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/run_compile_only_pipeline.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/run_compile_only_pipeline.rs new file mode 100644 index 0000000000..256110f993 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/run_compile_only_pipeline.rs @@ -0,0 +1,58 @@ +//! Compile a project for consumers that construct their own focused plans. + +use crate::compile_driver::main::setup::compile_project_phase::compile_project_phase; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyFloat}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// Compile a project for consumers that construct their own focused plans. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn run_compile_only_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_compile_only_pipeline", + options, + ( + &["discovered_inputs", "adapter", "options"], + &["on_progress"], + ), + )?; + let compiled: Bound<'py, PyAny> = compile_project_phase( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("options", arguments.value("options")?), + ("on_progress", arguments.value("on_progress")?), + ], + )?), + )?; + new_record_with( + py, + RecordId::CompilePipelineResult, + &[ + ("project", native_field(&compiled, "project")?), + ( + "plan_output", + new_record(py, RecordId::PlanOutput, (), None)?, + ), + ( + "compile_seconds", + native_field(&compiled, "compile_seconds")?, + ), + ("planning_seconds", PyFloat::new(py, 0.0).into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/selected_target_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/selected_target_settings.rs new file mode 100644 index 0000000000..022135c54b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/selected_target_settings.rs @@ -0,0 +1,31 @@ +//! The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. + +use crate::compile_attachment::main::effective_target_namespace::effective_target_namespace; +use crate::compile_driver::_helpers::setup::targets::target_settings; +use crate::compile_driver::models::TargetSettingsValues; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; + +/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. +pub fn selected_target_settings<'py>( + (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult { + let namespace: Bound<'py, PyTuple> = effective_target_namespace( + (discovered, selected_target), + (adapter_name, connection, adapter), + )?; + target_settings( + (discovered, &namespace.get_item(0)?), + ( + namespace.get_item(2)?.extract()?, + namespace.get_item(3)?.extract()?, + ), + (connection, effective_vars, adapter), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/timing_tracker.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/timing_tracker.rs new file mode 100644 index 0000000000..04da48f141 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/timing_tracker.rs @@ -0,0 +1,9 @@ +//! The current build's phase timing tracker, when one is recording. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::diagnostics::main::current_tracker::current_tracker; + +/// The current build's phase timing tracker, when one is recording. +pub fn timing_tracker(py: Python<'_>) -> PyResult> { + current_tracker(py) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/analyze_compile_project.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/analyze_compile_project.rs new file mode 100644 index 0000000000..db64c11911 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/analyze_compile_project.rs @@ -0,0 +1,16 @@ +//! Discover, compile and validate the project into one `CompileAnalysis`. + +use crate::compile_driver::_helpers::stages::analysis::analyze; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +/// Discover, compile and validate the project into one `CompileAnalysis`. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn analyze_compile_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + analyze(py, &options.unwrap_or_else(|| PyDict::new(py))) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/apply_unit_test_selectors.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/apply_unit_test_selectors.rs new file mode 100644 index 0000000000..e1653dd648 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/apply_unit_test_selectors.rs @@ -0,0 +1,15 @@ +//! Return options whose selectors name only resources, with unit tests selected apart. + +use crate::compile_driver::main::stages::unit_test_selectors_applied::unit_test_selectors_applied; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// Return options whose selectors name only resources, with unit tests selected apart. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs))] +pub fn apply_unit_test_selectors<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + unit_test_selectors_applied(options, discovered_inputs) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/build_deferred_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/build_deferred_locations.rs new file mode 100644 index 0000000000..af692c1cec --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/build_deferred_locations.rs @@ -0,0 +1,43 @@ +//! Build physical locations for all models and seeds under a deferred target. + +use crate::compile_driver::_helpers::stages::deferral::DeferredNaming; +use crate::compile_driver::_helpers::stages::deferral::deferred_locations; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Build physical locations for all models and seeds under a deferred target. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_deferred_locations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_deferred_locations", + options, + ( + &[ + "project", + "deferred_target_config", + "effective_vars", + "default_schema", + "default_database", + "render_qualified_name", + ], + &[], + ), + )?; + deferred_locations( + &arguments.value("project")?, + &DeferredNaming { + target_config: arguments.value("deferred_target_config")?, + effective_vars: arguments.value("effective_vars")?, + default_schema: arguments.value("default_schema")?, + default_database: arguments.value("default_database")?, + render_qualified_name: arguments.value("render_qualified_name")?, + }, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/discover_project_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/discover_project_inputs.rs new file mode 100644 index 0000000000..971e077989 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/discover_project_inputs.rs @@ -0,0 +1,57 @@ +//! Load all raw project inputs from disk before semantic resolution. + +use crate::compile_driver::_helpers::stages::discovery::DiscoveryRequest; +use crate::compile_driver::_helpers::stages::discovery::assembled_inputs; +use crate::compile_driver::_helpers::stages::discovery::lifecycle; +use crate::frontier::main::produced::produced; +use crate::frontier::models::FrontierStage; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_driver::constants::PROJECT_DISCOVERIES; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_front::discovery::models::SecretPolicy; + +/// Load all raw project inputs from disk before semantic resolution. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn discover_project_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_project_inputs", + options, + ( + &["project_dir"], + &[ + "sql_analysis_enabled_override", + "extract_output_column_locations", + "secret_policy", + ], + ), + )?; + count_stage(py, PROJECT_DISCOVERIES, 1)?; + let secret_policy: SecretPolicy = if arguments.passed("secret_policy")? { + SecretPolicy::parse(&arguments.value("secret_policy")?.extract::()?)? + } else { + SecretPolicy::Require + }; + let request: DiscoveryRequest = DiscoveryRequest { + project_dir: arguments.value("project_dir")?.unbind(), + sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), + extract_output_column_locations: !arguments.passed("extract_output_column_locations")? + || arguments.flag("extract_output_column_locations")?, + secret_policy, + }; + within(&lifecycle(py, "project_discovery")?, |_| { + let assembled: Bound<'py, PyAny> = + within(&lifecycle(py, "discovery_project_assembly")?, |_| { + assembled_inputs(py, &request) + })?; + produced(FrontierStage::DiscoveredProjectInputs, assembled) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/elapsed_ms.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/elapsed_ms.rs new file mode 100644 index 0000000000..bf161a969f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/elapsed_ms.rs @@ -0,0 +1,9 @@ +//! Milliseconds since a `time.monotonic()` reading, truncated as `int(... * 1000)`. + +use crate::compile_driver::main::stages::monotonic::monotonic; +use pyo3::prelude::{PyResult, Python}; + +/// Milliseconds since a `time.monotonic()` reading, truncated as `int(... * 1000)`. +pub fn elapsed_ms(py: Python<'_>, started: f64) -> PyResult { + Ok(((monotonic(py)? - started) * 1000.0) as i64) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/gather_deferred_relations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/gather_deferred_relations.rs new file mode 100644 index 0000000000..a1e01bf854 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/gather_deferred_relations.rs @@ -0,0 +1,27 @@ +//! Gather existing relations from the deferred target's relation-location schemas. + +use crate::compile_driver::_helpers::stages::deferral::deferred_relations; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Gather existing relations from the deferred target's relation-location schemas. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn gather_deferred_relations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "gather_deferred_relations", + options, + (&["adapter", "connection", "deferred_locations"], &[]), + )?; + deferred_relations( + &arguments.value("adapter")?, + &arguments.value("connection")?, + &arguments.value("deferred_locations")?, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/mod.rs new file mode 100644 index 0000000000..9c22b40db2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/mod.rs @@ -0,0 +1,25 @@ +pub mod analyze_compile_project; +pub mod apply_unit_test_selectors; +pub mod build_deferred_locations; +pub mod discover_project_inputs; +pub mod elapsed_ms; +pub mod gather_deferred_relations; +pub mod monotonic; +pub mod record_analysis_cache_metrics; +pub mod register_analysis; +pub mod register_analysis_cache; +pub mod register_assembly; +pub mod register_command; +pub mod register_deferral; +pub mod register_diff; +pub mod register_discovery; +pub mod register_selection_phases; +pub mod resolve_configured_rule_selection; +pub mod resolve_deferred_target_config; +pub mod resolve_model_deferral_inputs; +pub mod resolved_request; +pub mod run_compile_command; +pub mod run_diff_pipeline; +pub mod store_and_print; +pub mod unit_test_selectors_applied; +pub mod unrecorded_compile; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/monotonic.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/monotonic.rs new file mode 100644 index 0000000000..24bc12b19f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/monotonic.rs @@ -0,0 +1,9 @@ +//! A `time.monotonic()` reading. + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// A `time.monotonic()` reading. +pub fn monotonic(py: Python<'_>) -> PyResult { + imported(py, "time", "monotonic")?.call0()?.extract() +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/record_analysis_cache_metrics.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/record_analysis_cache_metrics.rs new file mode 100644 index 0000000000..3f5acbb3ef --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/record_analysis_cache_metrics.rs @@ -0,0 +1,23 @@ +//! Record semantic analysis cache outcomes for one invocation. + +use crate::compile_driver::models::AnalysisCacheCounts; +use sqlbuild_host::compile_context::main::record_compile_metric::record_compile_metric; + +/// Record semantic analysis cache outcomes for one invocation. +pub fn record_analysis_cache_metrics( + AnalysisCacheCounts { + batch_hits, + entry_hits, + misses, + bypasses, + }: AnalysisCacheCounts, +) { + for (metric, value) in [ + ("analysis_batch_cache_hits", batch_hits), + ("analysis_entry_cache_hits", entry_hits), + ("analysis_cache_misses", misses), + ("analysis_cache_bypasses", bypasses), + ] { + record_compile_metric(metric, i64::try_from(value).unwrap_or(i64::MAX)); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis.rs new file mode 100644 index 0000000000..3b8bb2b6df --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis.rs @@ -0,0 +1,10 @@ +//! Register the analysis functions and classes. + +use crate::compile_driver::main::stages::analyze_compile_project::analyze_compile_project; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the analysis functions and classes. +pub fn register_analysis(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(analyze_compile_project, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis_cache.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis_cache.rs new file mode 100644 index 0000000000..d57bcf8f43 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_analysis_cache.rs @@ -0,0 +1,10 @@ +//! Register the analysis cache functions and classes. + +use crate::compile_driver::_helpers::stages::analysis_cache::build_analysis_cache_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the analysis cache functions and classes. +pub fn register_analysis_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_analysis_cache_context, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_assembly.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_assembly.rs new file mode 100644 index 0000000000..10674365cd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_assembly.rs @@ -0,0 +1,12 @@ +//! Register the assembly functions and classes. + +use crate::compile_driver::_helpers::stages::assembly::assemble_project; +use crate::compile_driver::_helpers::stages::assembly::assembly_stage_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the assembly functions and classes. +pub fn register_assembly(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_project, module)?)?; + module.add_function(wrap_pyfunction!(assembly_stage_inputs, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_command.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_command.rs new file mode 100644 index 0000000000..2a9284fa2d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_command.rs @@ -0,0 +1,12 @@ +//! Register the command functions and classes. + +use crate::compile_driver::_helpers::stages::command::defer_compile_teardown_to_exit; +use crate::compile_driver::main::stages::run_compile_command::run_compile_command; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the command functions and classes. +pub fn register_command(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(defer_compile_teardown_to_exit, module)?)?; + module.add_function(wrap_pyfunction!(run_compile_command, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs new file mode 100644 index 0000000000..604d56c7e1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs @@ -0,0 +1,16 @@ +//! Register the deferral functions and classes. + +use crate::compile_driver::main::stages::build_deferred_locations::build_deferred_locations; +use crate::compile_driver::main::stages::gather_deferred_relations::gather_deferred_relations; +use crate::compile_driver::main::stages::resolve_deferred_target_config::resolve_deferred_target_config; +use crate::compile_driver::main::stages::resolve_model_deferral_inputs::resolve_model_deferral_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the deferral functions and classes. +pub fn register_deferral(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_deferred_locations, module)?)?; + module.add_function(wrap_pyfunction!(resolve_deferred_target_config, module)?)?; + module.add_function(wrap_pyfunction!(gather_deferred_relations, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_deferral_inputs, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_diff.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_diff.rs new file mode 100644 index 0000000000..d5250b9a22 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_diff.rs @@ -0,0 +1,12 @@ +//! Register the diff functions and classes. + +use crate::compile_driver::_helpers::stages::diff::resolve_diff_model_names; +use crate::compile_driver::main::stages::run_diff_pipeline::run_diff_pipeline; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the diff functions and classes. +pub fn register_diff(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_diff_model_names, module)?)?; + module.add_function(wrap_pyfunction!(run_diff_pipeline, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_discovery.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_discovery.rs new file mode 100644 index 0000000000..074298a7c1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_discovery.rs @@ -0,0 +1,12 @@ +//! Register the discovery functions and classes. + +use crate::compile_driver::_helpers::stages::discovery::build_discovered_project_inputs; +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the discovery functions and classes. +pub fn register_discovery(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_project_inputs, module)?)?; + module.add_function(wrap_pyfunction!(build_discovered_project_inputs, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_selection_phases.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_selection_phases.rs new file mode 100644 index 0000000000..5444fe7fe4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_selection_phases.rs @@ -0,0 +1,15 @@ +//! Register the selection phases functions and classes. + +use crate::compile_driver::_helpers::stages::selection_phases::resolve_analysis_model_names; +use crate::compile_driver::main::stages::apply_unit_test_selectors::apply_unit_test_selectors; +use crate::compile_driver::main::stages::resolve_configured_rule_selection::resolve_configured_rule_selection; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the selection phases functions and classes. +pub fn register_selection_phases(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(apply_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(resolve_configured_rule_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_analysis_model_names, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_configured_rule_selection.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_configured_rule_selection.rs new file mode 100644 index 0000000000..1e96525976 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_configured_rule_selection.rs @@ -0,0 +1,38 @@ +//! Align configured-rule subjects with models that received deep SQL analysis. + +use crate::compile_driver::_helpers::setup::pipeline_entries::compile_analysis_selection; +use crate::compile_driver::_helpers::stages::selection_phases::analysis_scope; +use crate::compile_driver::_helpers::stages::selection_phases::planner_scopes; +use crate::compile_driver::_helpers::stages::selection_phases::unselective; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Align configured-rule subjects with models that received deep SQL analysis. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs, project))] +pub fn resolve_configured_rule_selection<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = options.py(); + if !native_field(options, "sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + return Ok(PyFrozenSet::empty(py)?.into_any()); + } + let selection: Bound<'py, PyAny> = compile_analysis_selection(options, discovered_inputs)?; + if selection.is_none() || unselective(&selection)? { + return Ok(py.None().into_bound(py)); + } + let scope = analysis_scope(&planner_scopes(project, &selection)?)?; + let keys: Vec> = scope + .models + .values() + .map(|model| native_field(&model.bind(py), "key")) + .collect::>()?; + Ok(PyFrozenSet::new(py, keys)?.into_any()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_deferred_target_config.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_deferred_target_config.rs new file mode 100644 index 0000000000..fdf97222ce --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_deferred_target_config.rs @@ -0,0 +1,30 @@ +//! Validate and resolve the deferred target config. + +use crate::compile_driver::_helpers::stages::deferral::deferred_target_config; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Validate and resolve the deferred target config. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn resolve_deferred_target_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_deferred_target_config", + options, + ( + &["discovered_inputs", "defer_to", "current_target_name"], + &[], + ), + )?; + deferred_target_config( + &arguments.value("discovered_inputs")?, + &arguments.value("defer_to")?, + &arguments.value("current_target_name")?, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs new file mode 100644 index 0000000000..5795ff7349 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs @@ -0,0 +1,72 @@ +//! Resolve native target locations and warehouse relations for model deferral. + +use crate::compile_driver::_helpers::stages::deferral::DeferredNaming; +use crate::compile_driver::_helpers::stages::deferral::deferred_locations; +use crate::compile_driver::_helpers::stages::deferral::deferred_relations; +use crate::compile_driver::_helpers::stages::deferral::deferred_target_config; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// Resolve native target locations and warehouse relations for model deferral. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn resolve_model_deferral_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_model_deferral_inputs", + options, + ( + &[ + "project", + "discovered_inputs", + "adapter", + "connection", + "defer_to", + ], + &[], + ), + )?; + let defer_to: Bound<'py, PyAny> = arguments.value("defer_to")?; + if defer_to.is_none() { + return new_record(py, RecordId::DeferralInputs, (), None); + } + let project: Bound<'py, PyAny> = arguments.value("project")?; + let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; + let target_config: Bound<'py, PyAny> = deferred_target_config( + &arguments.value("discovered_inputs")?, + &defer_to, + &native_field(&project, "effective_target_name")?, + )?; + let locations: Bound<'py, PyDict> = deferred_locations( + &project, + &DeferredNaming { + target_config, + effective_vars: native_field(&project, "effective_vars")?, + default_schema: adapter.call_method0("default_schema")?, + default_database: adapter.call_method0("default_database")?, + render_qualified_name: native_field(&adapter, "render_qualified_name")?, + }, + )?; + let relations: Bound<'py, PyDict> = deferred_relations( + &adapter, + &arguments.value("connection")?, + locations.as_any(), + )?; + new_record_with( + py, + RecordId::DeferralInputs, + &[ + ("deferred_locations", locations.into_any()), + ("deferred_relations", relations.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolved_request.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolved_request.rs new file mode 100644 index 0000000000..bff1d8fef8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolved_request.rs @@ -0,0 +1,15 @@ +//! `request` with its project directory resolved to the working directory when unset. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; + +/// `request` with its project directory resolved to the working directory when unset. +pub fn resolved_request<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + if !native_field(request, "project_dir")?.is_none() { + return Ok(request.clone()); + } + let cwd: Bound<'py, PyAny> = imported(request.py(), "pathlib", "Path")?.call_method0("cwd")?; + replaced(&request.clone(), &[("project_dir", cwd)]) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_compile_command.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_compile_command.rs new file mode 100644 index 0000000000..320da87a99 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_compile_command.rs @@ -0,0 +1,41 @@ +//! Run one full compile, recording it for reuse; returns the exit code. + +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_driver::_helpers::stages::command::bypass_attempt; +use crate::compile_driver::_helpers::stages::command::collecting_later; +use crate::compile_driver::_helpers::stages::command::compile_status; +use crate::compile_driver::_helpers::stages::command::recorded_compile; +use crate::compile_driver::main::stages::monotonic::monotonic; +use crate::compile_driver::main::stages::resolved_request::resolved_request; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::models::Status; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Run one full compile, recording it for reuse; returns the exit code. +#[pyfunction] +#[pyo3(signature = (request, reuse_attempt=None))] +pub fn run_compile_command<'py>( + py: Python<'py>, + request: Bound<'py, PyAny>, + reuse_attempt: Option>, +) -> PyResult { + let total_start: f64 = match &reuse_attempt { + Some(attempt) => native_field(attempt, "started")?.extract()?, + None => monotonic(py)?, + }; + let effective_request: Bound<'py, PyAny> = resolved_request(&request)?; + let attempt: Bound<'py, PyAny> = match reuse_attempt { + Some(attempt) => attempt, + None => bypass_attempt(py, &effective_request, total_start)?, + }; + let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); + let status: Status<'py> = Status(compile_status(&request)?); + collecting_later(py, || { + recorded_compile( + (&effective_request, &attempt), + (&output, &status), + total_start, + ) + })? +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_diff_pipeline.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_diff_pipeline.rs new file mode 100644 index 0000000000..9ea99b552c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_diff_pipeline.rs @@ -0,0 +1,67 @@ +//! Compile both environments and resolve selected diffable model names. + +use crate::compile_driver::_helpers::stages::diff::diff_model_names; +use crate::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Compile both environments and resolve selected diffable model names. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn run_diff_pipeline<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_diff_pipeline", + options, + ( + &[ + "discovered_inputs", + "adapter", + "from_target", + "to_target", + "resolved_connection", + "no_sql_validation", + "select", + "exclude", + ], + &["cli_vars", "external_sql_reference_resolver"], + ), + )?; + let compile = |target: &str| -> PyResult> { + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("selected_target", arguments.value(target)?), + ( + "resolved_connection", + arguments.value("resolved_connection")?, + ), + ("no_sql_validation", arguments.value("no_sql_validation")?), + ("cli_vars", arguments.value("cli_vars")?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ], + )?), + ) + }; + let left: Bound<'py, PyAny> = compile("from_target")?; + let right: Bound<'py, PyAny> = compile("to_target")?; + let names: Bound<'py, PyTuple> = diff_model_names( + &right, + &arguments.value("select")?, + &arguments.value("exclude")?, + )?; + PyTuple::new(py, [left, right, names.into_any()]) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/store_and_print.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/store_and_print.rs new file mode 100644 index 0000000000..0a4522ab8d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/store_and_print.rs @@ -0,0 +1,50 @@ +//! Store a finished compile for reuse, then print its recorded output. + +use crate::compile_command::main::invocation::write_reusable_compile::write_reusable_compile; +use crate::compile_driver::types::PhasesOutcome; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Store a finished compile for reuse, then print its recorded output. +pub fn store_and_print<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + output: &Bound<'py, PyAny>, + phases: PhasesOutcome<'py>, +) -> PyResult { + let py: Python<'py> = request.py(); + let (outcome, input_reads, artifact_writes): PhasesOutcome<'py> = phases; + let stored: PyResult<()> = write_reusable_compile( + py, + Some(keywords( + py, + &[ + ("attempt", attempt.clone()), + ("output", output.clone()), + ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), + ("input_reads", input_reads), + ("artifact_writes", artifact_writes), + ( + "compile_cache_enabled", + PyBool::new(py, outcome.compile_cache_enabled) + .to_owned() + .into_any(), + ), + ( + "artifacts_written", + PyBool::new(py, outcome.artifacts_written) + .to_owned() + .into_any(), + ), + ("dag_artifact_path", outcome.dag_artifact_path), + ("json_output", native_field(request, "json_output")?), + ], + )?), + ); + let printed: PyResult> = output.call_method0("print_stdout"); + printed?; + stored?; + Ok(outcome.exit_code) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unit_test_selectors_applied.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unit_test_selectors_applied.rs new file mode 100644 index 0000000000..32c61c5d99 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unit_test_selectors_applied.rs @@ -0,0 +1,47 @@ +//! Options whose selectors name only resources, with unit tests selected apart. + +use crate::compile_driver::_helpers::stages::selection_phases::DATACLASSES_MODULE; +use crate::compile_driver::_helpers::stages::selection_phases::unselective; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; + +/// Options whose selectors name only resources, with unit tests selected apart. +pub fn unit_test_selectors_applied<'py>( + options: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let resolved: Bound<'py, PyAny> = if options.is_none() { + new_record(py, RecordId::CompilePipelineOptions, (), None)? + } else { + options.clone() + }; + if unselective(&resolved)? { + return Ok(resolved); + } + let split: Bound<'py, PyAny> = split_project_unit_test_selectors( + &native_field(&resolved, "select")?, + &native_field(&resolved, "exclude")?, + discovered, + native_field(&resolved, "accepts_unit_test_selectors")?.extract()?, + )?; + imported(py, DATACLASSES_MODULE, "replace")?.call( + (resolved,), + Some(&keywords( + py, + &[ + ("select", native_field(&split, "select")?), + ("exclude", native_field(&split, "exclude")?), + ( + "sql_test_selection", + native_field(&split, "sql_test_selection")?, + ), + ], + )?), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unrecorded_compile.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unrecorded_compile.rs new file mode 100644 index 0000000000..703d17a58f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/unrecorded_compile.rs @@ -0,0 +1,34 @@ +//! The resolved request, output recorder and phases of a compile neither stored nor printed. + +use crate::compile_command::models::RecordedCompileOutput; +use crate::compile_driver::_helpers::stages::command::collecting_later; +use crate::compile_driver::_helpers::stages::command::compile_phases; +use crate::compile_driver::main::stages::resolved_request::resolved_request; +use crate::compile_driver::types::PhasesOutcome; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::models::Status; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The resolved request, output recorder and phases of a compile neither stored nor printed. +pub fn unrecorded_compile<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult<( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + PyResult>, +)> { + let py: Python<'py> = request.py(); + let total_start: f64 = native_field(attempt, "started")?.extract()?; + let effective_request: Bound<'py, PyAny> = resolved_request(request)?; + let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); + let status: Status<'py> = Status(py.None().into_bound(py)); + let phases: PyResult> = collecting_later(py, || { + compile_phases( + (&effective_request, attempt), + (&output, &status), + total_start, + ) + })?; + Ok((effective_request, output, phases)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs new file mode 100644 index 0000000000..cb9dd32021 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs @@ -0,0 +1,6 @@ +//! The compile driver: the analysis sequence, project assembly and the analysis cache. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs new file mode 100644 index 0000000000..aece8e4552 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -0,0 +1,115 @@ +//! Public records of compile driver. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyBool; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// Semantic analysis cache outcome counts for one invocation. +#[derive(Debug)] +pub struct AnalysisCacheCounts { + pub batch_hits: usize, + pub entry_hits: usize, + pub misses: usize, + pub bypasses: usize, +} + +/// What one full compile reports for reuse recording. +#[derive(Debug)] +pub struct CommandOutcome<'py> { + pub exit_code: i64, + pub compile_cache_enabled: bool, + pub artifacts_written: bool, + pub dag_artifact_path: Bound<'py, PyAny>, +} + +/// The arguments of one compile input attachment; unset flags are `False`, other options `None`. +#[derive(Debug)] +pub struct InputsRequest { + pub discovered_inputs: Py, + pub adapter_context: Py, + pub selected_target: Py, + pub cli_vars: Py, + pub run_id: Py, + pub no_sql_validation: Py, + pub defer_model_sql_validation: Py, + pub external_sql_reference_resolver: Py, + pub resolved_connection: Py, + pub no_cache: Py, +} + +impl InputsRequest { + /// The request `build_compile_inputs` was called with. + pub(crate) fn read(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(Self { + discovered_inputs: arguments.value("discovered_inputs")?.unbind(), + adapter_context: arguments.value("adapter_context")?.unbind(), + selected_target: arguments.value("selected_target")?.unbind(), + cli_vars: arguments.value("cli_vars")?.unbind(), + run_id: arguments.value("run_id")?.unbind(), + no_sql_validation: arguments.value_or_false("no_sql_validation")?.unbind(), + defer_model_sql_validation: arguments + .value_or_false("defer_model_sql_validation")? + .unbind(), + external_sql_reference_resolver: arguments + .value("external_sql_reference_resolver")? + .unbind(), + resolved_connection: arguments.value("resolved_connection")?.unbind(), + no_cache: arguments.value_or_false("no_cache")?.unbind(), + }) + } +} + +/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. +#[derive(Debug, Clone)] +pub struct TargetSettingsValues { + pub(crate) name: Option, + pub(crate) database: Option, + pub(crate) schema: Option, + pub(crate) loader_schema: Option, +} + +impl TargetSettingsValues { + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. + pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let located = |value: &Option| -> PyResult> { + value + .as_deref() + .filter(|value| *value != PRESERVE_TARGET_VALUE) + .into_bound_py_any(py) + }; + let preserved = |value: &Option| -> Bound<'py, PyAny> { + PyBool::new(py, value.as_deref() == Some(PRESERVE_TARGET_VALUE)) + .to_owned() + .into_any() + }; + new_record_with( + py, + RecordId::TargetSettings, + &[ + ("name", self.name.clone().into_bound_py_any(py)?), + ("database", located(&self.database)?), + ("schema", located(&self.schema)?), + ("loader_schema", located(&self.loader_schema)?), + ("database_preserved", preserved(&self.database)), + ("schema_preserved", preserved(&self.schema)), + ], + ) + } + + /// The `ctx` a provider's `setup` receives in a command with these settings. + pub fn setup_context<'py>(&self, py: Python<'py>) -> PyResult> { + new_record_with( + py, + RecordId::ProviderSetupContext, + &[("target_config", self.record(py)?)], + ) + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/types.rs b/crates/sqlbuild-python-compile/src/compile_driver/types.rs new file mode 100644 index 0000000000..8e30752bcd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/types.rs @@ -0,0 +1,8 @@ +//! Public traits and type aliases of compile driver. + +use crate::compile_driver::models::CommandOutcome; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// One compile's outcome, input reads and artifact writes, before it is stored and printed. +pub type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/artifacts.rs b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/artifacts.rs new file mode 100644 index 0000000000..1d6b353dab --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/artifacts.rs @@ -0,0 +1,132 @@ +//! Write, clean and publish compiled artifacts natively, raising the errors Python raised. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::Receiver; +use std::sync::{Arc, Condvar, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyBytes; +use sqlbuild_cache::artifacts::errors::ArtifactError; +use sqlbuild_cache::artifacts::main::write_artifacts::write_artifacts; +use sqlbuild_cache::artifacts::models::WrittenArtifacts; + +pub(crate) type PublishedRow = (String, Vec<(String, String)>); + +/// One write group's `(path, contents)` artifacts and whether unchanged files are kept. +pub(crate) type ArtifactGroup = (Vec<(PathBuf, Vec)>, bool); + +pub(crate) const WRITER_THREAD: &str = "sqlbuild-artifact-writer"; +const WRITER_POLL: Duration = Duration::from_millis(50); + +/// Write each group in order until one fails or a stop is requested between groups. +pub(crate) fn write_groups( + groups: &Receiver, + stop: &AtomicBool, +) -> (Result<(), ArtifactError>, Duration) { + let mut elapsed: Duration = Duration::ZERO; + for (files, check_existing) in groups { + if stop.load(Ordering::SeqCst) { + break; + } + let started: Instant = Instant::now(); + let written: Result = + write_artifacts(&files, check_existing); + elapsed += started.elapsed(); + if let Err(error) = written { + return (Err(error), elapsed); + } + } + (Ok(()), elapsed) +} + +/// Whether the writer thread has returned, with a condition variable to wait on it. +#[derive(Debug, Default)] +pub(crate) struct WriterDone { + pub(crate) finished: Mutex, + pub(crate) changed: Condvar, +} + +/// Marks the writer done when the writer thread returns or unwinds. +pub(crate) struct DoneMarker(pub(crate) Arc); + +impl Drop for DoneMarker { + fn drop(&mut self) { + *self + .0 + .finished + .lock() + .unwrap_or_else(PoisonError::into_inner) = true; + self.0.changed.notify_all(); + } +} + +impl WriterDone { + /// Whether the writer returned within `timeout`. + pub(crate) fn finished_within(&self, timeout: Duration) -> bool { + let finished = self.finished.lock().unwrap_or_else(PoisonError::into_inner); + let (finished, _) = self + .changed + .wait_timeout_while(finished, timeout, |finished| !*finished) + .unwrap_or_else(PoisonError::into_inner); + *finished + } +} + +/// Wait without the GIL until the writer returns, checking for signals between waits. +pub(crate) fn wait_for_writer(py: Python<'_>, finished: &WriterDone) -> PyResult<()> { + while !py.detach(|| finished.finished_within(WRITER_POLL)) { + py.check_signals()?; + } + Ok(()) +} + +pub(crate) fn lossy(path: &std::path::Path) -> String { + path.to_string_lossy().into_owned() +} + +/// The exception Python's writer raises for the same failure. +pub(crate) fn artifact_error(py: Python<'_>, error: ArtifactError) -> PyErr { + match error { + ArtifactError::Io { path, error } => match error.raw_os_error() { + Some(code) => PyOSError::new_err((code, strerror(py, code), lossy(&path))), + None => PyOSError::new_err(error.to_string()), + }, + ArtifactError::Move { + source, + path, + error, + } => match error.raw_os_error() { + Some(code) => PyOSError::new_err(( + code, + strerror(py, code), + lossy(&source), + py.None(), + lossy(&path), + )), + None => PyOSError::new_err(error.to_string()), + }, + ArtifactError::ExistingNotUtf8 { path } => existing_decode_error(py, &path), + } +} + +fn strerror(py: Python<'_>, code: i32) -> String { + py.import("os") + .and_then(|os| os.call_method1("strerror", (code,))) + .and_then(|text| text.extract::()) + .unwrap_or_default() +} + +/// Decode the existing file in Python so the error is exactly the one `bytes.decode` raises. +fn existing_decode_error(py: Python<'_>, path: &PathBuf) -> PyErr { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) => return PyOSError::new_err(error.to_string()), + }; + match PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",)) { + Err(error) => error, + Ok(_) => PyOSError::new_err(format!("compiled artifact '{}' changed", path.display())), + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/mod.rs new file mode 100644 index 0000000000..c8646bee43 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod artifacts; +pub(crate) mod reuse; +pub(crate) mod runtime_target; diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/reuse.rs b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/reuse.rs new file mode 100644 index 0000000000..1561317fde --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/reuse.rs @@ -0,0 +1,99 @@ +//! Whole-project compile reuse on the shared native store, driven by the Python host's facts. + +use crate::compile_outputs::models::ReuseRulesPy; +use crate::compile_outputs::models::SettingsRow; +use crate::compile_outputs::types::ArtifactRow; +use std::collections::BTreeMap; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::PyValueError; +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::project_reuse::constants::{NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX}; +use sqlbuild_cache::project_reuse::models::{RecordedArtifact, ReuseRules, SettingsInputs}; +use sqlbuild_cache::project_snapshot::main::path_text::path_text; +use sqlbuild_cache::project_snapshot::models::SnapshotRules; + +pub(crate) type ReplayRow = (String, Vec, i64); +pub(crate) type LaunchIdentityRow = (String, BTreeMap, Vec); + +impl From for ReuseRules { + fn from(rules: ReuseRulesPy) -> Self { + Self { + snapshot: SnapshotRules { + excluded: rules.excluded, + excluded_root: rules.excluded_root, + presence_suffixes: rules.presence_suffixes, + output_files: rules.output_files, + }, + tracked_prefixes: rules.tracked_prefixes, + untracked_names: rules.untracked_names, + disabling_variables: rules.disabling_variables, + config_filenames: rules.config_filenames, + racy_window_ns: rules.racy_window_ns, + max_stored_entries: rules.max_stored_entries, + max_entry_bytes: rules.max_entry_bytes, + missing_environment_value: rules.missing_environment_value, + missing_file_digest: rules.missing_file_digest, + missing_path_mtime_ns: rules.missing_path_mtime_ns, + project_root_path_mtime_ns: rules.project_root_path_mtime_ns, + compiled_root: rules.compiled_root, + retired_directories: rules.retired_directories, + } + } +} + +pub(crate) fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("compile reuse attempt is poisoned") +} + +pub(crate) fn settings_inputs(row: SettingsRow) -> SettingsInputs { + let SettingsRow { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } = row; + SettingsInputs { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } +} + +pub(crate) fn recorded_artifact(artifact: ArtifactRow) -> RecordedArtifact { + match artifact { + (Some(digest), _, _) => RecordedArtifact::Written { digest }, + (None, size, mtime_ns) => RecordedArtifact::Kept { + size: size.unwrap_or_default(), + mtime_ns: mtime_ns.unwrap_or_default(), + }, + } +} + +pub(crate) fn native_digest(contents: &[u8]) -> String { + format!( + "{NATIVE_DIGEST_PREFIX}{}", + hex_digest(&bytes_digest(contents)) + ) +} + +/// The one store file kept for the selected target. +pub(crate) fn store_slot(directory: &Path, selected_target: Option<&OsStr>) -> PathBuf { + let label: String = selected_target.map_or_else(String::new, |target| { + format!("target:{}", path_text(target)) + }); + directory.join(format!( + "{}.{REUSE_STORE_SUFFIX}", + hex_digest(&bytes_digest(label.as_bytes())) + )) +} + +/// Host strings that may carry undecodable bytes, as exact text. +pub(crate) fn texts(values: &[OsString]) -> Vec { + values.iter().map(|value| path_text(value)).collect() +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/runtime_target.rs b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/runtime_target.rs new file mode 100644 index 0000000000..ab6181c6d9 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/_helpers/runtime_target.rs @@ -0,0 +1,123 @@ +//! SQL test artifact paths, and the executed SQL and Python check results written to `target/run`. + +use crate::compile_outputs::main::write_runtime_target::write_runtime_target; +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; +use sqlbuild_host::models::BuildExecutionResult; + +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::types::ReadResult; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +pub(crate) const RUN_DIR: &str = "run"; +pub(crate) const TESTS_DIR: &str = "tests"; +pub(crate) const CHECKS_DIR: &str = "checks"; +pub(crate) const PYTHON_CHECKS_FILE: &str = "python_checks.json"; +pub(crate) const PYTHON_CHECK_KIND: &str = "python_check"; +pub(crate) const JSON_INDENT: i64 = 2; + +/// One test's identity as the test runtime target matches entries to results. +#[derive(Hash, PartialEq, Eq)] +pub(crate) struct TestKey { + pub(crate) name: String, + pub(crate) source_path: Option, + pub(crate) block_index: Option, + pub(crate) case_name: Option, +} + +fn posix(path: &Bound<'_, PyAny>) -> PyResult> { + if path.is_none() { + return Ok(None); + } + path.call_method0("as_posix")?.extract().map(Some) +} + +pub(crate) fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> { + imported(py, "pathlib", "Path")?.call1((text,)) +} + +pub(crate) fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + Ok(TestKey { + name: native_field(value, name_field)?.extract()?, + source_path: posix(&native_field(value, "source_path")?)?, + block_index: native_field(value, "block_index")?.extract()?, + case_name: native_field(value, "case_name")?.extract()?, + }) +} + +pub(crate) fn path_facts( + test: &Bound<'_, PyAny>, + model_names: Vec, +) -> PyResult { + let TestKey { + name, + source_path, + block_index, + case_name, + } = test_key(test, "name")?; + Ok(SqlTestPathFacts { + name, + source_path, + block_index, + case_name, + model_names, + }) +} + +pub(crate) fn write_text(path: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + let py: Python<'_> = path.py(); + let mkdir: Bound<'_, PyDict> = PyDict::new(py); + mkdir.set_item("parents", true)?; + mkdir.set_item("exist_ok", true)?; + native_field(path, "parent")?.call_method("mkdir", (), Some(&mkdir))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("encoding", "utf-8")?; + path.call_method("write_text", (text,), Some(&options))?; + Ok(()) +} + +pub(crate) fn entry_paths( + entries: &Bound<'_, PyAny>, + path_of: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult> { + let mut paths: HashMap = HashMap::new(); + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + paths.insert(native_field(&entry, "name")?.extract()?, path_of(&entry)?); + } + Ok(paths) +} + +pub(crate) fn relative_posix(entry: &Bound<'_, PyAny>) -> PyResult { + native_field(entry, "relative_path")? + .call_method0("as_posix")? + .extract() +} + +pub(crate) fn check_status(result: &Bound<'_, PyAny>) -> PyResult<&'static str> { + if native_field(result, "passed")?.is_truthy()? { + return Ok("pass"); + } + if native_field(result, "warned")?.is_truthy()? { + return Ok("warn"); + } + Ok("fail") +} + +/// Write executed model and function lifecycle SQL under `target/run`. +#[pyfunction(name = "write_runtime_target")] +#[pyo3(signature = (*, target_dir, plan_output, result))] +pub(crate) fn write_runtime_target_binding( + target_dir: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + write_runtime_target( + target_dir, + plan_output, + &BuildExecutionResult::read(result)?, + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/artifact_digest.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/artifact_digest.rs new file mode 100644 index 0000000000..0624a9329f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/artifact_digest.rs @@ -0,0 +1,8 @@ +//! The native content digest of artifact bytes, as reuse verifies written artifacts. + +use crate::compile_outputs::_helpers::reuse::native_digest; + +/// The native content digest of artifact bytes, as reuse verifies written artifacts. +pub fn artifact_digest(contents: &[u8]) -> String { + native_digest(contents) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/check_compile_reuse.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/check_compile_reuse.rs new file mode 100644 index 0000000000..84bef11206 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/check_compile_reuse.rs @@ -0,0 +1,42 @@ +//! Check the stored compile for this invocation, replaying it on a hit. + +use crate::compile_outputs::_helpers::reuse::native_digest; +use crate::compile_outputs::_helpers::reuse::store_slot; +use crate::compile_outputs::_helpers::reuse::texts; +use crate::compile_outputs::models::NativeReuseAttempt; +use crate::compile_outputs::models::ReuseCheckPy; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; +use sqlbuild_cache::project_reuse::models::{HostIdentity, ReuseAttempt, ReuseCheck, ReuseRules}; +use sqlbuild_cache::project_snapshot::main::path_text::path_text; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::sync::Mutex; + +/// Check the stored compile for this invocation, replaying it on a hit. +#[pyfunction] +pub fn check_compile_reuse(py: Python<'_>, check: ReuseCheckPy) -> PyResult { + compiler_guard(|| { + let rules: ReuseRules = check.rules.into(); + let request: ReuseCheck<'_> = ReuseCheck { + project_dir: check.project_dir, + store_path: store_slot(&check.store_directory, check.selected_target.as_deref()), + identity: HostIdentity { + invocation_digest: native_digest(&check.invocation), + runtime: check + .runtime + .iter() + .map(|(name, value)| (name.clone(), path_text(value))) + .collect(), + search_path: texts(&check.search_path), + }, + bypass_requested: check.bypass_requested, + json_output: check.json_output, + rules: &rules, + }; + let attempt: ReuseAttempt = py.detach(|| check_reuse(request)); + Ok(NativeReuseAttempt { + attempt: Mutex::new(attempt), + }) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/compiled_sql_test_path.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/compiled_sql_test_path.rs new file mode 100644 index 0000000000..f42d2f8168 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/compiled_sql_test_path.rs @@ -0,0 +1,17 @@ +//! The target path of one compiled SQL test from its chain's model names. + +use crate::compile_outputs::_helpers::runtime_target::path_facts; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; +use std::path::PathBuf; + +/// The target path of one compiled SQL test from its chain's model names. +pub fn compiled_sql_test_path( + test: &Bound<'_, PyAny>, + model_names: Vec, +) -> PyResult { + Ok(PathBuf::from(sql_test_output_path(&path_facts( + test, + model_names, + )?))) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/mod.rs new file mode 100644 index 0000000000..e2a6efb650 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/mod.rs @@ -0,0 +1,13 @@ +pub mod artifact_digest; +pub mod check_compile_reuse; +pub mod compiled_sql_test_path; +pub mod planned_sql_test_path; +pub mod publish_staged_artifacts; +pub mod register_reuse; +pub mod register_runtime_target; +pub mod remove_stale_artifacts; +pub mod staged_artifact_files; +pub mod write_compiled_artifacts; +pub mod write_python_check_runtime_target; +pub mod write_runtime_target; +pub mod write_test_runtime_target; diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/planned_sql_test_path.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/planned_sql_test_path.rs new file mode 100644 index 0000000000..9f1c497a5e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/planned_sql_test_path.rs @@ -0,0 +1,19 @@ +//! The target path of one planned SQL test, relative to its artifact directory. + +use crate::compile_outputs::_helpers::runtime_target::path_facts; +use crate::compile_outputs::_helpers::runtime_target::path_object; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The target path of one planned SQL test, relative to its artifact directory. +pub fn planned_sql_test_path<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let mut model_names: Vec = Vec::new(); + for step in native_field(entry, "chain")?.try_iter()? { + model_names.push(native_field(&step?, "model_name")?.extract()?); + } + path_object( + entry.py(), + &sql_test_output_path(&path_facts(entry, model_names)?), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/publish_staged_artifacts.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/publish_staged_artifacts.rs new file mode 100644 index 0000000000..dcd572a00b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/publish_staged_artifacts.rs @@ -0,0 +1,37 @@ +//! Publish staged artifacts: `("changed", [])`, `("tree", [])` or `("files", [(source, path)])`. + +use crate::compile_outputs::_helpers::artifacts::PublishedRow; +use crate::compile_outputs::_helpers::artifacts::artifact_error; +use crate::compile_outputs::_helpers::artifacts::lossy; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_cache::artifacts::main::publish_staged::publish_staged; +use sqlbuild_cache::artifacts::models::Publication; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::collections::HashSet; +use std::path::PathBuf; + +/// Publish staged artifacts: `("changed", [])`, `("tree", [])` or `("files", [(source, path)])`. +pub fn publish_staged_artifacts( + py: Python<'_>, + staged_dir: PathBuf, + compiled_dir: PathBuf, + expected: Vec, +) -> PyResult { + compiler_guard(|| { + let expected: HashSet = expected.into_iter().collect(); + let publication: Publication = py + .detach(|| publish_staged(&staged_dir, &compiled_dir, &expected)) + .map_err(|error| artifact_error(py, error))?; + Ok(match publication { + Publication::StagedChanged => ("changed".to_owned(), Vec::new()), + Publication::MovedTree => ("tree".to_owned(), Vec::new()), + Publication::Files(files) => ( + "files".to_owned(), + files + .into_iter() + .map(|(source, path)| (lossy(&source), lossy(&path))) + .collect(), + ), + }) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/register_reuse.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/register_reuse.rs new file mode 100644 index 0000000000..f73c04e1b3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/register_reuse.rs @@ -0,0 +1,12 @@ +//! Register the reuse functions and classes. + +use crate::compile_outputs::main::check_compile_reuse::check_compile_reuse; +use crate::compile_outputs::models::NativeReuseAttempt; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the reuse functions and classes. +pub fn register_reuse(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(check_compile_reuse, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/register_runtime_target.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/register_runtime_target.rs new file mode 100644 index 0000000000..29d7591d45 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/register_runtime_target.rs @@ -0,0 +1,10 @@ +//! Register the runtime target functions and classes. + +use crate::compile_outputs::_helpers::runtime_target::write_runtime_target_binding; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the runtime target functions and classes. +pub fn register_runtime_target(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_runtime_target_binding, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/remove_stale_artifacts.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/remove_stale_artifacts.rs new file mode 100644 index 0000000000..cf4308fbed --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/remove_stale_artifacts.rs @@ -0,0 +1,28 @@ +//! Delete unmanaged compiled files and empty directories; returns how many files were removed. + +use crate::compile_outputs::_helpers::artifacts::artifact_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_cache::artifacts::main::remove_stale::remove_stale; +use sqlbuild_cache::artifacts::main::remove_stale_unless_unchanged::remove_stale_unless_unchanged; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::collections::HashSet; +use std::path::PathBuf; + +/// Delete unmanaged compiled files and empty directories; returns how many files were removed. +pub fn remove_stale_artifacts( + py: Python<'_>, + compiled_dir: PathBuf, + managed: Vec, + stamp: Option<(PathBuf, i64)>, +) -> PyResult { + compiler_guard(|| { + let managed: HashSet = managed.into_iter().collect(); + py.detach(|| match &stamp { + Some((stamp_path, racy_window_ns)) => { + remove_stale_unless_unchanged(&compiled_dir, &managed, stamp_path, *racy_window_ns) + } + None => remove_stale(&compiled_dir, &managed), + }) + .map_err(|error| artifact_error(py, error)) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/staged_artifact_files.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/staged_artifact_files.rs new file mode 100644 index 0000000000..48edaf3f59 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/staged_artifact_files.rs @@ -0,0 +1,15 @@ +//! Every staged artifact's path relative to `staged_dir`. + +use crate::compile_outputs::_helpers::artifacts::lossy; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_cache::artifacts::main::staged_files::staged_files; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::path::PathBuf; + +/// Every staged artifact's path relative to `staged_dir`. +pub fn staged_artifact_files(py: Python<'_>, staged_dir: PathBuf) -> PyResult> { + compiler_guard(|| { + let files: Vec = py.detach(|| staged_files(&staged_dir)); + Ok(files.iter().map(|path| lossy(path)).collect()) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/write_compiled_artifacts.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_compiled_artifacts.rs new file mode 100644 index 0000000000..b0d7acdc25 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_compiled_artifacts.rs @@ -0,0 +1,23 @@ +//! Write each `(path, contents)` artifact; returns how many were written and left unchanged. + +use crate::compile_outputs::_helpers::artifacts::artifact_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_cache::artifacts::errors::ArtifactError; +use sqlbuild_cache::artifacts::main::write_artifacts::write_artifacts; +use sqlbuild_cache::artifacts::models::WrittenArtifacts; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::path::PathBuf; + +/// Write each `(path, contents)` artifact; returns how many were written and left unchanged. +pub fn write_compiled_artifacts( + py: Python<'_>, + files: Vec<(PathBuf, Vec)>, + check_existing: bool, +) -> PyResult<(usize, usize)> { + compiler_guard(|| { + let outcome: Result = + py.detach(|| write_artifacts(&files, check_existing)); + let written: WrittenArtifacts = outcome.map_err(|error| artifact_error(py, error))?; + Ok((written.written, written.unchanged)) + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/write_python_check_runtime_target.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_python_check_runtime_target.rs new file mode 100644 index 0000000000..9b424ab48c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_python_check_runtime_target.rs @@ -0,0 +1,58 @@ +//! Write Python check results under `target/run/checks` when any ran. + +use crate::compile_outputs::_helpers::runtime_target::CHECKS_DIR; +use crate::compile_outputs::_helpers::runtime_target::JSON_INDENT; +use crate::compile_outputs::_helpers::runtime_target::PYTHON_CHECK_KIND; +use crate::compile_outputs::_helpers::runtime_target::PYTHON_CHECKS_FILE; +use crate::compile_outputs::_helpers::runtime_target::RUN_DIR; +use crate::compile_outputs::_helpers::runtime_target::check_status; +use crate::compile_outputs::_helpers::runtime_target::write_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Write Python check results under `target/run/checks` when any ran. +pub fn write_python_check_runtime_target( + target_dir: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let checks: Bound<'_, PyList> = PyList::empty(py); + for result in results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + let name: Bound<'_, PyAny> = native_field(&result, "node_name")?; + let check: Bound<'_, PyDict> = PyDict::new(py); + check.set_item("kind", PYTHON_CHECK_KIND)?; + check.set_item("name", &name)?; + check.set_item("display_name", &name)?; + check.set_item("check_id", format!("{PYTHON_CHECK_KIND}:{}", name.str()?))?; + check.set_item("status", check_status(&result)?)?; + check.set_item("passed", native_field(&result, "passed")?)?; + check.set_item( + "severity", + native_field(&result, "severity")?.getattr("value")?, + )?; + check.set_item("message", native_field(&result, "message")?)?; + check.set_item("error_message", native_field(&result, "error_message")?)?; + check.set_item("metadata", native_field(&result, "metadata")?)?; + checks.append(check)?; + } + if checks.is_empty() { + return Ok(()); + } + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("checks", checks)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("indent", JSON_INDENT)?; + let text: String = imported(py, "json", "dumps")? + .call((payload,), Some(&options))? + .extract()?; + write_text( + &target_dir + .div(RUN_DIR)? + .div(CHECKS_DIR)? + .div(PYTHON_CHECKS_FILE)?, + &format!("{text}\n"), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/write_runtime_target.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_runtime_target.rs new file mode 100644 index 0000000000..729b6a5984 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_runtime_target.rs @@ -0,0 +1,58 @@ +//! Write executed model and function lifecycle SQL under `target/run`. + +use crate::compile_outputs::_helpers::runtime_target::RUN_DIR; +use crate::compile_outputs::_helpers::runtime_target::entry_paths; +use crate::compile_outputs::_helpers::runtime_target::relative_posix; +use crate::compile_outputs::_helpers::runtime_target::write_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::target_paths::main::function_run_path::function_run_path; +use sqlbuild_analysis::target_paths::main::model_run_path::model_run_path; +use sqlbuild_analysis::target_paths::main::runtime_sql_text::runtime_sql_text; +use sqlbuild_host::models::{BuildExecutionResult, LifeCycleEvent, LifeCycleEventKind}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; + +/// Write executed model and function lifecycle SQL under `target/run`. +pub fn write_runtime_target( + target_dir: &Bound<'_, PyAny>, + plan: &Bound<'_, PyAny>, + result: &BuildExecutionResult>, +) -> PyResult<()> { + let run_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?; + let models: HashMap = + entry_paths(&native_field(plan, "model_entries")?, |entry| { + relative_posix(entry).map(|path| model_run_path(&path)) + })?; + let functions: HashMap = + entry_paths(&native_field(plan, "function_entries")?, |entry| { + Ok(function_run_path( + &relative_posix(entry)?, + &native_field(entry, "language")? + .getattr("value")? + .extract::()?, + )) + })?; + let models = result.model_results.iter().map(|model| { + ( + models.get(&model.model_name), + model.lifecycle_events.as_slice(), + ) + }); + let functions = result.function_results.iter().map(|function| { + ( + functions.get(&function.function_name), + function.lifecycle_events.as_slice(), + ) + }); + for (path, events) in models.chain(functions) { + let statements: Vec = events + .iter() + .filter(|event: &&LifeCycleEvent| event.kind == LifeCycleEventKind::Sql) + .map(|event| event.content.clone()) + .collect(); + if let Some(path) = path.filter(|_| !statements.is_empty()) { + write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/main/write_test_runtime_target.rs b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_test_runtime_target.rs new file mode 100644 index 0000000000..8839ac3b3a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/main/write_test_runtime_target.rs @@ -0,0 +1,47 @@ +//! Write the executed SQL unit-test comparison statements under `target/run/tests`. + +use crate::analysis::main::single_comparison_sql::single_comparison_sql; +use crate::compile_outputs::_helpers::runtime_target::RUN_DIR; +use crate::compile_outputs::_helpers::runtime_target::TESTS_DIR; +use crate::compile_outputs::_helpers::runtime_target::TestKey; +use crate::compile_outputs::_helpers::runtime_target::test_key; +use crate::compile_outputs::_helpers::runtime_target::write_text; +use crate::compile_outputs::main::planned_sql_test_path::planned_sql_test_path; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_host::models::SqlTestExecutionResult; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashSet; + +/// Write the executed SQL unit-test comparison statements under `target/run/tests`. +pub fn write_test_runtime_target( + (target_dir, adapter): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + plan: &Bound<'_, PyAny>, + results: &[SqlTestExecutionResult>], +) -> PyResult<()> { + let mut executed: HashSet = HashSet::new(); + for result in results { + executed.insert(TestKey { + name: result.test_name.clone(), + source_path: result.source_path.as_ref().map(|path| { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + }), + block_index: result.block_index, + case_name: result.case_name.clone(), + }); + } + let tests_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?.div(TESTS_DIR)?; + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !executed.contains(&test_key(&entry, "name")?) { + continue; + } + let sql: String = single_comparison_sql(&entry, adapter)?; + write_text( + &tests_dir.div(planned_sql_test_path(&entry)?)?, + &format!("{}\n", sql.trim_end_matches(python_isspace)), + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/mod.rs b/crates/sqlbuild-python-compile/src/compile_outputs/mod.rs new file mode 100644 index 0000000000..51032d4238 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/mod.rs @@ -0,0 +1,6 @@ +//! Compiled artifacts written, cleaned and published natively, whole-project compile reuse and run-target outputs. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/models.rs b/crates/sqlbuild-python-compile/src/compile_outputs/models.rs new file mode 100644 index 0000000000..7dddebc653 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/models.rs @@ -0,0 +1,354 @@ +//! Public records of compile outputs. + +use crate::compile_outputs::_helpers::artifacts::ArtifactGroup; +use crate::compile_outputs::_helpers::artifacts::DoneMarker; +use crate::compile_outputs::_helpers::artifacts::WRITER_THREAD; +use crate::compile_outputs::_helpers::artifacts::WriterDone; +use crate::compile_outputs::_helpers::artifacts::artifact_error; +use crate::compile_outputs::_helpers::artifacts::wait_for_writer; +use crate::compile_outputs::_helpers::artifacts::write_groups; +use crate::compile_outputs::_helpers::reuse::LaunchIdentityRow; +use crate::compile_outputs::_helpers::reuse::ReplayRow; +use crate::compile_outputs::_helpers::reuse::poisoned; +use crate::compile_outputs::_helpers::reuse::recorded_artifact; +use crate::compile_outputs::_helpers::reuse::settings_inputs; +use crate::compile_outputs::_helpers::reuse::texts; +use crate::compile_outputs::types::ArtifactRow; +use pyo3::FromPyObject; +use pyo3::IntoPyObject; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use sqlbuild_cache::artifacts::errors::ArtifactError; +use sqlbuild_cache::project_reuse::main::pending_digest_bytes::pending_digest_bytes; +use sqlbuild_cache::project_reuse::main::record_reuse::record_reuse; +use sqlbuild_cache::project_reuse::main::replay_stdout::replay_stdout; +use sqlbuild_cache::project_reuse::models::CompileRecord; +use sqlbuild_cache::project_reuse::models::HostIdentity; +use sqlbuild_cache::project_reuse::models::RecordOutcome; +use sqlbuild_cache::project_reuse::models::RecordResult; +use sqlbuild_cache::project_reuse::models::RecordedArtifact; +use sqlbuild_cache::project_reuse::models::ReuseAttempt; +use sqlbuild_cache::project_reuse::models::ReuseOutcome; +use sqlbuild_cache::project_reuse::models::ReuseRules; +use sqlbuild_cache::project_reuse::models::StoredOutput; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::ffi::OsString; +use std::panic::resume_unwind; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::mpsc::Sender; +use std::sync::mpsc::channel; +use std::thread::JoinHandle; +use std::time::Duration; + +/// Artifact groups written one after another on their own thread, stopping at the first error. +#[derive(Debug)] +pub struct BackgroundArtifactWriter { + pub(crate) groups: Sender, + /// Set to skip every group not yet started; the group being written still completes. + pub(crate) stop: Arc, + /// Marked once the writer returns, so a wait can check for signals meanwhile. + pub(crate) finished: Arc, + pub(crate) handle: JoinHandle<(Result<(), ArtifactError>, Duration)>, +} + +impl BackgroundArtifactWriter { + /// Start the writer thread, or `None` when no thread could start. + pub fn start() -> Option { + let (groups, received) = channel(); + let finished: Arc = Arc::new(WriterDone::default()); + let marker: DoneMarker = DoneMarker(Arc::clone(&finished)); + let stop: Arc = Arc::new(AtomicBool::new(false)); + let stopping: Arc = Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name(WRITER_THREAD.to_owned()) + .spawn(move || { + let _marker: DoneMarker = marker; + write_groups(&received, &stopping) + }); + match spawned { + Ok(handle) => Some(Self { + groups, + stop, + finished, + handle, + }), + Err(_) => None, + } + } + + /// Queue one group behind the groups already sent; a writer that stopped drops it. + pub fn send(&self, files: Vec<(PathBuf, Vec)>, check_existing: bool) { + let _ = self.groups.send((files, check_existing)); + } + + /// Wait without the GIL for every sent group; a signal stops it after the current group. + pub fn finish(self, py: Python<'_>) -> PyResult { + let Self { + groups, + stop, + finished, + handle, + } = self; + drop(groups); + if let Err(interrupt) = wait_for_writer(py, &finished) { + stop.store(true, Ordering::SeqCst); + let _ = py.detach(|| handle.join()); + return Err(interrupt); + } + compiler_guard(|| { + let joined = py.detach(|| handle.join()); + let (outcome, elapsed) = joined.unwrap_or_else(|payload| resume_unwind(payload)); + outcome.map_err(|error| artifact_error(py, error))?; + Ok(elapsed) + }) + } + + /// Stop after the group being written and wait for it without the GIL, ignoring the outcome. + pub fn stop(self, py: Python<'_>) { + let Self { + groups, + stop, + finished, + handle, + } = self; + stop.store(true, Ordering::SeqCst); + drop(groups); + drop(finished); + let _ = py.detach(|| handle.join()); + } +} + +/// Settings case sensitivity, names, prefixes, env files and secrets directories. +#[derive(Debug)] +pub struct SettingsRow { + pub case_sensitive: bool, + pub names: Vec, + pub prefixes: Vec, + pub env_files: Vec, + pub secrets_dirs: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for SettingsRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (case_sensitive, names, prefixes, env_files, secrets_dirs): ( + bool, + Vec, + Vec, + Vec, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + }) + } +} + +/// The host constants the walk, environment and store checks use. +#[derive(Debug, FromPyObject, IntoPyObject)] +#[pyo3(from_item_all)] +pub struct ReuseRulesPy { + pub excluded: Vec, + pub excluded_root: Vec, + pub presence_suffixes: Vec, + pub output_files: Vec<(u64, u64)>, + pub tracked_prefixes: Vec, + pub untracked_names: Vec, + pub disabling_variables: Vec<(String, String)>, + pub config_filenames: Vec, + pub racy_window_ns: i64, + pub max_stored_entries: usize, + pub max_entry_bytes: usize, + pub missing_environment_value: String, + pub missing_file_digest: String, + pub missing_path_mtime_ns: i64, + pub project_root_path_mtime_ns: i64, + pub compiled_root: PathBuf, + pub retired_directories: Vec, +} + +/// One reuse check: where the project and its store slot are, and the host's identity facts. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct ReuseCheckPy { + pub project_dir: PathBuf, + pub store_directory: PathBuf, + pub selected_target: Option, + pub invocation: Vec, + pub runtime: BTreeMap, + pub search_path: Vec, + pub bypass_requested: bool, + pub json_output: bool, + pub rules: ReuseRulesPy, +} + +/// What a full compile leaves for recording; `artifacts` maps a path to `(digest, size, mtime)`. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct CompileRecordPy { + pub stdout: Option, + pub stderr_lines: Vec, + pub exit_code: i64, + pub compile_cache_enabled: bool, + pub read_run_id: bool, + pub settings_inputs: Option>, + pub template_environment_names: Vec, + pub module_paths: Vec, + pub artifacts: HashMap, + pub artifacts_written: bool, + pub dag_path: Option, + pub json_output: bool, + pub rules: ReuseRulesPy, +} + +/// One invocation's native reuse state, from the check to the record. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeReuseAttempt { + pub(crate) attempt: Mutex, +} + +impl NativeReuseAttempt { + /// Run `body` on the native attempt. + pub fn with_attempt(&self, body: impl FnOnce(&ReuseAttempt) -> T) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(body(&attempt)) + } +} + +#[pymethods] +impl NativeReuseAttempt { + /// `"hit"`, `"miss"` or `"bypass"`. + pub(crate) fn outcome(&self) -> PyResult<&'static str> { + Ok(match self.attempt.lock().map_err(poisoned)?.outcome { + ReuseOutcome::Hit => "hit", + ReuseOutcome::Miss => "miss", + ReuseOutcome::Bypass => "bypass", + }) + } + + /// The store file holding this invocation's slot. + pub(crate) fn store_path(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(attempt.store_path.to_string_lossy().into_owned()) + } + + /// Whether a stored compile matched but could not be replayed. + pub(crate) fn replay_failed(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.replay_failed) + } + + /// How many project paths the check stamped. + pub(crate) fn snapshot_paths(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.snapshot.len()) + } + + /// Bytes recording would hash for files whose stamp moved. + pub(crate) fn pending_digest_bytes(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(pending_digest_bytes(&attempt)) + } + + /// On a hit, the stored report with these timings plus its stderr lines and exit code. + pub(crate) fn replay(&self, timings: Vec<(String, i64)>) -> PyResult> { + let attempt = self.attempt.lock().map_err(poisoned)?; + let Some(output) = attempt.replay.as_ref() else { + return Ok(None); + }; + let output: &StoredOutput = output; + Ok(replay_stdout(output, &timings) + .map(|stdout| (stdout, output.stderr_lines.clone(), output.exit_code))) + } + + /// The invocation digest, runtime and search path the check compared, for a launch record. + pub(crate) fn launch_identity(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + let identity: &HostIdentity = &attempt.identity; + Ok(( + identity.invocation_digest.clone(), + identity.runtime.clone(), + identity.search_path.clone(), + )) + } + + /// How many project files the check read for content digests. + pub(crate) fn digested_files(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.digested_files) + } + + /// Store this compile or clear the slot; a failure clears the slot and raises `OSError`. + pub fn record( + &self, + py: Python<'_>, + record: CompileRecordPy, + ) -> PyResult<(&'static str, usize)> { + compiler_guard(|| { + let rules: ReuseRules = record.rules.into(); + let mut compile: CompileRecord = CompileRecord { + stdout: record.stdout, + stderr_lines: record.stderr_lines, + exit_code: record.exit_code, + compile_cache_enabled: record.compile_cache_enabled, + read_run_id: record.read_run_id, + settings: record + .settings_inputs + .map(|rows| rows.into_iter().map(settings_inputs).collect()) + .ok_or_else(String::new), + template_environment_names: record.template_environment_names, + module_paths: texts(&record.module_paths), + artifacts: record + .artifacts + .into_iter() + .map(|(path, artifact)| (path, recorded_artifact(artifact))) + .collect(), + artifacts_written: record.artifacts_written, + dag_path: record.dag_path, + json_output: record.json_output, + image: sqlbuild_python_base::compile_context::main::delta::take_pending_image::take_pending_image(), + vouched_target_stamps: None, + }; + if let Some(merge) = + sqlbuild_python_base::compile_context::main::delta::take_record_merge::take_record_merge() + { + for stamp in &merge.kept { + compile + .artifacts + .entry(stamp.relative_path.clone()) + .or_insert(RecordedArtifact::Kept { + size: stamp.size, + mtime_ns: stamp.mtime_ns, + }); + } + compile.vouched_target_stamps = Some(merge.kept); + compile + .template_environment_names + .extend(merge.environment_names); + compile.template_environment_names.sort(); + compile.template_environment_names.dedup(); + compile.module_paths.extend(merge.module_paths); + } + let attempt = self.attempt.lock().map_err(poisoned)?; + let result: RecordResult = py + .detach(|| record_reuse(&attempt, &compile, &rules)) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.message))?; + let outcome: &'static str = match result.outcome { + RecordOutcome::NotAttempted => "not_attempted", + RecordOutcome::Stored => "stored", + RecordOutcome::Removed => "removed", + RecordOutcome::Skipped => "skipped", + }; + Ok((outcome, result.digested_files)) + }) + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_outputs/types.rs b/crates/sqlbuild-python-compile/src/compile_outputs/types.rs new file mode 100644 index 0000000000..3490588de3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_outputs/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of compile outputs. + +/// A written artifact's digest, or a kept artifact's size and mtime. +pub type ArtifactRow = (Option, Option, Option); diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/artifact.rs b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/artifact.rs new file mode 100644 index 0000000000..d513b1dbc4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/artifact.rs @@ -0,0 +1,23 @@ +//! `build_dag_json`: the static DAG artifact of a compiled project graph. + +use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; +use pyo3::prelude::{PyAnyMethods, PyErr}; +use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; +use sqlbuild_core::json::errors::JsonEmitError; + +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +pub(crate) fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { + match error { + DagArtifactError::LoaderDestination(message) => error_class(py, ErrorId::SpecConfig) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + DagArtifactError::MissingParameterType(name) => PyKeyError::new_err(name), + DagArtifactError::Unencodable(message) => PyTypeError::new_err(message), + DagArtifactError::Emit(JsonEmitError::NestingTooDeep) => { + PyRecursionError::new_err("maximum recursion depth exceeded") + } + DagArtifactError::Emit(error) => PyValueError::new_err(format!("{error:?}")), + } +} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/mod.rs new file mode 100644 index 0000000000..86cab0caef --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod artifact; +pub(crate) mod projection; +pub(crate) mod rows; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs rename to crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs index 95e2a446a7..4cc54cdc81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs +++ b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs @@ -2,9 +2,9 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyMapping, PySet, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::dag_artifact::main::dag_asset_plans::dag_asset_plans; use sqlbuild_analysis::dag_artifact::main::dag_asset_spec_kinds::dag_asset_spec_kinds; use sqlbuild_analysis::dag_artifact::main::dag_check_plans::dag_check_plans; @@ -15,7 +15,7 @@ use sqlbuild_analysis::dag_artifact::models::{ }; use sqlbuild_host::entries::models::ThirdPartyHost; -use crate::bindings::_helpers::dag_artifact::selection::selected_node_ids; +use crate::dag_artifact::_helpers::selection::selected_node_ids; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; @@ -144,7 +144,7 @@ fn materialization_type(node: &Bound<'_, PyAny>) -> PyResult { /// Dagster asset specs to build: each translated asset node, its upstream nodes and its kinds. #[pyfunction] #[pyo3(signature = (dag, /))] -fn dagster_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn dagster_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = dag.py(); let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; let upstream_nodes: Vec> = translated_nodes(&artifact)?; @@ -184,7 +184,7 @@ fn dagster_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult(dag: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn rivers_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = dag.py(); let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; let policy: AssetProjectionPolicy = AssetProjectionPolicy { @@ -219,7 +219,7 @@ fn check_description<'py>(check: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn dagster_check_plans<'py>( dag: &Bound<'py, PyAny>, include_scenarios: bool, only_scenarios: bool, @@ -258,7 +258,7 @@ fn dagster_check_plans<'py>( /// The translated nodes Dagster owns as assets among those `select` minus `exclude` matches. #[pyfunction] #[pyo3(signature = (dag, /, *, select, exclude = None))] -fn dagster_selected_asset_nodes<'py>( +pub(crate) fn dagster_selected_asset_nodes<'py>( dag: &Bound<'py, PyAny>, select: &str, exclude: Option<&str>, @@ -287,11 +287,3 @@ fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { kind: text(&mapping_get(node, "kind")?)?, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(dagster_asset_plans, module)?)?; - module.add_function(wrap_pyfunction!(dagster_check_plans, module)?)?; - module.add_function(wrap_pyfunction!(dagster_selected_asset_nodes, module)?)?; - module.add_function(wrap_pyfunction!(rivers_asset_plans, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/rows.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs rename to crates/sqlbuild-python-compile/src/dag_artifact/_helpers/rows.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/selection.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs rename to crates/sqlbuild-python-compile/src/dag_artifact/_helpers/selection.rs index f0bc584f2c..d0a521dfd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python-compile/src/dag_artifact/_helpers/selection.rs @@ -4,11 +4,9 @@ use sqlbuild_python_base::native_host::main::host_error::host_error; use std::collections::BTreeSet; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::dag_artifact::errors::DagSelectionError; use sqlbuild_analysis::dag_artifact::main::resolve_dag_selection::resolve_dag_selection as resolve; use sqlbuild_analysis::dag_artifact::models::{ @@ -130,7 +128,7 @@ fn raised(error_class: &Bound<'_, PyAny>, message: String, cause: Option) /// A SQLBuild DAG artifact from a mapping or JSON file path, validated, raising `error_class`. #[pyfunction] #[pyo3(signature = (dag, *, error_class))] -fn load_sqlbuild_dag<'py>( +pub(crate) fn load_sqlbuild_dag<'py>( dag: Bound<'py, PyAny>, error_class: &Bound<'py, PyAny>, ) -> PyResult> { @@ -202,8 +200,3 @@ fn load_sqlbuild_dag<'py>( } Ok(payload) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/main/build_dag_json.rs b/crates/sqlbuild-python-compile/src/dag_artifact/main/build_dag_json.rs new file mode 100644 index 0000000000..ecc15ef5d0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/main/build_dag_json.rs @@ -0,0 +1,37 @@ +//! The DAG JSON of `graph` (a `ProjectGraph`) and the optional discovered Python node graph. + +use crate::dag_artifact::_helpers::artifact::python_error; +use crate::dag_artifact::_helpers::rows::project_input; +use crate::dag_artifact::_helpers::rows::python_graph_input; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; +use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; + +/// The DAG JSON of `graph` (a `ProjectGraph`) and the optional discovered Python node graph. +#[pyfunction] +#[pyo3(signature = (*, graph, project_name, python_graph=None))] +pub fn build_dag_json( + graph: &Bound<'_, PyAny>, + project_name: &str, + python_graph: Option<&Bound<'_, PyAny>>, +) -> PyResult { + compiler_guard(|| { + let py = graph.py(); + let graph: &Bound<'_, CompiledProjectGraph> = graph.cast()?; + let project: DagProjectInput = project_input(graph.get().project(py))?; + let python: Option = python_graph + .filter(|python_graph| !python_graph.is_none()) + .map(python_graph_input) + .transpose()?; + build( + graph.get().native(py).get().graph(), + &project, + python.as_ref(), + project_name, + ) + .map_err(|error| python_error(error, py)) + }) +} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/main/mod.rs b/crates/sqlbuild-python-compile/src/dag_artifact/main/mod.rs new file mode 100644 index 0000000000..de6af656c6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/main/mod.rs @@ -0,0 +1,4 @@ +pub mod build_dag_json; +pub mod register_artifact; +pub mod register_projection; +pub mod register_selection; diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/main/register_artifact.rs b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_artifact.rs new file mode 100644 index 0000000000..d253af6403 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_artifact.rs @@ -0,0 +1,10 @@ +//! Register the artifact functions and classes. + +use crate::dag_artifact::main::build_dag_json::build_dag_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the artifact functions and classes. +pub fn register_artifact(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_dag_json, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/main/register_projection.rs b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_projection.rs new file mode 100644 index 0000000000..f7fb62346e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_projection.rs @@ -0,0 +1,17 @@ +//! Register the projection functions and classes. + +use crate::dag_artifact::_helpers::projection::dagster_asset_plans; +use crate::dag_artifact::_helpers::projection::dagster_check_plans; +use crate::dag_artifact::_helpers::projection::dagster_selected_asset_nodes; +use crate::dag_artifact::_helpers::projection::rivers_asset_plans; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the projection functions and classes. +pub fn register_projection(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dagster_asset_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_check_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_selected_asset_nodes, module)?)?; + module.add_function(wrap_pyfunction!(rivers_asset_plans, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/main/register_selection.rs b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_selection.rs new file mode 100644 index 0000000000..b906074c63 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/main/register_selection.rs @@ -0,0 +1,11 @@ +//! Register the selection functions and classes. + +use crate::dag_artifact::_helpers::selection::load_sqlbuild_dag; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the selection functions and classes. +pub fn register_selection(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/dag_artifact/mod.rs b/crates/sqlbuild-python-compile/src/dag_artifact/mod.rs new file mode 100644 index 0000000000..687c760a92 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/dag_artifact/mod.rs @@ -0,0 +1,4 @@ +//! The static DAG artifact of a compiled project graph, its orchestrator projections and selection. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-compile/src/frontier/_helpers/cache_directory.rs b/crates/sqlbuild-python-compile/src/frontier/_helpers/cache_directory.rs new file mode 100644 index 0000000000..0b27bca1d6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/_helpers/cache_directory.rs @@ -0,0 +1,13 @@ +//! The project-local compiler store root. + +use crate::frontier::main::cache_directory::cache_directory; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// Return the project's compiler store root. +#[pyfunction] +pub(crate) fn compiler_cache_directory<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + cache_directory(project_dir) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python-compile/src/frontier/_helpers/compile_frontier.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs rename to crates/sqlbuild-python-compile/src/frontier/_helpers/compile_frontier.rs index 770966753d..b7eeed9729 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs +++ b/crates/sqlbuild-python-compile/src/frontier/_helpers/compile_frontier.rs @@ -1,21 +1,20 @@ //! The single seam between the compiler stages and the native frontier. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::frontier::main::bridged::bridged; +use crate::frontier::models::FrontierStage; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::frontier::stage_capture::write_stage_capture; -use sqlbuild_python_base::boundary::main::call_with::call_with; +use crate::frontier::_helpers::stage_capture::write_stage_capture; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; /// Produce the frontier object for `until`, capturing it on request. #[pyfunction] #[pyo3(signature = (**options))] -fn compile_frontier<'py>( +pub(crate) fn compile_frontier<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -51,14 +50,6 @@ fn native_frontier<'py>( python_stage.call0() } -/// The compiler stages native callers produce through the frontier. -#[derive(Clone, Copy)] -pub(crate) enum FrontierStage { - DiscoveredProjectInputs, - CompileProjectInputs, - CompiledProject, -} - /// The `CompilerStage` member of `stage`. pub(crate) fn frontier_stage(py: Python<'_>, stage: FrontierStage) -> PyResult> { let member: &str = match stage { @@ -68,24 +59,3 @@ pub(crate) fn frontier_stage(py: Python<'_>, stage: FrontierStage) -> PyResult(stage: &Bound<'py, PyAny>) -> PyResult> { - call_with( - &host_entry(stage.py(), UserCodeHost::RunWithMacroBridge)?, - &[("stage", stage.clone())], - ) -} - -/// The frontier object `produced` for `stage`, captured on request. -pub(crate) fn produced<'py>( - stage: FrontierStage, - produced: Bound<'py, PyAny>, -) -> PyResult> { - write_stage_capture(&frontier_stage(produced.py(), stage)?, &produced)?; - Ok(produced) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compile_frontier, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/frontier/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/frontier/_helpers/mod.rs new file mode 100644 index 0000000000..330a9ea8a7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod cache_directory; +pub(crate) mod compile_frontier; +pub(crate) mod stage_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs rename to crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs index 4ac289aaa3..c70a591831 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs @@ -1,10 +1,9 @@ //! Stage captures: frontier objects written as canonical JSON so compiles compare stage by stage. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::frontier::constants::CAPTURE_DIR_ENV_VAR; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; -use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; use sqlbuild_cache::stage_capture::main::mask_capture_noise::mask_capture_noise as masked; use sqlbuild_cache::stage_capture::main::next_capture_number::next_capture_number; @@ -16,10 +15,9 @@ use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::frontier::models::CaptureEncoder; use sqlbuild_python_base::records::main::fields::field::field as native_field; -pub(crate) const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; -const CAPTURE_SUFFIX: &str = ".json"; -const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; -const PLAN_CAPTURE_NAME: &str = "plan_output"; +pub(crate) const CAPTURE_SUFFIX: &str = ".json"; +pub(crate) const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; +pub(crate) const PLAN_CAPTURE_NAME: &str = "plan_output"; /// The capture document of `value`, or the shared nodes written before encoding failed. fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result { @@ -36,7 +34,7 @@ fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result) -> PyResult { +pub(crate) fn capture_number(directory: &Bound<'_, PyAny>) -> PyResult { let directory: String = directory.str()?.extract()?; let names: Vec = match std::fs::read_dir(&directory) { Ok(entries) => entries @@ -71,26 +69,11 @@ pub(crate) fn write_stage_capture( write_capture_file(&path, value) } -/// Write the planner's `PlanOutput` when SQLBUILD_PLANNER_CAPTURE_DIR names a directory. -pub(crate) fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = plan_output.py(); - let directory: Bound<'_, PyAny> = - imported(py, "os", "environ")?.call_method1("get", (PLAN_CAPTURE_DIR_ENV_VAR,))?; - if !directory.is_truthy()? { - return Ok(()); - } - let number: u64 = capture_number(&directory)?; - let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? - .call1((directory,))? - .call_method1( - "__truediv__", - (format!("{number:03}-{PLAN_CAPTURE_NAME}{CAPTURE_SUFFIX}"),), - )?; - write_capture_file(&path, plan_output) -} - /// Write the capture document of `value` to `path`, creating its directory. -fn write_capture_file(path: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn write_capture_file( + path: &Bound<'_, PyAny>, + value: &Bound<'_, PyAny>, +) -> PyResult<()> { let py: Python<'_> = path.py(); call_with( &native_field(path, "parent")?.getattr("mkdir")?, @@ -132,23 +115,12 @@ fn write_capture_file(path: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>) -> PyRe /// Render a value as the JSON document a stage capture writes. #[pyfunction] -fn render_stage_capture(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn render_stage_capture(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { capture_text(py, value).map_err(|(_, error)| error) } /// Mask invocation ids, native store suffixes and refactor staging folders of a compile. #[pyfunction] -fn mask_capture_noise(text: &str) -> String { +pub(crate) fn mask_capture_noise(text: &str) -> String { masked(text) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py: Python<'_> = module.py(); - let omitted: Bound<'_, PyDict> = PyDict::new(py); - for (type_name, names) in OMITTED_ATTRIBUTES { - omitted.set_item(type_name, PyFrozenSet::new(py, names.iter())?)?; - } - module.add("STAGE_CAPTURE_OMITTED_ATTRIBUTES", omitted)?; - module.add_function(wrap_pyfunction!(render_stage_capture, module)?)?; - module.add_function(wrap_pyfunction!(mask_capture_noise, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/frontier/constants.rs b/crates/sqlbuild-python-compile/src/frontier/constants.rs new file mode 100644 index 0000000000..017f844ecd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/constants.rs @@ -0,0 +1,13 @@ +//! Public constants of frontier. + +pub const CACHE_DIRECTORY_NAME: &str = "cache"; + +pub const COMPILER_CACHE_DIRECTORY_NAME: &str = "compiler"; + +/// The namespace suffix of the current native compiler store. +pub const NATIVE_CACHE_NAMESPACE_SUFFIX: &str = "-native-v1"; + +/// The namespace suffixes of compiler store roots older releases wrote. +pub const RETIRED_CACHE_NAMESPACE_SUFFIXES: [&str; 3] = ["", "-native-v1", "-native-preview-v1"]; + +pub const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; diff --git a/crates/sqlbuild-python-compile/src/frontier/main/bridged.rs b/crates/sqlbuild-python-compile/src/frontier/main/bridged.rs new file mode 100644 index 0000000000..0fea5dda20 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/bridged.rs @@ -0,0 +1,14 @@ +//! `stage()` run inside the macro bridge user macros render through. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// `stage()` run inside the macro bridge user macros render through. +pub fn bridged<'py>(stage: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &host_entry(stage.py(), UserCodeHost::RunWithMacroBridge)?, + &[("stage", stage.clone())], + ) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/cache_directory.rs b/crates/sqlbuild-python-compile/src/frontier/main/cache_directory.rs new file mode 100644 index 0000000000..192c79d131 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/cache_directory.rs @@ -0,0 +1,16 @@ +//! `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. + +use crate::compile_command::constants::TARGET_DIRECTORY_NAME; +use crate::frontier::constants::CACHE_DIRECTORY_NAME; +use crate::frontier::constants::COMPILER_CACHE_DIRECTORY_NAME; +use crate::frontier::constants::NATIVE_CACHE_NAMESPACE_SUFFIX; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. +pub fn cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let store: String = format!("{COMPILER_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}"); + project_dir + .div(TARGET_DIRECTORY_NAME)? + .div(CACHE_DIRECTORY_NAME)? + .div(store) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/mod.rs b/crates/sqlbuild-python-compile/src/frontier/main/mod.rs new file mode 100644 index 0000000000..97653a341f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/mod.rs @@ -0,0 +1,7 @@ +pub mod bridged; +pub mod cache_directory; +pub mod produced; +pub mod register_cache_directory; +pub mod register_compile_frontier; +pub mod register_stage_capture; +pub mod write_plan_capture; diff --git a/crates/sqlbuild-python-compile/src/frontier/main/produced.rs b/crates/sqlbuild-python-compile/src/frontier/main/produced.rs new file mode 100644 index 0000000000..2522fdc253 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/produced.rs @@ -0,0 +1,15 @@ +//! The frontier object `produced` for `stage`, captured on request. + +use crate::frontier::_helpers::compile_frontier::frontier_stage; +use crate::frontier::_helpers::stage_capture::write_stage_capture; +use crate::frontier::models::FrontierStage; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The frontier object `produced` for `stage`, captured on request. +pub fn produced<'py>( + stage: FrontierStage, + produced: Bound<'py, PyAny>, +) -> PyResult> { + write_stage_capture(&frontier_stage(produced.py(), stage)?, &produced)?; + Ok(produced) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/register_cache_directory.rs b/crates/sqlbuild-python-compile/src/frontier/main/register_cache_directory.rs new file mode 100644 index 0000000000..7b2cd5d8ae --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/register_cache_directory.rs @@ -0,0 +1,10 @@ +//! Register the cache directory functions and classes. + +use crate::frontier::_helpers::cache_directory::compiler_cache_directory; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cache directory functions and classes. +pub fn register_cache_directory(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compiler_cache_directory, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/register_compile_frontier.rs b/crates/sqlbuild-python-compile/src/frontier/main/register_compile_frontier.rs new file mode 100644 index 0000000000..da98f9b83d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/register_compile_frontier.rs @@ -0,0 +1,10 @@ +//! Register the compile frontier functions and classes. + +use crate::frontier::_helpers::compile_frontier::compile_frontier; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the compile frontier functions and classes. +pub fn register_compile_frontier(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compile_frontier, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/register_stage_capture.rs b/crates/sqlbuild-python-compile/src/frontier/main/register_stage_capture.rs new file mode 100644 index 0000000000..04f7ba3018 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/register_stage_capture.rs @@ -0,0 +1,20 @@ +//! Register the stage capture functions and classes. + +use crate::frontier::_helpers::stage_capture::mask_capture_noise; +use crate::frontier::_helpers::stage_capture::render_stage_capture; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::wrap_pyfunction; +use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; + +/// Register the stage capture functions and classes. +pub fn register_stage_capture(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let omitted: Bound<'_, PyDict> = PyDict::new(py); + for (type_name, names) in OMITTED_ATTRIBUTES { + omitted.set_item(type_name, PyFrozenSet::new(py, names.iter())?)?; + } + module.add("STAGE_CAPTURE_OMITTED_ATTRIBUTES", omitted)?; + module.add_function(wrap_pyfunction!(render_stage_capture, module)?)?; + module.add_function(wrap_pyfunction!(mask_capture_noise, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs b/crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs new file mode 100644 index 0000000000..ebbe870469 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs @@ -0,0 +1,27 @@ +//! Write the planner's `PlanOutput` when SQLBUILD_PLANNER_CAPTURE_DIR names a directory. + +use crate::frontier::_helpers::stage_capture::CAPTURE_SUFFIX; +use crate::frontier::_helpers::stage_capture::PLAN_CAPTURE_DIR_ENV_VAR; +use crate::frontier::_helpers::stage_capture::PLAN_CAPTURE_NAME; +use crate::frontier::_helpers::stage_capture::capture_number; +use crate::frontier::_helpers::stage_capture::write_capture_file; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Write the planner's `PlanOutput` when SQLBUILD_PLANNER_CAPTURE_DIR names a directory. +pub fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = plan_output.py(); + let directory: Bound<'_, PyAny> = + imported(py, "os", "environ")?.call_method1("get", (PLAN_CAPTURE_DIR_ENV_VAR,))?; + if !directory.is_truthy()? { + return Ok(()); + } + let number: u64 = capture_number(&directory)?; + let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? + .call1((directory,))? + .call_method1( + "__truediv__", + (format!("{number:03}-{PLAN_CAPTURE_NAME}{CAPTURE_SUFFIX}"),), + )?; + write_capture_file(&path, plan_output) +} diff --git a/crates/sqlbuild-python-compile/src/frontier/mod.rs b/crates/sqlbuild-python-compile/src/frontier/mod.rs new file mode 100644 index 0000000000..6fd11cc718 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/mod.rs @@ -0,0 +1,6 @@ +//! The compiler store root, the seam between the compiler stages and the native frontier, and stage captures. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/frontier/models.rs b/crates/sqlbuild-python-compile/src/frontier/models.rs new file mode 100644 index 0000000000..2adc99040e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/frontier/models.rs @@ -0,0 +1,9 @@ +//! Public records of frontier. + +/// The compiler stages native callers produce through the frontier. +#[derive(Debug, Clone, Copy)] +pub enum FrontierStage { + DiscoveredProjectInputs, + CompileProjectInputs, + CompiledProject, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python-compile/src/hand_built/_helpers/compiled_model.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs rename to crates/sqlbuild-python-compile/src/hand_built/_helpers/compiled_model.rs index 9a364b646f..6367040f2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python-compile/src/hand_built/_helpers/compiled_model.rs @@ -1,10 +1,10 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::analysis::binding_locations::{ - ExpansionRow, LocatedRow, locate_model_binding_diagnostics, -}; -use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; -use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; +use crate::analysis::main::locate_model_binding_diagnostics::locate_model_binding_diagnostics; +use crate::analysis::models::LocatedRow; +use crate::analysis::types::ExpansionRow; +use crate::analysis_session::main::dynamic_contract_proof::dynamic_contract_proof; +use crate::analysis_session::main::session_outcome_analysis::session_outcome_analysis; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; @@ -17,7 +17,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::analysis::models::Positional; -type Object<'py> = Bound<'py, PyAny>; +pub(crate) type Object<'py> = Bound<'py, PyAny>; /// The compiled model of `model_input` from its session outcome, fallback rows and proof. pub(crate) fn compiled_model<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/mod.rs b/crates/sqlbuild-python-compile/src/hand_built/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/hand_built/mod.rs rename to crates/sqlbuild-python-compile/src/hand_built/_helpers/mod.rs diff --git a/crates/sqlbuild-python-compile/src/hand_built/main/compiled_model.rs b/crates/sqlbuild-python-compile/src/hand_built/main/compiled_model.rs new file mode 100644 index 0000000000..2c425fcb07 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/hand_built/main/compiled_model.rs @@ -0,0 +1,16 @@ +//! The compiled model of `model_input` from its session outcome, fallback rows and proof. + +use crate::hand_built::_helpers::compiled_model; +use crate::hand_built::_helpers::compiled_model::Object; +use pyo3::prelude::PyResult; + +/// The compiled model of `model_input` from its session outcome, fallback rows and proof. +#[inline] +pub fn compiled_model<'py>( + model_input: &Object<'py>, + deps: &Object<'py>, + analysis: &Object<'py>, + macro_deps: &Object<'py>, +) -> PyResult> { + compiled_model::compiled_model(model_input, deps, analysis, macro_deps) +} diff --git a/crates/sqlbuild-python-compile/src/hand_built/main/mod.rs b/crates/sqlbuild-python-compile/src/hand_built/main/mod.rs new file mode 100644 index 0000000000..260df91b6c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/hand_built/main/mod.rs @@ -0,0 +1 @@ +pub mod compiled_model; diff --git a/crates/sqlbuild-python-compile/src/hand_built/mod.rs b/crates/sqlbuild-python-compile/src/hand_built/mod.rs new file mode 100644 index 0000000000..c05575318c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/hand_built/mod.rs @@ -0,0 +1,4 @@ +//! Compiled models for model inputs built outside the native compile-input stage. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/mod.rs new file mode 100644 index 0000000000..9e5e1830a5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod model; +pub(crate) mod model_loop; +pub(crate) mod resources; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/mod.rs new file mode 100644 index 0000000000..4380bcf139 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod model_loop; +pub(crate) mod model_loop_calls; +pub(crate) mod model_loop_config; +pub(crate) mod model_loop_contracts; +pub(crate) mod model_loop_external; +pub(crate) mod model_loop_header; +pub(crate) mod model_loop_hooks; +pub(crate) mod model_loop_host; +pub(crate) mod model_loop_inputs; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs new file mode 100644 index 0000000000..ced7a517f7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs @@ -0,0 +1,352 @@ +//! The compile-input model loop: each model's steps run in the attachment's order, natively. + +use crate::input_loops::models::LoopSwitches; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::compiled_project::models::SplicedSpan; +use sqlbuild_analysis::sql_macros::main::moved_typed_calls::moved_typed_calls; +use sqlbuild_analysis::sql_macros::main::spliced_typed_calls::spliced_typed_calls; +use sqlbuild_analysis::sql_macros::models::{TextSplice, TypedCall}; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::input_loops::_helpers::model::model_loop_hooks::SqlHookClasses; +use crate::input_loops::_helpers::model::model_loop_host::DeclarationScan; +use crate::input_loops::_helpers::model::model_loop_host::InterpolationRow; +use crate::input_loops::_helpers::model::model_loop_host::ModelLoopRow; +use crate::input_loops::_helpers::model::model_loop_host::RecordClasses; +use crate::input_loops::_helpers::model_loop::model_loop_macros::MacroClasses; +use crate::input_loops::_helpers::model_loop::model_loop_python_hooks::HookClasses; +use crate::input_loops::_helpers::model_loop::model_loop_python_hooks::HookSignatures; +use crate::input_loops::_helpers::model_loop::model_loop_visibility::ModelVisibility; +use crate::input_loops::_helpers::model_loop::model_loop_visibility::VisibilityClasses; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; +use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; +use sqlbuild_python_front::model_config::models::ConfigClasses; +use sqlbuild_python_front::model_config::models::ContractClasses; +use sqlbuild_python_front::render::models::ConstantRendering; +use sqlbuild_python_front::render::types::SpanRow; +use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; +use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_batch; +use sqlbuild_python_front::sqltext::models::InterpolationRow as SqlInterpolationRow; +use sqlbuild_python_front::sqltext::models::InterpolationSources; + +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const CURSOR_KEY: &str = "cursor"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const VALUES_ATTRIBUTE: &str = "values"; +pub(crate) const RETENTION_ATTRIBUTE: &str = "time_travel_retention"; +pub(crate) const UNMANAGED_ATTRIBUTE: &str = "unmanaged"; +pub(crate) const TABLE_TYPE_ATTRIBUTE: &str = "table_type"; +pub(crate) const DECLARED_ATTRIBUTE: &str = "declared"; +pub(crate) const VALIDATE_METHOD: &str = "validate"; +const HELP_ARGUMENT: &str = "help"; +const CODE_ARGUMENT: &str = "code"; +pub(crate) const ANALYSIS_CONFLICT: &str = + "MODEL sql_analysis conflicts with legacy sql_validation"; +pub(crate) const ANALYSIS_CONFLICT_CODE: &str = "P003"; +const SQL_HOOKS_KEY: &str = "sql_hooks"; +const HOOK_FILES_KEY: &str = "hook_files"; +const DEFINITIONS_KEY: &str = "definitions"; +const COMPILE_ERROR_KEY: &str = "compile_error"; + +/// The native validator, header contract classes, and the loop's Python classes. +pub(crate) type LoopChecks = (Py, Py, Py); + +/// `(analysis switches, project config path, the adapter's analysis dialect)`. +pub(crate) type LoopSettingsInput = (LoopSwitches, String, Option); + +/// The Python classes and error factory the loop builds usages, hooks and errors with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct LoopClasses<'py> { + pub(crate) usage_record: Bound<'py, PyAny>, + pub(crate) compile_error: Bound<'py, PyAny>, + pub(crate) sql_hook_entry: Bound<'py, PyAny>, + /// Discovered audit factories by name. + pub(crate) audit_factories: Bound<'py, PyDict>, + /// `UsageKind.GENERATED`, for usages an enum-typed column generates. + pub(crate) generated_usage: Bound<'py, PyAny>, + /// `ContractPolicy.ENFORCED`. + pub(crate) contract_enforced: Bound<'py, PyAny>, + /// Public reusable model schemas by name. + pub(crate) model_schemas: Bound<'py, PyDict>, + /// Each indexed schema's and Python hook's declaration record, by `(kind, name)`. + pub(crate) named_records: Bound<'py, PyDict>, + /// Project enums by name, as a schema's column types resolve them. + pub(crate) schema_enums: Bound<'py, PyDict>, + /// `CompilerDiagnostic` and `report_compile_diagnostic`, for S006. + pub(crate) compiler_diagnostic: Bound<'py, PyAny>, + pub(crate) report_diagnostic: Bound<'py, PyAny>, + /// `pathlib.Path`. + pub(crate) path_class: Bound<'py, PyAny>, + /// The dbt manifest's `ExternalSqlReferenceResolver`, or `None`. + pub(crate) external_resolver: Bound<'py, PyAny>, + /// `SqlReferenceKind`. + pub(crate) reference_kind: Bound<'py, PyAny>, + /// What a model's effective config is built from. + pub(crate) config: ConfigClasses<'py>, + /// What a model's visible declarations are resolved from. + pub(crate) visibility: VisibilityClasses<'py>, + /// What a model's Python hooks are checked against. + pub(crate) hooks: HookClasses<'py>, + /// What each model's SQL hooks are expanded with. + pub(crate) sql_hooks: SqlHookClasses<'py>, + /// What each model's macro calls are expanded with. + pub(crate) macros: MacroClasses<'py>, + /// `sql_analysis_opt_out_help(model_owned=True)`, the help of a model's syntax error. + pub(crate) syntax_help: String, + /// The same help for SQL a project's other resources own. + pub(crate) source_syntax_help: String, + /// `InvalidScopePathError`. + pub(crate) scope_path_error: Bound<'py, PyAny>, + /// What each finished model's input record is built with. + pub(crate) record: RecordClasses<'py>, + /// What model input views project their attributes with. + pub(crate) input_view: Bound<'py, PyAny>, +} + +/// What one run reads beside each model: the host, Python classes and header contract classes. +pub(crate) struct LoopContext<'a, 'py> { + pub(crate) host: &'a Bound<'py, ConstantRendering>, + pub(crate) classes: &'a LoopClasses<'py>, + pub(crate) contracts: &'a ContractClasses<'py>, + /// The lexical rules the project's SQL is authored in. + pub(crate) syntax: &'a LexicalSyntax, +} + +/// The declarations a model sees and the identity its usages are recorded for. +pub(crate) struct VisibleDeclarations<'py> { + pub(crate) declarations: Bound<'py, PyAny>, + pub(crate) consumer: Bound<'py, PyAny>, +} + +/// What the model loop knows about a model before finishing it. +pub(crate) struct LoopModel<'a, 'py> { + pub(crate) row: &'a ModelLoopRow, + pub(crate) config: Bound<'py, PyAny>, + pub(crate) visible: &'a VisibleDeclarations<'py>, + pub(crate) signatures: &'a HookSignatures, + pub(crate) visibility: &'a ModelVisibility<'py>, +} + +/// Replace the SQL hook classes' discovered `hook_files` with their `definitions` by name. +pub(crate) fn index_sql_hook_definitions(loop_classes: &Bound<'_, PyDict>) -> PyResult<()> { + let py: Python<'_> = loop_classes.py(); + let hooks: Bound<'_, PyDict> = loop_classes + .as_any() + .get_item(SQL_HOOKS_KEY)? + .downcast_into()?; + let definitions: Bound<'_, PyDict> = PyDict::new(py); + for hook_file in hooks.as_any().get_item(HOOK_FILES_KEY)?.try_iter()? { + let hook_file: Bound<'_, PyAny> = hook_file?; + let name: Bound<'_, PyAny> = native_field(&hook_file, "name")?; + if let Some(existing) = definitions.get_item(&name)? { + let message: String = format!( + "Duplicate SQL hook name '{}' found in {} and {}", + name.str()?, + native_field(&existing, "relative_path")?.str()?, + native_field(&hook_file, "relative_path")?.str()? + ); + let error: Bound<'_, PyAny> = loop_classes + .as_any() + .get_item(COMPILE_ERROR_KEY)? + .call1((message,))?; + return Err(PyErr::from_value(error)); + } + definitions.set_item(name, hook_file)?; + } + hooks.set_item(DEFINITIONS_KEY, definitions) +} + +/// Each model's `@@`-interpolated query and declaration scan, all interpolated before any runs. +pub(crate) fn prepared_rows<'py>( + py: Python<'py>, + (classes, syntax): (&LoopClasses<'py>, &LexicalSyntax), + model_files: Vec>, +) -> PyResult> { + let hooks = &classes.sql_hooks; + let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); + let mut paths: Vec = Vec::with_capacity(model_files.len()); + for model_file in &model_files { + let record: &NativeModelFile = model_file.get(); + let texts: ModelFileTexts = record.path_texts(py)?; + sqls.push((record.query_text().to_string(), texts.file_path.clone())); + paths.push(texts); + } + let interpolated: Vec = interpolation_batch( + InterpolationSources( + hooks.effective_vars.clone(), + hooks.environment.clone(), + None, + hooks.render_project_var.clone(), + Some(syntax.clone()), + ), + &sqls, + )?; + let mut rows: Vec = Vec::with_capacity(model_files.len()); + for (((model_file, paths), (authored, _)), interpolated) in model_files + .into_iter() + .zip(paths) + .zip(sqls) + .zip(interpolated) + { + let SqlInterpolationRow { + sql: rendered, + spans, + reads, + error, + } = interpolated; + let sql: String = rendered.unwrap_or(authored); + let scan: Option = error + .is_none() + .then(|| declaration_scan(&sql, Some(syntax))); + let reads: Vec<(String, String)> = reads + .into_iter() + .map(|(kind, name)| (kind.to_owned(), name)) + .collect(); + let interpolation: InterpolationRow = InterpolationRow { + sql, + reads, + error, + spans, + }; + rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); + } + Ok(rows) +} + +/// `_raise_validation_outcome`: nothing, the validator's error, or a rejected reference's error. +pub(crate) fn validation_outcome( + outcome: &Bound<'_, PyAny>, + external: Vec>, + values: &Bound<'_, PyDict>, +) -> PyResult<()> { + if outcome.is_none() { + return Ok(()); + } + if let Ok(index) = outcome.extract::() + && let Some(error) = external.into_iter().nth(index).flatten() + { + return Err(error); + } + Err(config_error(outcome, Some(values))) +} + +/// `applied_interpolation`: record the interpolation's template reads, then its SQL or error. +pub(crate) fn applied_interpolation( + context: &LoopContext<'_, '_>, + interpolation: &InterpolationRow, +) -> PyResult { + let InterpolationRow { + sql, reads, error, .. + } = interpolation; + record_reads(context, reads)?; + match error { + Some(message) => Err(compile_error(context.classes, message, None)), + None => Ok(sql.clone()), + } +} + +/// Record interpolation's environment and context reads for compile reuse. +pub(crate) fn record_reads( + context: &LoopContext<'_, '_>, + reads: &[(String, String)], +) -> PyResult<()> { + if !reads.is_empty() { + let py: Python<'_> = context.host.py(); + context + .classes + .config + .record_template_reads + .call1((PyTuple::new(py, reads)?,))?; + } + Ok(()) +} + +/// `CompileInputError(message, code=...)`, raised as the attachment raises it. +pub(crate) fn compile_error(classes: &LoopClasses<'_>, message: &str, code: Option<&str>) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); + if let Some(code) = code { + keywords.set_item(CODE_ARGUMENT, code)?; + } + classes.compile_error.call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// `CompileInputError(message, help=help)`. +pub(crate) fn helped_compile_error( + classes: &LoopClasses<'_>, + message: &str, + help: Option<&str>, +) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); + keywords.set_item(HELP_ARGUMENT, help)?; + classes.compile_error.call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The Python exception a native config error raises as, given the values it names. +pub(crate) fn config_error(error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyDict>>) -> PyErr { + raised_config_exception(error, values.map(Bound::as_any)) +} + +/// The spliced SQL's expression macro calls moved through macro expansion into the query's bytes. +pub(crate) fn query_typed_calls( + calls: Vec, + (spliced, expanded, query): (&str, &str, &str), + macro_spans: &[SpanRow], +) -> Vec { + if calls.is_empty() { + return calls; + } + let calls: Vec = if macro_spans.is_empty() { + calls + } else { + spliced_typed_calls(calls, (spliced, expanded), &spliced_spans(macro_spans)) + }; + if expanded == query { + return calls; + } + let Ok((canonical, spans)) = + replace_intrinsics_with_spans(expanded, "", |name| format!("{name}()")) + else { + return Vec::new(); + }; + if canonical != query { + return Vec::new(); + } + let splices: Vec = spans + .into_iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + moved_typed_calls(calls, &splices) +} + +/// Span rows as the spliced spans the scan-fault locator maps through. +pub(crate) fn spliced_spans(rows: &[SpanRow]) -> Vec { + rows.iter() + .map(|row| SplicedSpan { + source_start: row.source_start, + source_end: row.source_end, + output_start: row.output_start, + output_end: row.output_end, + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_calls.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_calls.rs index 80467fdb25..c58109a26a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_calls.rs @@ -14,7 +14,8 @@ use sqlbuild_sqltext::sql_references::models::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; const MODEL_RESOURCE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_config.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_config.rs index 76248a4473..55af96724a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_config.rs @@ -13,7 +13,8 @@ use sqlbuild_model_config::config_presence::main::contains_template::contains_te use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_config_value; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, config_error}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::config_error; use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; use sqlbuild_python_front::model_config::main::config::config_fields::config_fields; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs index b1e5f7f3f0..af92dbcadd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs @@ -7,8 +7,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::input_loops::model_loop::LoopContext; -use crate::bindings::_helpers::input_loops::model_loop_schema::DeclaredColumns; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model_loop::model_loop_schema::DeclaredColumns; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_external.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_external.rs index fc19a93486..0e276ba70e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_external.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyDict; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; use sqlbuild_project::models::ReferenceRow; const DBT_REF_KIND: &str = "dbt_ref"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_header.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_header.rs index c342a1607f..9bda143277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_header.rs @@ -17,16 +17,17 @@ use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_model_config::model_validation::models::ColumnMigration; use sqlbuild_model_config::types::NodeKind; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, compile_error, config_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_contracts::{ - EnumColumns, enum_columns, factory_audits, -}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::present; -use crate::bindings::_helpers::input_loops::model_loop_schema::{ - DeclaredColumns, ModelSchema, merged_columns, schema_enums, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::config_error; +use crate::input_loops::_helpers::model::model_loop_contracts::EnumColumns; +use crate::input_loops::_helpers::model::model_loop_contracts::enum_columns; +use crate::input_loops::_helpers::model::model_loop_contracts::factory_audits; +use crate::input_loops::_helpers::model::model_loop_inputs::present; +use crate::input_loops::_helpers::model_loop::model_loop_schema::DeclaredColumns; +use crate::input_loops::_helpers::model_loop::model_loop_schema::ModelSchema; +use crate::input_loops::_helpers::model_loop::model_loop_schema::merged_columns; +use crate::input_loops::_helpers::model_loop::model_loop_schema::schema_enums; use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; use sqlbuild_python_base::compile_driver::constants::NATIVE_HEADER_COLUMNS; use sqlbuild_python_base::compile_driver::constants::PARSED_HEADER_COLUMNS; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_hooks.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_hooks.rs index 73a36178f3..23aee1c7db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_hooks.rs @@ -18,22 +18,23 @@ use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, - validation_outcome, -}; -use crate::bindings::_helpers::input_loops::model_loop_calls::{CallOrigin, report_invalid_calls}; -use crate::bindings::_helpers::input_loops::model_loop_external::external_errors; -use crate::bindings::_helpers::input_loops::model_loop_inputs::{ - HOOK_KEYS, present, reference_row, -}; -use crate::bindings::_helpers::input_loops::model_loop_macros::{ - argument_references, expanded_with, -}; -use crate::bindings::_helpers::input_loops::model_loop_named::{ - SQL_HOOK_DECLARATION, named_usages, -}; -use crate::bindings::_helpers::input_loops::model_loop_visibility::ModelVisibility; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::VisibleDeclarations; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::record_reads; +use crate::input_loops::_helpers::model::model_loop::validation_outcome; +use crate::input_loops::_helpers::model::model_loop_calls::CallOrigin; +use crate::input_loops::_helpers::model::model_loop_calls::report_invalid_calls; +use crate::input_loops::_helpers::model::model_loop_external::external_errors; +use crate::input_loops::_helpers::model::model_loop_inputs::HOOK_KEYS; +use crate::input_loops::_helpers::model::model_loop_inputs::present; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_row; +use crate::input_loops::_helpers::model_loop::model_loop_macros::argument_references; +use crate::input_loops::_helpers::model_loop::model_loop_macros::expanded_with; +use crate::input_loops::_helpers::model_loop::model_loop_named::SQL_HOOK_DECLARATION; +use crate::input_loops::_helpers::model_loop::model_loop_named::named_usages; +use crate::input_loops::_helpers::model_loop::model_loop_visibility::ModelVisibility; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::argument_pairs::argument_pairs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs index add77b1fbe..bd3e640752 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs @@ -13,10 +13,10 @@ use sqlbuild_analysis::compiled_project::models::{ use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_discovery::model_files::models::HeaderFacts; -use crate::bindings::_helpers::input_loops::model_loop_header::AttachedHeader; -use crate::bindings::_helpers::input_loops::model_loop_inputs::{ - HOOK_KEYS, reference_fact, stripped_values, -}; +use crate::input_loops::_helpers::model::model_loop_header::AttachedHeader; +use crate::input_loops::_helpers::model::model_loop_inputs::HOOK_KEYS; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_fact; +use crate::input_loops::_helpers::model::model_loop_inputs::stripped_values; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/mod.rs new file mode 100644 index 0000000000..d7077c7a10 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod model_loop_macros; +pub(crate) mod model_loop_named; +pub(crate) mod model_loop_opt_out; +pub(crate) mod model_loop_prepass; +pub(crate) mod model_loop_python_hooks; +pub(crate) mod model_loop_schema; +pub(crate) mod model_loop_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs index 7db7b0df27..a4bef0ac64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs @@ -10,10 +10,10 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, VisibleDeclarations, compile_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::VisibleDeclarations; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_row; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::expansion::expanded::expanded; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_named.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_named.rs index d0dfcb0890..503513302b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_named.rs @@ -9,7 +9,7 @@ use sqlbuild_scopes::scope_index::models::{ DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, }; -use crate::bindings::_helpers::input_loops::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::LoopContext; use sqlbuild_python_base::records::main::fields::field::field as native_field; const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs index c3618fe353..4d65cb532d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs @@ -22,17 +22,19 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::bindings::_helpers::input_loops::model_loop_prepass::spawned; +use crate::input_loops::_helpers::model_loop::model_loop_prepass::spawned; use sqlbuild_python_base::cache::main::parse_key::parse_key; use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, compile_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::{HOOK_KEYS, present}; -use crate::bindings::_helpers::project_assembly::resources::{syntax_error_message, syntax_valid}; -use crate::bindings::_helpers::project_assembly::syntax_validation::sql_entry_label; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop_inputs::HOOK_KEYS; +use crate::input_loops::_helpers::model::model_loop_inputs::present; +use crate::input_loops::models::NativeModelLoop; +use crate::project_assembly::main::sql_entry_label::sql_entry_label; +use crate::project_assembly::main::syntax_error_message::syntax_error_message; +use crate::project_assembly::main::syntax_valid::syntax_valid; use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -97,9 +99,9 @@ struct SiteCheck { /// A hook value being walked: its key, its file and the label its statements are named by. struct HookSite<'a> { - hook_name: &'a str, + pub(crate) hook_name: &'a str, path: String, - label: Option, + pub(crate) label: Option, } /// Whether the opt-out is rejected, and where it was authored. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_prepass.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_prepass.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs index 0c97ea2e85..db5025480a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_prepass.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs @@ -8,7 +8,7 @@ use sqlbuild_discovery::model_files::main::header_facts::header_facts; use sqlbuild_discovery::model_files::models::HeaderFacts; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::input_loops::model_loop_host::ModelLoopRow; +use crate::input_loops::_helpers::model::model_loop_host::ModelLoopRow; /// The compile pools' fixed worker count. const PREPASS_WORKERS: usize = 4; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_python_hooks.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_python_hooks.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_python_hooks.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_python_hooks.rs index 18b895d331..6aa530f2aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_python_hooks.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_python_hooks.rs @@ -7,10 +7,10 @@ use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; -use crate::bindings::_helpers::input_loops::model_loop_named::{ - PYTHON_HOOK_DECLARATION, named_usages, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model_loop::model_loop_named::PYTHON_HOOK_DECLARATION; +use crate::input_loops::_helpers::model_loop::model_loop_named::named_usages; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Hook keys in the sorted order `validate_python_hook_config` reads them. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_schema.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_schema.rs index 51a18e67a4..69efc69017 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_schema.rs @@ -5,8 +5,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::present; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop_inputs::present; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::models::HeaderColumn; use sqlbuild_python_project::compiled_project::main::python_column_fact::python_column_fact; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs index 0c032f1090..049158b393 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs @@ -8,7 +8,8 @@ use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, VisibleDeclarations}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::VisibleDeclarations; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::types::MacroScopes; use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs index a12c692d64..b5e80f1bc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs @@ -4,20 +4,17 @@ use std::collections::HashMap; use std::path::PathBuf; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; -use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES}; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -use crate::bindings::_helpers::input_loops::resource_attachment::{ - attach_audits, attach_functions, attach_sources, attach_sql_tests, -}; -use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; +use crate::input_loops::_helpers::resources::resource_attachment::attach_functions; +use crate::input_loops::_helpers::resources::resource_attachment::attach_sources; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -33,7 +30,6 @@ use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_project::models::AuditBlockRow; use sqlbuild_python_project::compiled_project::models::AuditFileRow; -use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; use sqlbuild_python_project::compiled_project::models::FunctionRow; use sqlbuild_python_project::compiled_project::models::FunctionSettings; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; @@ -47,7 +43,7 @@ use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORIT const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PATHLIB_MODULE: &str = "pathlib"; -const TEST_TIMING_KEY: &str = "test_input_compile_ms"; +pub(crate) const TEST_TIMING_KEY: &str = "test_input_compile_ms"; const DESCRIPTION_KEY: &str = "description"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; @@ -67,7 +63,10 @@ fn stem(value: &Bound<'_, PyAny>, name: &str) -> PyResult { } /// Each item of the iterable `value.`. -fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { +pub(crate) fn items<'py>( + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { native_field(value, name)?.try_iter()?.collect() } @@ -270,7 +269,7 @@ fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { }) } -fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { +pub(crate) fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { Ok(TestFileRow { file_path: text(&file, "file_path")?, relative_path: text(&file, "relative_path")?, @@ -284,7 +283,7 @@ fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { }) } -fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { +pub(crate) fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { Ok(ScenarioRow { file_path: text(&scenario, "file_path")?, relative_path: text(&scenario, "relative_path")?, @@ -328,7 +327,7 @@ fn parameter_renderer<'py>(value_renderer: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn sql_test_settings<'py>( py: Python<'py>, (value_renderer, syntax, scan_cache): ( &Bound<'py, PyAny>, @@ -400,7 +399,7 @@ fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { }) } -fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { +pub(crate) fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = file.py(); let generic: Bound<'_, PyAny> = enum_member(py, RecordId::DeclarationKind, "AUDIT")?; Ok(AuditFileRow { @@ -452,7 +451,7 @@ fn builtin_audit_file<'py>( ) } -fn audit_settings<'py>( +pub(crate) fn audit_settings<'py>( py: Python<'py>, effective_settings: &Bound<'py, PyAny>, ) -> PyResult> { @@ -515,7 +514,7 @@ fn loader_rows<'py>(discovered: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn diagnostic_settings<'py>( discovered: &Bound<'py, PyAny>, source_inputs: &Bound<'py, PyAny>, ) -> PyResult> { @@ -605,85 +604,3 @@ fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { settings.line_width, )) } - -/// Attach every SQL test case, scenario and audit, then report every resource diagnostic. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn attach_checks<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "attach_checks", - options, - ( - &[ - "discovered_inputs", - "native_project", - "settings", - "scan_cache", - "source_inputs", - ], - &[], - ), - )?; - let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; - let native_project: &Bound<'py, NativeCompiledProject> = - &arguments.value("native_project")?.downcast_into()?; - let (effective_settings, value_renderer, syntax): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = arguments.value("settings")?.extract()?; - let scan_cache: &Bound<'py, PyAny> = &arguments.value("scan_cache")?; - let source_inputs: &Bound<'py, PyAny> = &arguments.value("source_inputs")?; - let (tests, scenarios) = record_compile_timing(TEST_TIMING_KEY, || -> PyResult<_> { - let attached = attach_sql_tests( - native_project, - items(discovered_inputs, "test_files")? - .into_iter() - .map(test_file_row) - .collect::>()?, - items(discovered_inputs, "scenario_files")? - .into_iter() - .map(scenario_row) - .collect::>()?, - sql_test_settings(py, (&value_renderer, &syntax, scan_cache))?.extract()?, - )?; - scan_cache.call_method0("save")?; - Ok(attached) - })?; - let (audits, audit_diagnostics) = attach_audits( - native_project, - items(discovered_inputs, "audit_files")? - .into_iter() - .map(audit_file_row) - .collect::>()?, - audit_settings(py, &effective_settings)?.extract()?, - )?; - let settings: DiagnosticSettings = - diagnostic_settings(discovered_inputs, source_inputs)?.extract()?; - let (orphans, descriptions, cursors) = - resource_diagnostics(&native_project.borrow(), py, &settings)?; - let diagnostics: Vec> = audit_diagnostics - .into_iter() - .chain(orphans) - .chain(descriptions) - .chain(cursors) - .collect(); - PyTuple::new( - py, - [ - PyTuple::new(py, tests)?.into_any(), - PyTuple::new(py, scenarios)?.into_any(), - PyTuple::new(py, audits)?.into_any(), - PyTuple::new(py, diagnostics)?.into_any(), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(attach_resources, module)?)?; - module.add_function(wrap_pyfunction!(attach_checks, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs index f433c73f53..7d04d8292b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs @@ -1,5 +1,6 @@ //! `build_project_audit_inputs`: singular and attached generic audits attached natively. +use crate::input_loops::models::AttachmentOwner; use std::collections::{BTreeMap, HashMap}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -22,17 +23,17 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, compile_error, helped_compile_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_calls::origin_references; -use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; -use crate::bindings::_helpers::input_loops::model_loop_named::{ - AUDIT_DECLARATION, resource_named_usages, -}; -use crate::bindings::_helpers::input_loops::source_loop::{ - AuthoredExpansion, expanded_authored_sql, file_declarations, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::helped_compile_error; +use crate::input_loops::_helpers::model::model_loop_calls::origin_references; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_row; +use crate::input_loops::_helpers::model_loop::model_loop_named::AUDIT_DECLARATION; +use crate::input_loops::_helpers::model_loop::model_loop_named::resource_named_usages; +use crate::input_loops::_helpers::resources::source_loop::AuthoredExpansion; +use crate::input_loops::_helpers::resources::source_loop::expanded_authored_sql; +use crate::input_loops::_helpers::resources::source_loop::file_declarations; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::AttachedTarget; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; @@ -74,16 +75,6 @@ struct Definition<'a> { builtin: bool, } -/// One resource's attached audits: its target, consumer path, and owner files and instances. -pub(crate) struct AttachmentOwner<'py> { - pub(crate) kind: &'static str, - pub(crate) name: String, - pub(crate) relation: String, - pub(crate) path: String, - /// `(owner file, column, instance)` in attachment order. - pub(crate) instances: Vec<(String, Option, AuditInstance<'py>)>, -} - /// Every singular audit, then each owner's attached audits, and the built-in shadow warnings. pub(crate) fn attached_audits<'py>( model_loop: &NativeModelLoop, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/expanded_offsets.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/expanded_offsets.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/expanded_offsets.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/expanded_offsets.rs index b5587988a0..8f3111cca9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/expanded_offsets.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/expanded_offsets.rs @@ -1,8 +1,8 @@ //! Map offsets in expanded SQL back onto the authored text that produced them. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::map_expanded_offset::map_expanded_offset as mapped; use sqlbuild_analysis::compiled_project::models::SplicedSpan; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -12,7 +12,7 @@ use sqlbuild_python_base::records::types::RecordId; /// `map_expanded_offset(*, offset, passes)`: a `MappedOffset` in the first pass's source. #[pyfunction] #[pyo3(signature = (*, offset, passes))] -fn map_expanded_offset<'py>( +pub(crate) fn map_expanded_offset<'py>( py: Python<'py>, offset: usize, passes: &Bound<'py, PyAny>, @@ -37,7 +37,3 @@ fn map_expanded_offset<'py>( arguments.set_item("generated", generated)?; new_record(py, RecordId::MappedOffset, (), Some(&arguments)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(map_expanded_offset, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs index b0fa440e2a..c29484c4f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs @@ -28,14 +28,17 @@ use sqlbuild_attachments::functions::models::{ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, compile_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_calls::{CallOrigin, report_invalid_calls}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; -use crate::bindings::_helpers::input_loops::source_loop::{ - AuthoredExpansion, expanded_authored_sql, expanded_template, parse_checked, scoped_declarations, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop_calls::CallOrigin; +use crate::input_loops::_helpers::model::model_loop_calls::report_invalid_calls; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_row; +use crate::input_loops::_helpers::resources::source_loop::AuthoredExpansion; +use crate::input_loops::_helpers::resources::source_loop::expanded_authored_sql; +use crate::input_loops::_helpers::resources::source_loop::expanded_template; +use crate::input_loops::_helpers::resources::source_loop::parse_checked; +use crate::input_loops::_helpers::resources::source_loop::scoped_declarations; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::PythonRuntime; use sqlbuild_project::models::ReferenceRow; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs index 25ff2af941..48155dd3e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs @@ -1,28 +1,24 @@ //! The Python classes, project declarations and callbacks the native model loop builds with. +use crate::input_loops::main::table_backed::table_backed; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_host::entries::models::FacadeHost; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::project::declaration_rows_function::declaration_rows_function; /// The materializations a model's storage policies are inherited for, besides none. -const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; +pub(crate) const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; /// The declaration kinds whose project-owned records the loop names. const NAMED_KINDS: [&str; 4] = ["AUDIT", "SCHEMA", "SQL_HOOK", "PYTHON_HOOK"]; -fn dict<'py>( +pub(crate) fn dict<'py>( py: Python<'py>, entries: Vec<(&str, Bound<'py, PyAny>)>, ) -> PyResult> { @@ -33,14 +29,6 @@ fn dict<'py>( Ok(built) } -/// The values of a table-backed `MaterializationType`, as text. -pub(crate) fn table_backed<'py>(py: Python<'py>) -> PyResult>> { - TABLE_BACKED_MATERIALIZATIONS - .iter() - .map(|name| enum_member(py, RecordId::MaterializationType, name)) - .collect() -} - /// The builder, storage defaults and classes a model's `CompileModelConfig` is made of. pub(crate) fn native_config_classes<'py>( session: &Bound<'py, PyAny>, @@ -87,7 +75,7 @@ pub(crate) fn native_config_classes<'py>( } /// Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`. -fn named_records<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn named_records<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = resolver.py(); let records: Bound<'py, PyDict> = PyDict::new(py); if resolver.is_none() { @@ -119,7 +107,7 @@ fn named_records<'py>(resolver: &Bound<'py, PyAny>) -> PyResult(resolver: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn schema_enums<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = resolver.py(); let enums: Bound<'py, PyDict> = PyDict::new(py); if resolver.is_none() { @@ -143,7 +131,7 @@ fn schema_enums<'py>(resolver: &Bound<'py, PyAny>) -> PyResult(function: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn hook_signature<'py>(function: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = function.py(); let inspect: Bound<'py, PyAny> = py.import("inspect")?.into_any(); let empty: Bound<'py, PyAny> = native_field(&inspect, "Parameter")?.getattr("empty")?; @@ -167,7 +155,7 @@ fn hook_signature<'py>(function: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn visibility_classes<'py>( context: &Bound<'py, PyAny>, visible_declarations: &Bound<'py, PyAny>, ) -> PyResult> { @@ -238,7 +226,7 @@ fn visibility_classes<'py>( ) } -fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { +pub(crate) fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { let compiled: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("object_key", record_class(py, RecordId::CompiledObjectKey)?), ( @@ -341,102 +329,3 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ], ) } - -/// The classes, error factories and project declarations the model loop reads. -pub(crate) fn model_loop_classes<'py>( - audit_factories: &Bound<'py, PyAny>, - context: &Bound<'py, PyAny>, - external_resolver: &Bound<'py, PyAny>, - host_inputs: &Bound<'py, PyDict>, -) -> PyResult> { - let py: Python<'py> = context.py(); - let host = |key: &str| -> PyResult> { host_inputs.as_any().get_item(key) }; - let resolver: Bound<'py, PyAny> = native_field(context, "declaration_resolver")?; - let mut provider_names: Vec = host("provider_names")? - .try_iter()? - .map(|name| name?.extract()) - .collect::>()?; - provider_names.sort(); - let hooks: Bound<'py, PyDict> = dict( - py, - vec![ - ( - "functions", - by_attribute(&host("hook_functions")?, "name")?.into_any(), - ), - ("provider_names", provider_names.into_bound_py_any(py)?), - ( - "python_hook_entry", - record_class(py, RecordId::PythonHookEntry)?, - ), - ( - "signature", - wrap_pyfunction!(hook_signature, py)?.into_any(), - ), - ], - )?; - let record: Bound<'py, PyDict> = dict( - py, - vec![( - "declaration_rows", - declaration_rows_function(py)?.into_any(), - )], - )?; - dict( - py, - vec![ - ("usage_record", record_class(py, RecordId::UsageRecord)?), - ("compile_error", error_class(py, ErrorId::CompileInput)?), - ("syntax_help", host("syntax_help")?), - ("source_syntax_help", host("source_syntax_help")?), - ("macros", host("macros")?), - ("sql_hooks", host("sql_hooks")?), - ( - "scope_path_error", - error_class(py, ErrorId::InvalidScopePath)?, - ), - ("sql_hook_entry", record_class(py, RecordId::SqlHookEntry)?), - ( - "audit_factories", - by_attribute(audit_factories, "name")?.into_any(), - ), - ( - "generated_usage", - enum_member(py, RecordId::UsageKind, "GENERATED")?, - ), - ( - "contract_enforced", - enum_member(py, RecordId::ContractPolicy, "ENFORCED")?, - ), - ( - "model_schemas", - native_field(context, "public_model_schemas")?, - ), - ("named_records", named_records(&resolver)?.into_any()), - ("schema_enums", schema_enums(&resolver)?.into_any()), - ( - "compiler_diagnostic", - record_class(py, RecordId::CompilerDiagnostic)?, - ), - ("report_diagnostic", host("report_diagnostic")?), - ("path_class", imported(py, "pathlib", "Path")?), - ("external_resolver", external_resolver.clone()), - ( - "reference_kind", - record_class(py, RecordId::SqlReferenceKind)?, - ), - ( - "config", - native_config_classes(&host("config_session")?, &host("record_template_reads")?)? - .into_any(), - ), - ("hooks", hooks.into_any()), - ( - "visibility", - visibility_classes(context, &host("visible_declarations")?)?.into_any(), - ), - ("record", record.into_any()), - ("input_view", input_view_classes(py)?.into_any()), - ], - ) -} diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/mod.rs new file mode 100644 index 0000000000..b4d263bbd1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod attachment; +pub(crate) mod audit_loop; +pub(crate) mod expanded_offsets; +pub(crate) mod function_loop; +pub(crate) mod loop_classes; +pub(crate) mod resource_attachment; +pub(crate) mod resource_diagnostics; +pub(crate) mod scenario_loop; +pub(crate) mod source_loop; +pub(crate) mod sql_test_loop; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs index 55d0611ed3..43edebe9bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs @@ -4,17 +4,16 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::input_loops::audit_loop::{ - AttachmentOwner, model_owner, resource_owner, -}; -use crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop; -use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; +use crate::input_loops::_helpers::resources::audit_loop::model_owner; +use crate::input_loops::_helpers::resources::audit_loop::resource_owner; +use crate::input_loops::models::AttachmentOwner; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use sqlbuild_python_project::compiled_project::models::{ - AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, ResourceAttachment, ScenarioRow, - SqlTestSettings, TestFileRow, + AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, ScenarioRow, SqlTestSettings, + TestFileRow, }; use sqlbuild_python_project::compiled_project::types::{ AttachedSqlTests, SourceRow, SourceSettings, @@ -31,15 +30,6 @@ const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; const SOURCE_OWNER: &str = "source"; const SEED_OWNER: &str = "seed"; -/// The attachment runs the project's Python methods dispatch to, installed at registration. -pub(crate) const RESOURCE_ATTACHMENT: ResourceAttachment = ResourceAttachment { - sources: attach_sources, - functions: attach_functions, - diagnostics: resource_diagnostics, - sql_tests: attach_sql_tests, - audits: attach_audits, -}; - /// Attach every discovered source in order with the kept loop; return each source's view. pub(crate) fn attach_sources( project: &Bound<'_, NativeCompiledProject>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/scenario_loop.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/scenario_loop.rs index 5147d83a59..eecaa65007 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/scenario_loop.rs @@ -14,14 +14,17 @@ use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenari use sqlbuild_attachments::test_targets::models::ScenarioCteSources; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::input_loops::model_loop::compile_error; -use crate::bindings::_helpers::input_loops::model_loop_calls::origin_references; -use crate::bindings::_helpers::input_loops::source_loop::{ - AuthoredExpansion, expanded_authored_sql, scoped_declarations, -}; -use crate::bindings::_helpers::input_loops::sql_test_loop::{ - FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, -}; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop_calls::origin_references; +use crate::input_loops::_helpers::resources::source_loop::AuthoredExpansion; +use crate::input_loops::_helpers::resources::source_loop::expanded_authored_sql; +use crate::input_loops::_helpers::resources::source_loop::scoped_declarations; +use crate::input_loops::_helpers::resources::sql_test_loop::FileOrigin; +use crate::input_loops::_helpers::resources::sql_test_loop::TestScope; +use crate::input_loops::_helpers::resources::sql_test_loop::assertion_targets; +use crate::input_loops::_helpers::resources::sql_test_loop::checked_body; +use crate::input_loops::_helpers::resources::sql_test_loop::source_names; +use crate::input_loops::_helpers::resources::sql_test_loop::unique; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_project::compiled_project::models::ScenarioRow; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs index 484c2e0534..a244800c61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs @@ -7,15 +7,14 @@ use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_fre use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, compile_error, record_reads, -}; -use crate::bindings::_helpers::input_loops::model_loop_macros::{ - argument_references as macro_argument_references, expanded_with, -}; -use crate::bindings::_helpers::project_assembly::resources::{ - syntax_error_message, syntax_error_messages, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::record_reads; +use crate::input_loops::_helpers::model_loop::model_loop_macros::argument_references as macro_argument_references; +use crate::input_loops::_helpers::model_loop::model_loop_macros::expanded_with; +use crate::input_loops::models::NativeModelLoop; +use crate::project_assembly::main::syntax_error_message::syntax_error_message; +use crate::project_assembly::main::syntax_error_messages::syntax_error_messages; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::expanded::expanded; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs index 9f0b8e0f3e..85fe3fcf93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs @@ -35,15 +35,16 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::input_loops::model_loop::{ - LoopContext, NativeModelLoop, compile_error, helped_compile_error, -}; -use crate::bindings::_helpers::input_loops::model_loop_calls::{ - CallOrigin, origin_references, report_invalid_call, -}; -use crate::bindings::_helpers::input_loops::source_loop::{ - AuthoredExpansion, expanded_authored_sql, scoped_declarations, -}; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::helped_compile_error; +use crate::input_loops::_helpers::model::model_loop_calls::CallOrigin; +use crate::input_loops::_helpers::model::model_loop_calls::origin_references; +use crate::input_loops::_helpers::model::model_loop_calls::report_invalid_call; +use crate::input_loops::_helpers::resources::source_loop::AuthoredExpansion; +use crate::input_loops::_helpers::resources::source_loop::expanded_authored_sql; +use crate::input_loops::_helpers::resources::source_loop::scoped_declarations; +use crate::input_loops::models::NativeModelLoop; use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-compile/src/input_loops/constants.rs b/crates/sqlbuild-python-compile/src/input_loops/constants.rs new file mode 100644 index 0000000000..0d843d9ac6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/constants.rs @@ -0,0 +1,17 @@ +//! Public constants of input loops. + +use crate::input_loops::_helpers::resources::resource_attachment::attach_audits; +use crate::input_loops::_helpers::resources::resource_attachment::attach_functions; +use crate::input_loops::_helpers::resources::resource_attachment::attach_sources; +use crate::input_loops::_helpers::resources::resource_attachment::attach_sql_tests; +use crate::input_loops::_helpers::resources::resource_diagnostics::resource_diagnostics; +use sqlbuild_python_project::compiled_project::models::ResourceAttachment; + +/// The attachment runs the project's Python methods dispatch to, installed at registration. +pub const RESOURCE_ATTACHMENT: ResourceAttachment = ResourceAttachment { + sources: attach_sources, + functions: attach_functions, + diagnostics: resource_diagnostics, + sql_tests: attach_sql_tests, + audits: attach_audits, +}; diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/attach_checks.rs b/crates/sqlbuild-python-compile/src/input_loops/main/attach_checks.rs new file mode 100644 index 0000000000..0627b17079 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/attach_checks.rs @@ -0,0 +1,97 @@ +//! Attach every SQL test case, scenario and audit, then report every resource diagnostic. + +use crate::input_loops::_helpers::resources::attachment::TEST_TIMING_KEY; +use crate::input_loops::_helpers::resources::attachment::audit_file_row; +use crate::input_loops::_helpers::resources::attachment::audit_settings; +use crate::input_loops::_helpers::resources::attachment::diagnostic_settings; +use crate::input_loops::_helpers::resources::attachment::items; +use crate::input_loops::_helpers::resources::attachment::scenario_row; +use crate::input_loops::_helpers::resources::attachment::sql_test_settings; +use crate::input_loops::_helpers::resources::attachment::test_file_row; +use crate::input_loops::_helpers::resources::resource_attachment::attach_audits; +use crate::input_loops::_helpers::resources::resource_attachment::attach_sql_tests; +use crate::input_loops::_helpers::resources::resource_diagnostics::resource_diagnostics; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; + +/// Attach every SQL test case, scenario and audit, then report every resource diagnostic. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn attach_checks<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "attach_checks", + options, + ( + &[ + "discovered_inputs", + "native_project", + "settings", + "scan_cache", + "source_inputs", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let native_project: &Bound<'py, NativeCompiledProject> = + &arguments.value("native_project")?.downcast_into()?; + let (effective_settings, value_renderer, syntax): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = arguments.value("settings")?.extract()?; + let scan_cache: &Bound<'py, PyAny> = &arguments.value("scan_cache")?; + let source_inputs: &Bound<'py, PyAny> = &arguments.value("source_inputs")?; + let (tests, scenarios) = record_compile_timing(TEST_TIMING_KEY, || -> PyResult<_> { + let attached = attach_sql_tests( + native_project, + items(discovered_inputs, "test_files")? + .into_iter() + .map(test_file_row) + .collect::>()?, + items(discovered_inputs, "scenario_files")? + .into_iter() + .map(scenario_row) + .collect::>()?, + sql_test_settings(py, (&value_renderer, &syntax, scan_cache))?.extract()?, + )?; + scan_cache.call_method0("save")?; + Ok(attached) + })?; + let (audits, audit_diagnostics) = attach_audits( + native_project, + items(discovered_inputs, "audit_files")? + .into_iter() + .map(audit_file_row) + .collect::>()?, + audit_settings(py, &effective_settings)?.extract()?, + )?; + let settings: DiagnosticSettings = + diagnostic_settings(discovered_inputs, source_inputs)?.extract()?; + let (orphans, descriptions, cursors) = + resource_diagnostics(&native_project.borrow(), py, &settings)?; + let diagnostics: Vec> = audit_diagnostics + .into_iter() + .chain(orphans) + .chain(descriptions) + .chain(cursors) + .collect(); + PyTuple::new( + py, + [ + PyTuple::new(py, tests)?.into_any(), + PyTuple::new(py, scenarios)?.into_any(), + PyTuple::new(py, audits)?.into_any(), + PyTuple::new(py, diagnostics)?.into_any(), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/attach_resources.rs b/crates/sqlbuild-python-compile/src/input_loops/main/attach_resources.rs new file mode 100644 index 0000000000..b1bca90fb2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/attach_resources.rs @@ -0,0 +1,17 @@ +//! Pair every seed, then attach every function and source, in that order. + +use crate::input_loops::_helpers::resources::attachment; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Pair every seed, then attach every function and source, in that order. +#[inline] +pub fn attach_resources<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + attachment::attach_resources(py, options) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs new file mode 100644 index 0000000000..ecd159c3e5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs @@ -0,0 +1,8 @@ +pub mod attach_checks; +pub mod attach_resources; +pub mod model_loop_classes; +pub mod pooled; +pub mod register_attachment; +pub mod register_expanded_offsets; +pub mod register_model_loop; +pub mod table_backed; diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/model_loop_classes.rs b/crates/sqlbuild-python-compile/src/input_loops/main/model_loop_classes.rs new file mode 100644 index 0000000000..39785b01e4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/model_loop_classes.rs @@ -0,0 +1,120 @@ +//! The classes, error factories and project declarations the model loop reads. + +use crate::input_loops::_helpers::resources::loop_classes::dict; +use crate::input_loops::_helpers::resources::loop_classes::hook_signature; +use crate::input_loops::_helpers::resources::loop_classes::input_view_classes; +use crate::input_loops::_helpers::resources::loop_classes::named_records; +use crate::input_loops::_helpers::resources::loop_classes::native_config_classes; +use crate::input_loops::_helpers::resources::loop_classes::schema_enums; +use crate::input_loops::_helpers::resources::loop_classes::visibility_classes; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObjectExt, wrap_pyfunction}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::declaration_rows_function::declaration_rows_function; + +/// The classes, error factories and project declarations the model loop reads. +pub fn model_loop_classes<'py>( + audit_factories: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external_resolver: &Bound<'py, PyAny>, + host_inputs: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let host = |key: &str| -> PyResult> { host_inputs.as_any().get_item(key) }; + let resolver: Bound<'py, PyAny> = native_field(context, "declaration_resolver")?; + let mut provider_names: Vec = host("provider_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + provider_names.sort(); + let hooks: Bound<'py, PyDict> = dict( + py, + vec![ + ( + "functions", + by_attribute(&host("hook_functions")?, "name")?.into_any(), + ), + ("provider_names", provider_names.into_bound_py_any(py)?), + ( + "python_hook_entry", + record_class(py, RecordId::PythonHookEntry)?, + ), + ( + "signature", + wrap_pyfunction!(hook_signature, py)?.into_any(), + ), + ], + )?; + let record: Bound<'py, PyDict> = dict( + py, + vec![( + "declaration_rows", + declaration_rows_function(py)?.into_any(), + )], + )?; + dict( + py, + vec![ + ("usage_record", record_class(py, RecordId::UsageRecord)?), + ("compile_error", error_class(py, ErrorId::CompileInput)?), + ("syntax_help", host("syntax_help")?), + ("source_syntax_help", host("source_syntax_help")?), + ("macros", host("macros")?), + ("sql_hooks", host("sql_hooks")?), + ( + "scope_path_error", + error_class(py, ErrorId::InvalidScopePath)?, + ), + ("sql_hook_entry", record_class(py, RecordId::SqlHookEntry)?), + ( + "audit_factories", + by_attribute(audit_factories, "name")?.into_any(), + ), + ( + "generated_usage", + enum_member(py, RecordId::UsageKind, "GENERATED")?, + ), + ( + "contract_enforced", + enum_member(py, RecordId::ContractPolicy, "ENFORCED")?, + ), + ( + "model_schemas", + native_field(context, "public_model_schemas")?, + ), + ("named_records", named_records(&resolver)?.into_any()), + ("schema_enums", schema_enums(&resolver)?.into_any()), + ( + "compiler_diagnostic", + record_class(py, RecordId::CompilerDiagnostic)?, + ), + ("report_diagnostic", host("report_diagnostic")?), + ("path_class", imported(py, "pathlib", "Path")?), + ("external_resolver", external_resolver.clone()), + ( + "reference_kind", + record_class(py, RecordId::SqlReferenceKind)?, + ), + ( + "config", + native_config_classes(&host("config_session")?, &host("record_template_reads")?)? + .into_any(), + ), + ("hooks", hooks.into_any()), + ( + "visibility", + visibility_classes(context, &host("visible_declarations")?)?.into_any(), + ), + ("record", record.into_any()), + ("input_view", input_view_classes(py)?.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs b/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs new file mode 100644 index 0000000000..c8100b7377 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs @@ -0,0 +1,9 @@ +//! `each` of every item in order, run on the compile pool, or in turn when the pool cannot start. + +use crate::input_loops::_helpers::model_loop::model_loop_prepass; + +/// `each` of every item in order, run on the compile pool, or in turn when the pool cannot start. +#[inline] +pub fn pooled(items: &[T], each: impl Fn(&T) -> R + Sync + Send) -> Vec { + model_loop_prepass::pooled(items, each) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/register_attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/main/register_attachment.rs new file mode 100644 index 0000000000..7509b0d477 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/register_attachment.rs @@ -0,0 +1,12 @@ +//! Register the attachment functions and classes. + +use crate::input_loops::_helpers::resources::attachment::attach_resources; +use crate::input_loops::main::attach_checks::attach_checks; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the attachment functions and classes. +pub fn register_attachment(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(attach_resources, module)?)?; + module.add_function(wrap_pyfunction!(attach_checks, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/register_expanded_offsets.rs b/crates/sqlbuild-python-compile/src/input_loops/main/register_expanded_offsets.rs new file mode 100644 index 0000000000..ec6764be37 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/register_expanded_offsets.rs @@ -0,0 +1,10 @@ +//! Register the expanded offsets functions and classes. + +use crate::input_loops::_helpers::resources::expanded_offsets::map_expanded_offset; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the expanded offsets functions and classes. +pub fn register_expanded_offsets(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(map_expanded_offset, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/register_model_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/main/register_model_loop.rs new file mode 100644 index 0000000000..f59b939c95 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/register_model_loop.rs @@ -0,0 +1,9 @@ +//! Register the model loop functions and classes. + +use crate::input_loops::models::NativeModelLoop; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the model loop functions and classes. +pub fn register_model_loop(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/table_backed.rs b/crates/sqlbuild-python-compile/src/input_loops/main/table_backed.rs new file mode 100644 index 0000000000..c98e3b9038 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/main/table_backed.rs @@ -0,0 +1,14 @@ +//! The values of a table-backed `MaterializationType`, as text. + +use crate::input_loops::_helpers::resources::loop_classes::TABLE_BACKED_MATERIALIZATIONS; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::types::RecordId; + +/// The values of a table-backed `MaterializationType`, as text. +pub fn table_backed<'py>(py: Python<'py>) -> PyResult>> { + TABLE_BACKED_MATERIALIZATIONS + .iter() + .map(|name| enum_member(py, RecordId::MaterializationType, name)) + .collect() +} diff --git a/crates/sqlbuild-python-compile/src/input_loops/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/mod.rs new file mode 100644 index 0000000000..4ac7b2f355 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/input_loops/mod.rs @@ -0,0 +1,6 @@ +//! The native model loop and the loops that attach the project's other resources to it. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/models.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs rename to crates/sqlbuild-python-compile/src/input_loops/models.rs index f086f52c52..2eb18b2ce3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/models.rs @@ -1,101 +1,122 @@ -//! The compile-input model loop: each model's steps run in the attachment's order, natively. - -use sqlbuild_python_base::positional_record; -use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{FromPyObject, pyclass, pymethods}; +//! Public records of input loops. + +use crate::input_loops::_helpers::model::model_loop::ANALYSIS_CONFLICT; +use crate::input_loops::_helpers::model::model_loop::ANALYSIS_CONFLICT_CODE; +use crate::input_loops::_helpers::model::model_loop::CURSOR_KEY; +use crate::input_loops::_helpers::model::model_loop::DECLARED_ATTRIBUTE; +use crate::input_loops::_helpers::model::model_loop::INCREMENTAL_MATERIALIZATION; +use crate::input_loops::_helpers::model::model_loop::LoopChecks; +use crate::input_loops::_helpers::model::model_loop::LoopClasses; +use crate::input_loops::_helpers::model::model_loop::LoopContext; +use crate::input_loops::_helpers::model::model_loop::LoopModel; +use crate::input_loops::_helpers::model::model_loop::LoopSettingsInput; +use crate::input_loops::_helpers::model::model_loop::MATERIALIZED_KEY; +use crate::input_loops::_helpers::model::model_loop::RETENTION_ATTRIBUTE; +use crate::input_loops::_helpers::model::model_loop::TABLE_TYPE_ATTRIBUTE; +use crate::input_loops::_helpers::model::model_loop::UNMANAGED_ATTRIBUTE; +use crate::input_loops::_helpers::model::model_loop::VALIDATE_METHOD; +use crate::input_loops::_helpers::model::model_loop::VALUES_ATTRIBUTE; +use crate::input_loops::_helpers::model::model_loop::VisibleDeclarations; +use crate::input_loops::_helpers::model::model_loop::applied_interpolation; +use crate::input_loops::_helpers::model::model_loop::compile_error; +use crate::input_loops::_helpers::model::model_loop::index_sql_hook_definitions; +use crate::input_loops::_helpers::model::model_loop::prepared_rows; +use crate::input_loops::_helpers::model::model_loop::query_typed_calls; +use crate::input_loops::_helpers::model::model_loop::spliced_spans; +use crate::input_loops::_helpers::model::model_loop::validation_outcome; +use crate::input_loops::_helpers::model::model_loop_calls::CallOrigin; +use crate::input_loops::_helpers::model::model_loop_calls::report_invalid_calls; +use crate::input_loops::_helpers::model::model_loop_config::ModelConfigs; +use crate::input_loops::_helpers::model::model_loop_external::external_errors; +use crate::input_loops::_helpers::model::model_loop_header::AttachedHeader; +use crate::input_loops::_helpers::model::model_loop_header::HeaderSource; +use crate::input_loops::_helpers::model::model_loop_header::header_schema; +use crate::input_loops::_helpers::model::model_loop_hooks::ExpandedHooks; +use crate::input_loops::_helpers::model::model_loop_hooks::HookModel; +use crate::input_loops::_helpers::model::model_loop_hooks::expanded_hooks; +use crate::input_loops::_helpers::model::model_loop_host::FinishedModel; +use crate::input_loops::_helpers::model::model_loop_host::ModelLoopRow; +use crate::input_loops::_helpers::model::model_loop_host::RecordInputs; +use crate::input_loops::_helpers::model::model_loop_host::finished_record; +use crate::input_loops::_helpers::model::model_loop_inputs::analysis_override; +use crate::input_loops::_helpers::model::model_loop_inputs::hook_presence; +use crate::input_loops::_helpers::model::model_loop_inputs::reference_row; +use crate::input_loops::_helpers::model_loop::model_loop_macros::ExpandedMacros; +use crate::input_loops::_helpers::model_loop::model_loop_macros::MacroModel; +use crate::input_loops::_helpers::model_loop::model_loop_macros::expanded_macros; +use crate::input_loops::_helpers::model_loop::model_loop_named::SCHEMA_DECLARATION; +use crate::input_loops::_helpers::model_loop::model_loop_named::named_usages; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::AuthoredQuery; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::GapQueue; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::OptOut; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::OptOutModel; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::gap_job; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::reject_unparseable_query; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::rejected_opt_out; +use crate::input_loops::_helpers::model_loop::model_loop_opt_out::skip_unavailable; +use crate::input_loops::_helpers::model_loop::model_loop_prepass::NativeHeader; +use crate::input_loops::_helpers::model_loop::model_loop_prepass::all_header_facts; +use crate::input_loops::_helpers::model_loop::model_loop_prepass::native_header; +use crate::input_loops::_helpers::model_loop::model_loop_python_hooks::HookSignatures; +use crate::input_loops::_helpers::model_loop::model_loop_schema::ModelSchema; +use crate::input_loops::_helpers::model_loop::model_loop_schema::model_schema; +use crate::input_loops::_helpers::model_loop::model_loop_schema::schema_column_names; +use crate::input_loops::_helpers::model_loop::model_loop_visibility::ModelVisibility; +use crate::input_loops::_helpers::model_loop::model_loop_visibility::VisibilityModel; +use crate::input_loops::_helpers::resources::audit_loop::attached_audits; +use crate::input_loops::_helpers::resources::function_loop::attached_functions; +use crate::input_loops::_helpers::resources::function_loop::checked_table_function_arities; +use crate::input_loops::_helpers::resources::source_loop::attached_sources; +use crate::input_loops::_helpers::resources::sql_test_loop::TestScope; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use sqlbuild_analysis::compiled_project::main::analysis_gate::analysis_gate; use sqlbuild_analysis::compiled_project::main::calls_macros::calls_macros; use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; -use sqlbuild_analysis::compiled_project::models::{ - AnalysisGate, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, -}; -use sqlbuild_analysis::sql_macros::main::moved_typed_calls::moved_typed_calls; -use sqlbuild_analysis::sql_macros::main::spliced_typed_calls::spliced_typed_calls; -use sqlbuild_analysis::sql_macros::models::{TextSplice, TypedCall}; -use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; +use sqlbuild_analysis::compiled_project::models::AnalysisGate; +use sqlbuild_analysis::compiled_project::models::ModelLoopSettings; +use sqlbuild_analysis::compiled_project::models::ScanFaultOrigin; +use sqlbuild_analysis::compiled_project::models::SplicedSpan; +use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; use sqlbuild_attachments::test_targets::models::TargetCatalog; use sqlbuild_discovery::model_files::models::HeaderFacts; -use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; -use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; - -use crate::bindings::_helpers::input_loops::audit_loop::{AttachmentOwner, attached_audits}; -use crate::bindings::_helpers::input_loops::function_loop::{ - attached_functions, checked_table_function_arities, -}; -use crate::bindings::_helpers::input_loops::model_loop_calls::{CallOrigin, report_invalid_calls}; -use crate::bindings::_helpers::input_loops::model_loop_config::ModelConfigs; -use crate::bindings::_helpers::input_loops::model_loop_external::external_errors; -use crate::bindings::_helpers::input_loops::model_loop_header::{ - AttachedHeader, HeaderSource, header_schema, -}; -use crate::bindings::_helpers::input_loops::model_loop_hooks::{ - ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, -}; -use crate::bindings::_helpers::input_loops::model_loop_host::{ - DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, RecordClasses, RecordInputs, - finished_record, -}; -use crate::bindings::_helpers::input_loops::model_loop_inputs::{ - analysis_override, hook_presence, reference_row, -}; -use crate::bindings::_helpers::input_loops::model_loop_macros::{ - ExpandedMacros, MacroClasses, MacroModel, expanded_macros, -}; -use crate::bindings::_helpers::input_loops::model_loop_named::{SCHEMA_DECLARATION, named_usages}; -use crate::bindings::_helpers::input_loops::model_loop_opt_out::{ - AuthoredQuery, GapQueue, OptOut, OptOutModel, gap_job, reject_unparseable_query, - rejected_opt_out, skip_unavailable, -}; -use crate::bindings::_helpers::input_loops::model_loop_prepass::{ - NativeHeader, all_header_facts, native_header, -}; -use crate::bindings::_helpers::input_loops::model_loop_python_hooks::{ - HookClasses, HookSignatures, -}; -use crate::bindings::_helpers::input_loops::model_loop_schema::{ - ModelSchema, model_schema, schema_column_names, -}; -use crate::bindings::_helpers::input_loops::model_loop_visibility::{ - ModelVisibility, VisibilityClasses, VisibilityModel, -}; -use crate::bindings::_helpers::input_loops::source_loop::attached_sources; -use crate::bindings::_helpers::input_loops::sql_test_loop::TestScope; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; -use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; -use sqlbuild_python_front::model_config::models::ConfigClasses; +use sqlbuild_python_front::discovery::models::ModelFileTexts; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::models::AuditInstance; use sqlbuild_python_front::model_config::models::ContractClasses; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::render::main::calls::span_row::span_row; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; use sqlbuild_python_front::render::models::ConstantRendering; +use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, -}; -use sqlbuild_python_front::render::models::{SqlCallOrigin, SqlMacroSource}; +use sqlbuild_python_front::render::models::ExpandedDeclarations; +use sqlbuild_python_front::render::models::ExpansionConsumer; +use sqlbuild_python_front::render::models::SqlCallOrigin; +use sqlbuild_python_front::render::models::SqlMacroSource; use sqlbuild_python_front::render::types::SpanRow; -use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; -use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_batch; -use sqlbuild_python_front::sqltext::models::InterpolationRow as SqlInterpolationRow; -use sqlbuild_python_front::sqltext::models::InterpolationSources; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_project::compiled_project::models::AuditFileRow; use sqlbuild_python_project::compiled_project::models::AuditSettings; @@ -114,135 +135,78 @@ use sqlbuild_python_project::resource_views::models::NativeSourceInput; use sqlbuild_python_project::resource_views::types::AttachedAudit; use sqlbuild_python_project::resource_views::types::AttachedFunction; use sqlbuild_python_project::resource_views::types::AttachedSource; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; -const MATERIALIZED_KEY: &str = "materialized"; -const CURSOR_KEY: &str = "cursor"; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -const VALUES_ATTRIBUTE: &str = "values"; -const RETENTION_ATTRIBUTE: &str = "time_travel_retention"; -const UNMANAGED_ATTRIBUTE: &str = "unmanaged"; -const TABLE_TYPE_ATTRIBUTE: &str = "table_type"; -const DECLARED_ATTRIBUTE: &str = "declared"; -const VALIDATE_METHOD: &str = "validate"; -const HELP_ARGUMENT: &str = "help"; -const CODE_ARGUMENT: &str = "code"; -const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; -const ANALYSIS_CONFLICT_CODE: &str = "P003"; -const SQL_HOOKS_KEY: &str = "sql_hooks"; -const HOOK_FILES_KEY: &str = "hook_files"; -const DEFINITIONS_KEY: &str = "definitions"; -const COMPILE_ERROR_KEY: &str = "compile_error"; - -/// The native validator, header contract classes, and the loop's Python classes. -type LoopChecks = (Py, Py, Py); -positional_record! { - extract; - /// Whether SQL analysis runs, is skipped, is deferred, and is required. - pub(crate) struct LoopSwitches { - pub(crate) sql_analysis: bool, - pub(crate) no_sql_validation: bool, - pub(crate) defer_sql_validation: bool, - pub(crate) require_sql_analysis: bool, - } +/// One resource's attached audits: its target, consumer path, and owner files and instances. +#[derive(Debug)] +pub struct AttachmentOwner<'py> { + pub kind: &'static str, + pub name: String, + pub relation: String, + pub path: String, + /// `(owner file, column, instance)` in attachment order. + pub instances: Vec<(String, Option, AuditInstance<'py>)>, } -/// `(analysis switches, project config path, the adapter's analysis dialect)`. -type LoopSettingsInput = (LoopSwitches, String, Option); +/// Whether SQL analysis runs, is skipped, is deferred, and is required. +#[derive(Debug)] +pub struct LoopSwitches { + pub sql_analysis: bool, + pub no_sql_validation: bool, + pub defer_sql_validation: bool, + pub require_sql_analysis: bool, +} -/// The Python classes and error factory the loop builds usages, hooks and errors with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct LoopClasses<'py> { - pub(crate) usage_record: Bound<'py, PyAny>, - pub(crate) compile_error: Bound<'py, PyAny>, - pub(crate) sql_hook_entry: Bound<'py, PyAny>, - /// Discovered audit factories by name. - pub(crate) audit_factories: Bound<'py, PyDict>, - /// `UsageKind.GENERATED`, for usages an enum-typed column generates. - pub(crate) generated_usage: Bound<'py, PyAny>, - /// `ContractPolicy.ENFORCED`. - pub(crate) contract_enforced: Bound<'py, PyAny>, - /// Public reusable model schemas by name. - pub(crate) model_schemas: Bound<'py, PyDict>, - /// Each indexed schema's and Python hook's declaration record, by `(kind, name)`. - pub(crate) named_records: Bound<'py, PyDict>, - /// Project enums by name, as a schema's column types resolve them. - pub(crate) schema_enums: Bound<'py, PyDict>, - /// `CompilerDiagnostic` and `report_compile_diagnostic`, for S006. - pub(crate) compiler_diagnostic: Bound<'py, PyAny>, - pub(crate) report_diagnostic: Bound<'py, PyAny>, - /// `pathlib.Path`. - pub(crate) path_class: Bound<'py, PyAny>, - /// The dbt manifest's `ExternalSqlReferenceResolver`, or `None`. - pub(crate) external_resolver: Bound<'py, PyAny>, - /// `SqlReferenceKind`. - pub(crate) reference_kind: Bound<'py, PyAny>, - /// What a model's effective config is built from. - pub(crate) config: ConfigClasses<'py>, - /// What a model's visible declarations are resolved from. - pub(crate) visibility: VisibilityClasses<'py>, - /// What a model's Python hooks are checked against. - pub(crate) hooks: HookClasses<'py>, - /// What each model's SQL hooks are expanded with. - pub(crate) sql_hooks: SqlHookClasses<'py>, - /// What each model's macro calls are expanded with. - pub(crate) macros: MacroClasses<'py>, - /// `sql_analysis_opt_out_help(model_owned=True)`, the help of a model's syntax error. - pub(crate) syntax_help: String, - /// The same help for SQL a project's other resources own. - pub(crate) source_syntax_help: String, - /// `InvalidScopePathError`. - pub(crate) scope_path_error: Bound<'py, PyAny>, - /// What each finished model's input record is built with. - pub(crate) record: RecordClasses<'py>, - /// What model input views project their attributes with. - pub(crate) input_view: Bound<'py, PyAny>, +impl<'py> pyo3::FromPyObject<'py> for LoopSwitches { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis): ( + bool, + bool, + bool, + bool, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + sql_analysis, + no_sql_validation, + defer_sql_validation, + require_sql_analysis, + }) + } } /// The ordered model loop of one compile-input stage. #[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeModelLoop { - pub(crate) validator: Py, - pub(crate) syntax: LexicalSyntax, - pub(crate) reserved_markers: Vec, - pub(crate) settings: ModelLoopSettings, - pub(crate) project_config_path: String, +pub struct NativeModelLoop { + pub validator: Py, + pub syntax: LexicalSyntax, + pub reserved_markers: Vec, + pub settings: ModelLoopSettings, + pub project_config_path: String, /// Whether the dialect's SQL is analysed by Polyglot or by its native module. - pub(crate) route: AnalysisRoute, + pub route: AnalysisRoute, /// The adapter's analysis dialect name, which P026 names when no native parser reads it. - pub(crate) dialect: Option, - contract_classes: Py, - loop_classes: Py, -} - -/// What one run reads beside each model: the host, Python classes and header contract classes. -pub(crate) struct LoopContext<'a, 'py> { - pub(crate) host: &'a Bound<'py, ConstantRendering>, - pub(crate) classes: &'a LoopClasses<'py>, - pub(crate) contracts: &'a ContractClasses<'py>, - /// The lexical rules the project's SQL is authored in. - pub(crate) syntax: &'a LexicalSyntax, -} - -/// The declarations a model sees and the identity its usages are recorded for. -pub(crate) struct VisibleDeclarations<'py> { - pub(crate) declarations: Bound<'py, PyAny>, - pub(crate) consumer: Bound<'py, PyAny>, + pub dialect: Option, + pub(crate) contract_classes: Py, + pub(crate) loop_classes: Py, } -/// What the model loop knows about a model before finishing it. -struct LoopModel<'a, 'py> { - row: &'a ModelLoopRow, - config: Bound<'py, PyAny>, - visible: &'a VisibleDeclarations<'py>, - signatures: &'a HookSignatures, - visibility: &'a ModelVisibility<'py>, +impl std::fmt::Debug for NativeModelLoop { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeModelLoop") + .finish_non_exhaustive() + } } #[pymethods] impl NativeModelLoop { #[new] - pub(crate) fn new( + pub fn new( checks: LoopChecks, syntax: LexicalSyntaxInput, reserved_markers: Vec, @@ -279,94 +243,8 @@ impl NativeModelLoop { } } -/// Replace the SQL hook classes' discovered `hook_files` with their `definitions` by name. -fn index_sql_hook_definitions(loop_classes: &Bound<'_, PyDict>) -> PyResult<()> { - let py: Python<'_> = loop_classes.py(); - let hooks: Bound<'_, PyDict> = loop_classes - .as_any() - .get_item(SQL_HOOKS_KEY)? - .downcast_into()?; - let definitions: Bound<'_, PyDict> = PyDict::new(py); - for hook_file in hooks.as_any().get_item(HOOK_FILES_KEY)?.try_iter()? { - let hook_file: Bound<'_, PyAny> = hook_file?; - let name: Bound<'_, PyAny> = native_field(&hook_file, "name")?; - if let Some(existing) = definitions.get_item(&name)? { - let message: String = format!( - "Duplicate SQL hook name '{}' found in {} and {}", - name.str()?, - native_field(&existing, "relative_path")?.str()?, - native_field(&hook_file, "relative_path")?.str()? - ); - let error: Bound<'_, PyAny> = loop_classes - .as_any() - .get_item(COMPILE_ERROR_KEY)? - .call1((message,))?; - return Err(PyErr::from_value(error)); - } - definitions.set_item(name, hook_file)?; - } - hooks.set_item(DEFINITIONS_KEY, definitions) -} - -/// Each model's `@@`-interpolated query and declaration scan, all interpolated before any runs. -fn prepared_rows<'py>( - py: Python<'py>, - (classes, syntax): (&LoopClasses<'py>, &LexicalSyntax), - model_files: Vec>, -) -> PyResult> { - let hooks = &classes.sql_hooks; - let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); - let mut paths: Vec = Vec::with_capacity(model_files.len()); - for model_file in &model_files { - let record: &NativeModelFile = model_file.get(); - let texts: ModelFileTexts = record.path_texts(py)?; - sqls.push((record.query_text().to_string(), texts.file_path.clone())); - paths.push(texts); - } - let interpolated: Vec = interpolation_batch( - InterpolationSources( - hooks.effective_vars.clone(), - hooks.environment.clone(), - None, - hooks.render_project_var.clone(), - Some(syntax.clone()), - ), - &sqls, - )?; - let mut rows: Vec = Vec::with_capacity(model_files.len()); - for (((model_file, paths), (authored, _)), interpolated) in model_files - .into_iter() - .zip(paths) - .zip(sqls) - .zip(interpolated) - { - let SqlInterpolationRow { - sql: rendered, - spans, - reads, - error, - } = interpolated; - let sql: String = rendered.unwrap_or(authored); - let scan: Option = error - .is_none() - .then(|| declaration_scan(&sql, Some(syntax))); - let reads: Vec<(String, String)> = reads - .into_iter() - .map(|(kind, name)| (kind.to_owned(), name)) - .collect(); - let interpolation: InterpolationRow = InterpolationRow { - sql, - reads, - error, - spans, - }; - rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); - } - Ok(rows) -} - impl NativeModelLoop { - fn run_models<'py>( + pub(crate) fn run_models<'py>( &self, py: Python<'py>, project: Bound<'py, NativeCompiledProject>, @@ -453,7 +331,7 @@ impl NativeModelLoop { impl NativeModelLoop { /// Build every model in order into `project`; return each model's input view. - pub(crate) fn run<'py>( + pub fn run<'py>( slf: &Bound<'py, Self>, project: Bound<'py, NativeCompiledProject>, host: Bound<'py, ConstantRendering>, @@ -477,7 +355,7 @@ impl NativeModelLoop { } /// Attach every discovered source in order into `project`; return each source's view. - pub(crate) fn run_sources<'py>( + pub fn run_sources<'py>( &self, project: &Bound<'py, NativeCompiledProject>, host: &Bound<'py, ConstantRendering>, @@ -498,7 +376,7 @@ impl NativeModelLoop { } /// Attach every singular audit, then each owner's attached audits. - pub(crate) fn run_audits<'py>( + pub fn run_audits<'py>( &self, host: &Bound<'py, ConstantRendering>, files: &[AuditFileRow], @@ -518,7 +396,7 @@ impl NativeModelLoop { } /// Attach every SQL test case, then every scenario, against the project's resources. - pub(crate) fn run_sql_tests<'py>( + pub fn run_sql_tests<'py>( &self, host: &Bound<'py, ConstantRendering>, rows: (&[TestFileRow], &[ScenarioRow]), @@ -586,7 +464,7 @@ impl NativeModelLoop { } /// Attach every SQL, then Python, function in order, checking table function call arities. - pub(crate) fn run_functions<'py>( + pub fn run_functions<'py>( &self, host: &Bound<'py, ConstantRendering>, rows: (Vec, Vec), @@ -608,7 +486,7 @@ impl NativeModelLoop { } /// Run the model's steps in the attachment's order; `None` hands it to the attachment. - fn finish<'py>( + pub(crate) fn finish<'py>( &self, py: Python<'py>, context: &LoopContext<'_, 'py>, @@ -872,7 +750,7 @@ impl NativeModelLoop { } /// Analysed, rejected opt-out, skipped; a native parse queued on `gaps` unmarks it on a gap. - fn analysis<'py>( + pub(crate) fn analysis<'py>( &self, context: &LoopContext<'_, 'py>, model: &OptOutModel<'_, 'py>, @@ -917,7 +795,7 @@ impl NativeModelLoop { } /// The query with canonical cursor intrinsics, or the message the attachment raises. - fn canonical_query( + pub(crate) fn canonical_query( &self, name: &str, sql: &str, @@ -948,7 +826,7 @@ impl NativeModelLoop { } /// Validate the model natively, raising the validator's first error as Python does. - fn validate( + pub(crate) fn validate( &self, context: &LoopContext<'_, '_>, config: (&Bound<'_, PyAny>, &Bound<'_, PyDict>), @@ -985,139 +863,3 @@ impl NativeModelLoop { validation_outcome(&outcome, external, values) } } - -/// `_raise_validation_outcome`: nothing, the validator's error, or a rejected reference's error. -pub(crate) fn validation_outcome( - outcome: &Bound<'_, PyAny>, - external: Vec>, - values: &Bound<'_, PyDict>, -) -> PyResult<()> { - if outcome.is_none() { - return Ok(()); - } - if let Ok(index) = outcome.extract::() - && let Some(error) = external.into_iter().nth(index).flatten() - { - return Err(error); - } - Err(config_error(outcome, Some(values))) -} - -/// `applied_interpolation`: record the interpolation's template reads, then its SQL or error. -fn applied_interpolation( - context: &LoopContext<'_, '_>, - interpolation: &InterpolationRow, -) -> PyResult { - let InterpolationRow { - sql, reads, error, .. - } = interpolation; - record_reads(context, reads)?; - match error { - Some(message) => Err(compile_error(context.classes, message, None)), - None => Ok(sql.clone()), - } -} - -/// Record interpolation's environment and context reads for compile reuse. -pub(crate) fn record_reads( - context: &LoopContext<'_, '_>, - reads: &[(String, String)], -) -> PyResult<()> { - if !reads.is_empty() { - let py: Python<'_> = context.host.py(); - context - .classes - .config - .record_template_reads - .call1((PyTuple::new(py, reads)?,))?; - } - Ok(()) -} - -/// `CompileInputError(message, code=...)`, raised as the attachment raises it. -pub(crate) fn compile_error(classes: &LoopClasses<'_>, message: &str, code: Option<&str>) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); - if let Some(code) = code { - keywords.set_item(CODE_ARGUMENT, code)?; - } - classes.compile_error.call((message,), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - -/// `CompileInputError(message, help=help)`. -pub(crate) fn helped_compile_error( - classes: &LoopClasses<'_>, - message: &str, - help: Option<&str>, -) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); - keywords.set_item(HELP_ARGUMENT, help)?; - classes.compile_error.call((message,), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - -/// The Python exception a native config error raises as, given the values it names. -pub(crate) fn config_error(error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyDict>>) -> PyErr { - raised_config_exception(error, values.map(Bound::as_any)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} - -/// The spliced SQL's expression macro calls moved through macro expansion into the query's bytes. -fn query_typed_calls( - calls: Vec, - (spliced, expanded, query): (&str, &str, &str), - macro_spans: &[SpanRow], -) -> Vec { - if calls.is_empty() { - return calls; - } - let calls: Vec = if macro_spans.is_empty() { - calls - } else { - spliced_typed_calls(calls, (spliced, expanded), &spliced_spans(macro_spans)) - }; - if expanded == query { - return calls; - } - let Ok((canonical, spans)) = - replace_intrinsics_with_spans(expanded, "", |name| format!("{name}()")) - else { - return Vec::new(); - }; - if canonical != query { - return Vec::new(); - } - let splices: Vec = spans - .into_iter() - .map(|span| TextSplice { - source: (span.source_start, span.source_end), - output: (span.output_start, span.output_end), - }) - .collect(); - moved_typed_calls(calls, &splices) -} - -/// Span rows as the spliced spans the scan-fault locator maps through. -fn spliced_spans(rows: &[SpanRow]) -> Vec { - rows.iter() - .map(|row| SplicedSpan { - source_start: row.source_start, - source_end: row.source_end, - output_start: row.output_start, - output_end: row.output_end, - }) - .collect() -} diff --git a/crates/sqlbuild-python-compile/src/lib.rs b/crates/sqlbuild-python-compile/src/lib.rs new file mode 100644 index 0000000000..01a1cd453e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lib.rs @@ -0,0 +1,23 @@ +//! The compile layer of the `sqlbuild._native` bindings: the compile driver and command, input loops, project assembly, analysis and the analysis session, rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier and auditing. + +#![forbid(unsafe_code)] + +pub mod analysis; +pub mod analysis_session; +pub mod auditing; +pub mod compile_attachment; +pub mod compile_command; +pub mod compile_delta; +pub mod compile_driver; +pub mod compile_outputs; +pub mod dag_artifact; +pub mod frontier; +pub mod hand_built; +pub mod input_loops; +pub mod lineage; +pub mod manifest; +pub mod project_assembly; +pub mod rules; +pub mod scopes; +pub mod semantic_checks; +pub mod sql_test_glue; diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs new file mode 100644 index 0000000000..cf57c2b260 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs @@ -0,0 +1,301 @@ +//! `ProjectColumnLineage`: the project column lineage graph with lazily materialised views. + +use sqlbuild_python_base::positional_record; +use std::collections::HashMap; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; +use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; + +use crate::lineage::models::LineageClasses; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; +use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; + +/// The Python objects one edge record materialises from. +pub(crate) enum EdgeParts { + /// An edge given to the eager constructor, returned as is. + Given(Py), + /// Fact values: source resource type, transform kind and confidence objects. + Facts { + resource_type: Py, + transform: Py, + confidence: Py, + }, +} + +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: i64, + confidence_code: i64, + sources: Bound<'py, PyAny>, + } +} + +/// Indexed edge records, materialised edges shared between equal records. +pub(crate) struct EdgeRecords { + pub(crate) keys: Vec, + pub(crate) parts: Vec, + pub(crate) index: EdgeIndex, + pub(crate) materialised: Vec>>, +} + +impl std::fmt::Debug for EdgeRecords { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("EdgeRecords") + .finish_non_exhaustive() + } +} + +impl EdgeRecords { + pub(crate) fn new(keys: Vec, parts: Vec) -> Self { + let index: EdgeIndex = edge_index(&keys); + let materialised: Vec>> = keys.iter().map(|_| None).collect(); + Self { + keys, + parts, + index, + materialised, + } + } + + /// The edge at `position`, built once per distinct fact edge. + pub(crate) fn edge( + &mut self, + classes: &LineageClasses<'_>, + position: usize, + ) -> PyResult> { + let py: Python<'_> = classes.py(); + let (resource_type, transform, confidence) = match &self.parts[position] { + EdgeParts::Given(edge) => return Ok(edge.clone_ref(py)), + EdgeParts::Facts { + resource_type, + transform, + confidence, + } => (resource_type, transform, confidence), + }; + let first: usize = self.index.first_equal[position]; + if let Some(edge) = &self.materialised[first] { + return Ok(edge.clone_ref(py)); + } + let key: &EdgeKey = &self.keys[position]; + let edge: Py = classes + .edge( + classes.qualified(resource_type.bind(py), &key.source_name, &key.source_column)?, + &classes.qualified( + &classes.model_resource, + &key.target_model, + &key.target_column, + )?, + transform.bind(py), + confidence.bind(py), + )? + .unbind(); + self.materialised[first] = Some(edge.clone_ref(py)); + Ok(edge) + } +} + +pub(crate) fn empty_key() -> EdgeKey { + EdgeKey { + source_type: String::new(), + source_name: String::new(), + source_column: String::new(), + target_model: String::new(), + target_column: String::new(), + transform: String::new(), + confidence: String::new(), + } +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +/// Edge records accumulated in model order. +#[derive(Default)] +pub(crate) struct RecordBuilder { + pub(crate) keys: Vec, + pub(crate) parts: Vec, +} + +impl RecordBuilder { + /// Records of compact facts, or of plain column facts, targeting `model_name`. + pub(crate) fn push_facts( + &mut self, + classes: &LineageClasses<'_>, + model_name: &str, + columns: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if !columns.is_instance(&classes.compact_facts)? { + return self.push_columns(model_name, columns); + } + let py: Python<'_> = columns.py(); + let pool: Bound<'_, PyAny> = native_field(columns, "string_pool")?; + let remapped: bool = native_field(columns, "resource_names")?.is_truthy()? + || native_field(columns, "resource_name_indexes")?.is_truthy()?; + let mut members = CodeMembers::default(); + for row in native_field(columns, "rows")?.try_iter()? { + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row?.extract()?; + let output_column: String = pool.get_item(name_index)?.extract()?; + for source in sources.try_iter()? { + let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + source?.extract()?; + let resource_type: Bound<'_, PyAny> = pool.get_item(kind)?; + let resource_name: String = if remapped { + columns.call_method1("resource_name", (name,))?.extract()? + } else { + pool.get_item(name)?.extract()? + }; + let transform: Py = members.transform(columns, transform_code)?; + let confidence: Py = members.confidence(columns, confidence_code)?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: resource_name, + source_column: pool.get_item(column)?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(transform.bind(py))?, + confidence: text(confidence.bind(py))?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform, + confidence, + }); + } + } + Ok(()) + } + + /// Records of columns with `upstream_columns`, `transform_kind` and `confidence`. + pub(crate) fn push_columns( + &mut self, + model_name: &str, + columns: &Bound<'_, PyAny>, + ) -> PyResult<()> { + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let output_column: String = native_field(&column, "output_column")?.extract()?; + let transform: Bound<'_, PyAny> = native_field(&column, "transform_kind")?; + let confidence: Bound<'_, PyAny> = native_field(&column, "confidence")?; + for source in native_field(&column, "upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let resource_type: Bound<'_, PyAny> = native_field(&source, "resource_type")?; + self.keys.push(EdgeKey { + source_type: text(&resource_type)?, + source_name: native_field(&source, "resource_name")?.extract()?, + source_column: native_field(&source, "column_name")?.extract()?, + target_model: model_name.to_owned(), + target_column: output_column.clone(), + transform: text(&transform)?, + confidence: text(&confidence)?, + }); + self.parts.push(EdgeParts::Facts { + resource_type: resource_type.unbind(), + transform: transform.clone().unbind(), + confidence: confidence.clone().unbind(), + }); + } + } + Ok(()) + } +} + +/// Looks up the enum member of one native code. +type CodeLookup = fn(Python<'_>, usize) -> PyResult>; + +/// The transform kind and confidence members of native codes, looked up once per code. +#[derive(Default)] +struct CodeMembers { + members: HashMap<(&'static str, i64), Py>, +} + +impl CodeMembers { + fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts.py(), ("transform_kind", transform_kind_member), code) + } + + fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { + self.member(facts.py(), ("confidence", confidence_member), code) + } + + fn member( + &mut self, + py: Python<'_>, + (kind, lookup): (&'static str, CodeLookup), + code: i64, + ) -> PyResult> { + if let Some(member) = self.members.get(&(kind, code)) { + return Ok(member.clone_ref(py)); + } + let member: Py = lookup(py, usize::try_from(code)?)?; + let _ = self.members.insert((kind, code), member.clone_ref(py)); + Ok(member) + } +} + +/// The edge count of compact facts or plain column facts. +pub(crate) fn fact_edge_count( + classes: &LineageClasses<'_>, + columns: &Bound<'_, PyAny>, +) -> PyResult { + if !columns.is_instance(&classes.compact_facts)? { + return upstream_count(columns); + } + let mut count: usize = 0; + for row in native_field(columns, "rows")?.try_iter()? { + count += row?.get_item(3)?.len()?; + } + Ok(count) +} + +pub(crate) fn upstream_count(columns: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = 0; + for column in columns.try_iter()? { + count += native_field(&column?, "upstream_columns")?.len()?; + } + Ok(count) +} + +/// `ModelColumnLineage` of compact or plain column facts. +pub(crate) fn model_from_facts<'py>( + classes: &LineageClasses<'py>, + model_name: &str, + columns: &Bound<'py, PyAny>, + has_star: bool, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut lineage_columns: Vec> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let mut sources: Vec> = Vec::new(); + for source in native_field(&column, "upstream_columns")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + sources.push(classes.lineage_source.call1(( + native_field(&source, "resource_type")?, + native_field(&source, "resource_name")?, + native_field(&source, "column_name")?, + ))?); + } + lineage_columns.push(classes.column_lineage.call1(( + native_field(&column, "output_column")?, + native_field(&column, "transform_kind")?, + py.None(), + PyTuple::new(py, sources)?, + &classes.unknown_nullability, + native_field(&column, "confidence")?, + ))?); + } + classes.model(model_name, lineage_columns, has_star) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/mod.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/mod.rs new file mode 100644 index 0000000000..d4bd34255e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod column_lineage_graph; +pub(crate) mod project_lineage; +pub(crate) mod project_schema; +pub(crate) mod relation_fingerprint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_lineage.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs rename to crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_lineage.rs index da32d00298..1097e6e5ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_lineage.rs @@ -3,16 +3,12 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::HashMap; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use sqlbuild_analysis::lineage::main::build_fast_lineage::build_fast_lineage; use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; -use sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses; use sqlbuild_analysis::lineage::models::{ DirectSemanticUse, FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, RichLineageOutcome, RichLineageRequest, @@ -21,14 +17,15 @@ use sqlbuild_analysis::lineage_graph::main::analysis_dialect::analysis_dialect; use sqlbuild_analysis::lineage_graph::models::AnalysisDialect; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::lineage::column_lineage_graph::{ - CompactColumns, CompactModel, ProjectColumnLineage, -}; -use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_lineage_event}; -use crate::bindings::_helpers::lineage::project_schema::{ - fast_schema, inferred_names, rich_schema, -}; -use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; +use crate::lineage::_helpers::graph::project_schema::fast_schema; +use crate::lineage::_helpers::graph::project_schema::inferred_names; +use crate::lineage::_helpers::graph::project_schema::rich_schema; +use crate::lineage::_helpers::lineage::lineage_objects::log_lineage_event; +use crate::lineage::models::CompactColumns; +use crate::lineage::models::CompactModel; +use crate::lineage::models::LineageClasses; +use crate::lineage::models::ProjectColumnLineage; +use crate::project_assembly::main::model_placeholders::model_placeholders; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -37,74 +34,11 @@ use sqlbuild_python_project::analysis::models::ProjectCatalog; use sqlbuild_python_project::compiled_views::main::compiled_models::star_flags::star_flags; use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; -const FAST_MODE: &str = "fast"; -const RICH_MODE: &str = "rich"; -const GENERIC_DIALECT: &str = "generic"; +pub(crate) const FAST_MODE: &str = "fast"; +pub(crate) const RICH_MODE: &str = "rich"; +pub(crate) const GENERIC_DIALECT: &str = "generic"; -/// The project's column lineage graph in `mode`, or `None` without SQL analysis. -#[pyfunction] -pub(crate) fn build_project_column_lineage<'py>( - project: &Bound<'py, PyAny>, - dialect: Option<&Bound<'py, PyAny>>, - mode: &Bound<'py, PyAny>, - model_names: Option<&Bound<'py, PyAny>>, -) -> PyResult>> { - let mode: String = mode.str()?.extract()?; - if mode != FAST_MODE && mode != RICH_MODE { - return Ok(None); - } - let classes: LineageClasses<'py> = LineageClasses::new(project.py())?; - if !native_field(project, "settings")? - .getattr("sql_analysis")? - .is_truthy()? - { - return Ok(None); - } - let models: Vec> = selected_models(project, model_names)?; - let lineage: Bound<'py, PyAny> = if mode == FAST_MODE { - fast_lineage(&classes, project, dialect, &models)? - } else { - rich_lineage(&classes, project, dialect, &models)? - }; - Ok(Some(lineage.unbind())) -} - -/// Direct non-projection semantic column uses of the requested models. -#[pyfunction] -pub(crate) fn build_direct_semantic_uses<'py>( - project: &Bound<'py, PyAny>, - dialect: Option<&str>, - model_names: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project.py(); - let classes: LineageClasses<'py> = LineageClasses::new(py)?; - let schema = rich_schema(project)?; - let mut models: Vec<(String, String, String)> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let name: Bound<'py, PyAny> = native_field(&model, "name")?; - if model_names.contains(&name)? { - models.push(( - name.extract()?, - native_field(&model, "query_sql")?.extract()?, - native_field(&model, "authored_sql")?.extract()?, - )); - } - } - let dialect: &str = dialect - .filter(|name| !name.is_empty()) - .unwrap_or(GENERIC_DIALECT); - let uses: Vec = py - .compiler_detach(|| direct_semantic_uses(&schema, &models, dialect)) - .map_err(compiler_error)?; - let objects: Vec> = uses - .into_iter() - .map(|usage| semantic_use(&classes, usage)) - .collect::>()?; - PyTuple::new(py, objects) -} - -fn semantic_use<'py>( +pub(crate) fn semantic_use<'py>( classes: &LineageClasses<'py>, usage: DirectSemanticUse, ) -> PyResult> { @@ -126,7 +60,7 @@ fn semantic_use<'py>( )) } -fn selected_models<'py>( +pub(crate) fn selected_models<'py>( project: &Bound<'py, PyAny>, model_names: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { @@ -159,7 +93,7 @@ enum FastFacts<'py> { Native(Bound<'py, PyAny>, usize), } -fn fast_lineage<'py>( +pub(crate) fn fast_lineage<'py>( classes: &LineageClasses<'py>, project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, @@ -342,7 +276,7 @@ fn unknown_dialect_error(py: Python<'_>, dialect: &str) -> PyErr { ) } -fn rich_lineage<'py>( +pub(crate) fn rich_lineage<'py>( classes: &LineageClasses<'py>, project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, @@ -433,9 +367,3 @@ fn polyglot_dialect( } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_project_column_lineage, module)?)?; - module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_schema.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs rename to crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_schema.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/relation_fingerprint.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs rename to crates/sqlbuild-python-compile/src/lineage/_helpers/graph/relation_fingerprint.rs index 052795bb04..806ffa8c44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/graph/relation_fingerprint.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::lineage::main::relation_fingerprint::relation_fingerprint; use sqlbuild_analysis::lineage::models::InterruptedListingPolicy; @@ -33,8 +33,3 @@ pub(crate) fn relation_lineage_fingerprint( )) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(relation_lineage_fingerprint, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs rename to crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs index 49a3cb18d0..7f53bc803a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs @@ -5,14 +5,14 @@ use std::collections::{HashMap, HashSet}; use std::hash::Hash; use std::path::{Path, PathBuf}; -use crate::bindings::_helpers::lineage::relation_fingerprint::relation_lineage_fingerprint; +use crate::lineage::_helpers::graph::relation_fingerprint::relation_lineage_fingerprint; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const CACHE_SCHEMA_VERSION: &str = "1"; -const CACHE_ALGORITHM_VERSION: &str = "relation-lineage-graph-v4"; +pub(crate) const CACHE_SCHEMA_VERSION: &str = "1"; +pub(crate) const CACHE_ALGORITHM_VERSION: &str = "relation-lineage-graph-v4"; const CACHE_RELATIVE_PATH: [&str; 5] = [ "target", "cache", @@ -20,7 +20,7 @@ const CACHE_RELATIVE_PATH: [&str; 5] = [ "v1", "structural-graph.sqlite3", ]; -const CACHE_MAX_BYTES: u64 = 50_000_000; +pub(crate) const CACHE_MAX_BYTES: u64 = 50_000_000; const CACHE_MAX_ROWS: usize = 1_000_000; const SQLITE_TIMEOUT_SECONDS: f64 = 5.0; /// The first Python whose `rglob` skips a directory listing that fails mid-way. @@ -94,13 +94,13 @@ positional_record! { } type TripleRow = (String, String, String); -fn cache_path(project_dir: &Path) -> PathBuf { +pub(crate) fn cache_path(project_dir: &Path) -> PathBuf { CACHE_RELATIVE_PATH .iter() .fold(project_dir.to_path_buf(), |path, part| path.join(part)) } -fn skips_cache(py: Python<'_>, error: &PyErr) -> PyResult { +pub(crate) fn skips_cache(py: Python<'_>, error: &PyErr) -> PyResult { let database_error: Bound<'_, PyAny> = py.import("sqlite3")?.getattr("DatabaseError")?; if error.is_instance(py, &database_error) { return Ok(true); @@ -163,7 +163,7 @@ pub(crate) fn lineage_fingerprint( relation_lineage_fingerprint(py, project_dir.extract()?, prefix, uncacheable) } -fn rows<'py, T: pyo3::FromPyObject<'py>>( +pub(crate) fn rows<'py, T: pyo3::FromPyObject<'py>>( connection: &Bound<'py, PyAny>, sql: &str, ) -> PyResult> { @@ -173,7 +173,7 @@ fn rows<'py, T: pyo3::FromPyObject<'py>>( .extract() } -fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { let resources: Vec = rows( connection, "SELECT resource_type, resource_name, relative_path, qualified_name FROM resources", @@ -298,7 +298,11 @@ impl Groups { } } -fn connect<'py>(py: Python<'py>, target: &str, uri: bool) -> PyResult> { +pub(crate) fn connect<'py>( + py: Python<'py>, + target: &str, + uri: bool, +) -> PyResult> { let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item("timeout", SQLITE_TIMEOUT_SECONDS)?; if uri { @@ -309,48 +313,6 @@ fn connect<'py>(py: Python<'py>, target: &str, uri: bool) -> PyResult, - project_dir: &Path, - fingerprint: &str, -) -> PyResult> { - let path: PathBuf = cache_path(project_dir); - match std::fs::metadata(&path) { - Ok(metadata) if metadata.len() <= CACHE_MAX_BYTES => {} - _ => return Ok(None), - } - let read = || -> PyResult> { - let connection: Bound<'_, PyAny> = - connect(py, &format!("file:{}?mode=ro", path.display()), true)?; - let result = (|| -> PyResult> { - let metadata: Vec<(String, String)> = - rows(&connection, "SELECT key, value FROM metadata")?; - let mut metadata: HashMap = metadata.into_iter().collect(); - let expected: [(&str, &str); 3] = [ - ("algorithm_version", CACHE_ALGORITHM_VERSION), - ("input_fingerprint", fingerprint), - ("schema_version", CACHE_SCHEMA_VERSION), - ]; - let matches: bool = metadata.len() == expected.len() - && expected - .iter() - .all(|(key, value)| metadata.remove(*key).as_deref() == Some(*value)); - if !matches { - return Ok(None); - } - read_index(&connection) - })(); - connection.call_method0("close")?; - result - }; - match read() { - Ok(index) => Ok(index), - Err(error) if skips_cache(py, &error)? => Ok(None), - Err(error) => Err(error), - } -} - fn insert(connection: &Bound<'_, PyAny>, sql: &str, rows: Bound<'_, PyList>) -> PyResult<()> { connection.call_method1("executemany", (sql, rows))?; Ok(()) @@ -360,7 +322,7 @@ fn key_parts(key: &LineageKey) -> (&str, &str) { (key.resource_type.as_str(), key.name.as_str()) } -fn write_index( +pub(crate) fn write_index( connection: &Bound<'_, PyAny>, fingerprint: &str, index: &LineageIndex, @@ -457,46 +419,3 @@ fn write_index( connection.call_method0("commit")?; Ok(()) } - -/// Publish `index` atomically; a failure only skips the cache. -pub(crate) fn write_lineage_cache( - py: Python<'_>, - project_dir: &Path, - fingerprint: &str, - index: &LineageIndex, -) -> PyResult<()> { - let path: PathBuf = cache_path(project_dir); - let Some(directory) = path.parent() else { - return Ok(()); - }; - if std::fs::create_dir_all(directory).is_err() { - return Ok(()); - } - let mut temporary: Option = None; - let mut write = || -> PyResult<()> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("prefix", ".structural-graph-")?; - options.set_item("dir", directory)?; - let (descriptor, name): (i64, PathBuf) = py - .import("tempfile")? - .getattr("mkstemp")? - .call((), Some(&options))? - .extract()?; - py.import("os")?.getattr("close")?.call1((descriptor,))?; - temporary = Some(name.clone()); - let connection: Bound<'_, PyAny> = connect(py, &name.to_string_lossy(), false)?; - let written: PyResult<()> = write_index(&connection, fingerprint, index); - connection.call_method0("close")?; - written?; - std::fs::rename(&name, &path)?; - Ok(()) - }; - let outcome: PyResult<()> = write(); - if let Some(name) = temporary { - let _ = std::fs::remove_file(name); - } - match outcome { - Err(error) if !skips_cache(py, &error)? => Err(error), - _ => Ok(()), - } -} diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs new file mode 100644 index 0000000000..b2d7594d35 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs @@ -0,0 +1,126 @@ +//! The relation lineage index read from a project graph, and its Python selector views. + +use crate::lineage::main::lineage_key::lineage_key; +use crate::lineage::main::lineage_keys::lineage_keys; +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_analysis::lineage_command::models::{LineageKey, LineageNode}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::types::RecordId; + +/// Python `CompiledObjectKey`s for lineage keys, one object per key. +pub(crate) struct PythonKeys<'py> { + class: Bound<'py, PyAny>, + made: HashMap>, +} + +impl<'py> PythonKeys<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + class: record_class(py, RecordId::CompiledObjectKey)?, + made: HashMap::new(), + }) + } + + pub(crate) fn key(&mut self, key: &LineageKey) -> PyResult> { + if let Some(made) = self.made.get(key) { + return Ok(made.clone()); + } + let made: Bound<'py, PyAny> = self + .class + .call1((key.resource_type.as_str(), key.name.as_str()))?; + self.made.insert(key.clone(), made.clone()); + Ok(made) + } + + pub(crate) fn tuple(&mut self, keys: &[LineageKey]) -> PyResult> { + let items: Vec> = keys + .iter() + .map(|key| self.key(key)) + .collect::>()?; + PyTuple::new(self.class.py(), items) + } +} + +pub(crate) fn ordered_edges( + edges: &Bound<'_, PyAny>, +) -> PyResult)>> { + let mut ordered: Vec<(LineageKey, Vec)> = Vec::new(); + for item in mapping_items(edges)?.try_iter()? { + let (key, targets): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + ordered.push((lineage_key(&key)?, lineage_keys(&targets)?)); + } + Ok(ordered) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +/// A source entry's `database.schema.table`, as far as it is set. +fn source_relation_name(entry: &Bound<'_, PyAny>) -> PyResult> { + let part = |name: &str| -> PyResult> { + match native_field(entry, name) { + Ok(value) if !value.is_none() => Ok(Some(value.extract()?)), + _ => Ok(None), + } + }; + let Some(table) = part("table")? else { + return Ok(None); + }; + Ok(Some(match (part("database")?, part("schema")?) { + (Some(database), Some(schema)) => format!("{database}.{schema}.{table}"), + (_, Some(schema)) => format!("{schema}.{table}"), + _ => table, + })) +} + +/// Display facts of every model, seed and source, models first as Python looks them up. +pub(crate) fn project_nodes( + project: &Bound<'_, PyAny>, +) -> PyResult> { + let mut nodes: HashMap = HashMap::new(); + let mut add = |node: LineageNode| { + nodes.entry(node.key.clone()).or_insert(node); + }; + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + add(LineageNode { + key: lineage_key(&native_field(&model, "key")?)?, + relative_path: optional_text(native_field(&model, "relative_path")?)?, + qualified_name: optional_text( + native_field(&model, "destination")?.getattr("qualified_name")?, + )?, + }); + } + for seed in native_field(project, "seeds")?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + add(LineageNode { + key: lineage_key(&native_field(&seed, "key")?)?, + relative_path: optional_text( + native_field(&seed, "seed_file")?.getattr("relative_path")?, + )?, + qualified_name: optional_text( + native_field(&seed, "destination")?.getattr("qualified_name")?, + )?, + }); + } + for source in native_field(project, "sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + add(LineageNode { + key: lineage_key(&native_field(&source, "key")?)?, + relative_path: optional_text( + native_field(&source, "source_file")?.getattr("relative_path")?, + )?, + qualified_name: source_relation_name(&native_field(&source, "source_entry")?)?, + }); + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_objects.rs new file mode 100644 index 0000000000..d14110e66b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_objects.rs @@ -0,0 +1,30 @@ +//! The Python lineage models and enums native lineage builds. + +use sqlbuild_host::entries::models::ExecutorHost; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +const LOGGING_MODULE: &str = "logging"; +const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; + +/// A structured debug event on the `sqlbuild.lineage` logger for one model. +pub(crate) fn log_lineage_event( + py: Python<'_>, + message: &str, + model_name: &str, + error: Option<&str>, +) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import(LOGGING_MODULE)? + .getattr("getLogger")? + .call1((LINEAGE_LOGGER,))?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("logger", logger)?; + fields.set_item("message", message)?; + fields.set_item("sqlbuild_model", model_name)?; + fields.set_item("sqlbuild_error", error)?; + let _ = host_entry(py, ExecutorHost::LogDebugEvent)?.call((), Some(&fields))?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/mod.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/mod.rs new file mode 100644 index 0000000000..8c875cc8df --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod lineage_cache; +pub(crate) mod lineage_index; +pub(crate) mod lineage_objects; diff --git a/crates/sqlbuild-python-compile/src/lineage/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/lineage/_helpers/mod.rs new file mode 100644 index 0000000000..97fefd1e96 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod graph; +pub(crate) mod lineage; diff --git a/crates/sqlbuild-python-compile/src/lineage/main/build_direct_semantic_uses.rs b/crates/sqlbuild-python-compile/src/lineage/main/build_direct_semantic_uses.rs new file mode 100644 index 0000000000..ac8d5579d0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/build_direct_semantic_uses.rs @@ -0,0 +1,49 @@ +//! Direct non-projection semantic column uses of the requested models. + +use crate::lineage::_helpers::graph::project_lineage::GENERIC_DIALECT; +use crate::lineage::_helpers::graph::project_lineage::semantic_use; +use crate::lineage::_helpers::graph::project_schema::rich_schema; +use crate::lineage::models::LineageClasses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses; +use sqlbuild_analysis::lineage::models::DirectSemanticUse; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::types::CompilerDetach; + +/// Direct non-projection semantic column uses of the requested models. +#[pyfunction] +pub fn build_direct_semantic_uses<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&str>, + model_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let classes: LineageClasses<'py> = LineageClasses::new(py)?; + let schema = rich_schema(project)?; + let mut models: Vec<(String, String, String)> = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: Bound<'py, PyAny> = native_field(&model, "name")?; + if model_names.contains(&name)? { + models.push(( + name.extract()?, + native_field(&model, "query_sql")?.extract()?, + native_field(&model, "authored_sql")?.extract()?, + )); + } + } + let dialect: &str = dialect + .filter(|name| !name.is_empty()) + .unwrap_or(GENERIC_DIALECT); + let uses: Vec = py + .compiler_detach(|| direct_semantic_uses(&schema, &models, dialect)) + .map_err(compiler_error)?; + let objects: Vec> = uses + .into_iter() + .map(|usage| semantic_use(&classes, usage)) + .collect::>()?; + PyTuple::new(py, objects) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/build_project_column_lineage.rs b/crates/sqlbuild-python-compile/src/lineage/main/build_project_column_lineage.rs new file mode 100644 index 0000000000..54791dc29e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/build_project_column_lineage.rs @@ -0,0 +1,39 @@ +//! The project's column lineage graph in `mode`, or `None` without SQL analysis. + +use crate::lineage::_helpers::graph::project_lineage::FAST_MODE; +use crate::lineage::_helpers::graph::project_lineage::RICH_MODE; +use crate::lineage::_helpers::graph::project_lineage::fast_lineage; +use crate::lineage::_helpers::graph::project_lineage::rich_lineage; +use crate::lineage::_helpers::graph::project_lineage::selected_models; +use crate::lineage::models::LineageClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's column lineage graph in `mode`, or `None` without SQL analysis. +#[pyfunction] +pub fn build_project_column_lineage<'py>( + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + mode: &Bound<'py, PyAny>, + model_names: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let mode: String = mode.str()?.extract()?; + if mode != FAST_MODE && mode != RICH_MODE { + return Ok(None); + } + let classes: LineageClasses<'py> = LineageClasses::new(project.py())?; + if !native_field(project, "settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(None); + } + let models: Vec> = selected_models(project, model_names)?; + let lineage: Bound<'py, PyAny> = if mode == FAST_MODE { + fast_lineage(&classes, project, dialect, &models)? + } else { + rich_lineage(&classes, project, dialect, &models)? + }; + Ok(Some(lineage.unbind())) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/graph_index.rs b/crates/sqlbuild-python-compile/src/lineage/main/graph_index.rs new file mode 100644 index 0000000000..012db0b955 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/graph_index.rs @@ -0,0 +1,56 @@ +//! The lineage index of a compiled `ProjectGraph`; every named key has a node. + +use crate::lineage::_helpers::lineage::lineage_index::ordered_edges; +use crate::lineage::_helpers::lineage::lineage_index::project_nodes; +use crate::lineage::main::lineage_key::lineage_key; +use crate::lineage::main::lineage_keys::lineage_keys; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; + +/// The lineage index of a compiled `ProjectGraph`; every named key has a node. +pub fn graph_index(graph: &Bound<'_, PyAny>) -> PyResult { + let views: Bound<'_, PyAny> = native_field(graph, "lineage_views")?; + let mut all_keys: Vec<(String, LineageKey)> = Vec::new(); + for item in native_field(&views, "all_keys")? + .call_method0("items")? + .try_iter()? + { + let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; + all_keys.push((name, lineage_key(&key)?)); + } + let mut tags: Vec<(String, Vec)> = Vec::new(); + for item in native_field(&views, "tag_index")? + .call_method0("items")? + .try_iter()? + { + let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; + tags.push((tag, lineage_keys(&keys)?)); + } + let mut paths: Vec<(LineageKey, String)> = Vec::new(); + for item in native_field(&views, "path_index")? + .call_method0("items")? + .try_iter()? + { + let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; + paths.push((lineage_key(&key)?, folder)); + } + let mut nodes: HashMap = + project_nodes(&native_field(graph, "project")?)?; + for (_, key) in &all_keys { + nodes.entry(key.clone()).or_insert_with(|| LineageNode { + key: key.clone(), + relative_path: None, + qualified_name: None, + }); + } + Ok(LineageIndex { + nodes, + upstream: ordered_edges(&native_field(&views, "upstream_deps")?)?, + downstream: ordered_edges(&native_field(&views, "downstream_deps")?)?, + tags, + paths, + all_keys, + }) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/index_views.rs b/crates/sqlbuild-python-compile/src/lineage/main/index_views.rs new file mode 100644 index 0000000000..bf6468febf --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/index_views.rs @@ -0,0 +1,47 @@ +//! `LineageGraphViews` of an index, for the planner's selector functions. + +use crate::lineage::_helpers::lineage::lineage_index::PythonKeys; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// `LineageGraphViews` of an index, for the planner's selector functions. +pub fn index_views<'py>(py: Python<'py>, index: &LineageIndex) -> PyResult> { + let mut keys: PythonKeys<'py> = PythonKeys::new(py)?; + let edges = |pairs: &[(LineageKey, Vec)], + keys: &mut PythonKeys<'py>| + -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, targets) in pairs { + dict.set_item(keys.key(key)?, keys.tuple(targets)?)?; + } + Ok(dict) + }; + let upstream: Bound<'py, PyDict> = edges(&index.upstream, &mut keys)?; + let downstream: Bound<'py, PyDict> = edges(&index.downstream, &mut keys)?; + let tags: Bound<'py, PyDict> = PyDict::new(py); + for (tag, tagged) in &index.tags { + let items: Vec> = tagged + .iter() + .map(|key| keys.key(key)) + .collect::>()?; + tags.set_item(tag, PyFrozenSet::new(py, items)?)?; + } + let paths: Bound<'py, PyDict> = PyDict::new(py); + for (key, folder) in &index.paths { + paths.set_item(keys.key(key)?, folder)?; + } + let names: Bound<'py, PyDict> = PyDict::new(py); + for (name, key) in &index.all_keys { + names.set_item(name, keys.key(key)?)?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("upstream_deps", upstream)?; + options.set_item("downstream_deps", downstream)?; + options.set_item("tag_index", tags)?; + options.set_item("path_index", paths)?; + options.set_item("all_keys", names)?; + new_record(py, RecordId::LineageGraphViews, (), Some(&options)) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/lineage_fingerprint.rs b/crates/sqlbuild-python-compile/src/lineage/main/lineage_fingerprint.rs new file mode 100644 index 0000000000..ffb53f51b3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/lineage_fingerprint.rs @@ -0,0 +1,15 @@ +//! The cache key over authored inputs and `cli_vars`, or `None` when they cannot be hashed. + +use crate::lineage::_helpers::lineage::lineage_cache; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The cache key over authored inputs and `cli_vars`, or `None` when they cannot be hashed. +#[inline] +pub fn lineage_fingerprint( + project_dir: &Bound<'_, PyAny>, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult> { + lineage_cache::lineage_fingerprint(project_dir, cli_vars) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/lineage_key.rs b/crates/sqlbuild-python-compile/src/lineage/main/lineage_key.rs new file mode 100644 index 0000000000..0843c4257b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/lineage_key.rs @@ -0,0 +1,13 @@ +//! A Python `CompiledObjectKey` as a lineage key. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::lineage_command::models::LineageKey; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A Python `CompiledObjectKey` as a lineage key. +pub fn lineage_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(LineageKey { + resource_type: native_field(key, "resource_type")?.str()?.extract()?, + name: native_field(key, "name")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/lineage_keys.rs b/crates/sqlbuild-python-compile/src/lineage/main/lineage_keys.rs new file mode 100644 index 0000000000..78afbef61f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/lineage_keys.rs @@ -0,0 +1,10 @@ +//! Lineage keys of every Python key in `keys`. + +use crate::lineage::main::lineage_key::lineage_key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::lineage_command::models::LineageKey; + +/// Lineage keys of every Python key in `keys`. +pub fn lineage_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| lineage_key(&key?)).collect() +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/mod.rs b/crates/sqlbuild-python-compile/src/lineage/main/mod.rs new file mode 100644 index 0000000000..ea46f73530 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/mod.rs @@ -0,0 +1,12 @@ +pub mod build_direct_semantic_uses; +pub mod build_project_column_lineage; +pub mod graph_index; +pub mod index_views; +pub mod lineage_fingerprint; +pub mod lineage_key; +pub mod lineage_keys; +pub mod read_lineage_cache; +pub mod register_column_lineage_graph; +pub mod register_project_lineage; +pub mod register_relation_fingerprint; +pub mod write_lineage_cache; diff --git a/crates/sqlbuild-python-compile/src/lineage/main/read_lineage_cache.rs b/crates/sqlbuild-python-compile/src/lineage/main/read_lineage_cache.rs new file mode 100644 index 0000000000..5d8e6f51a3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/read_lineage_cache.rs @@ -0,0 +1,56 @@ +//! A verified cached index, or `None` on any cache fault. + +use crate::lineage::_helpers::lineage::lineage_cache::CACHE_ALGORITHM_VERSION; +use crate::lineage::_helpers::lineage::lineage_cache::CACHE_MAX_BYTES; +use crate::lineage::_helpers::lineage::lineage_cache::CACHE_SCHEMA_VERSION; +use crate::lineage::_helpers::lineage::lineage_cache::cache_path; +use crate::lineage::_helpers::lineage::lineage_cache::connect; +use crate::lineage::_helpers::lineage::lineage_cache::read_index; +use crate::lineage::_helpers::lineage::lineage_cache::rows; +use crate::lineage::_helpers::lineage::lineage_cache::skips_cache; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::lineage_command::models::LineageIndex; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// A verified cached index, or `None` on any cache fault. +pub fn read_lineage_cache( + py: Python<'_>, + project_dir: &Path, + fingerprint: &str, +) -> PyResult> { + let path: PathBuf = cache_path(project_dir); + match std::fs::metadata(&path) { + Ok(metadata) if metadata.len() <= CACHE_MAX_BYTES => {} + _ => return Ok(None), + } + let read = || -> PyResult> { + let connection: Bound<'_, PyAny> = + connect(py, &format!("file:{}?mode=ro", path.display()), true)?; + let result = (|| -> PyResult> { + let metadata: Vec<(String, String)> = + rows(&connection, "SELECT key, value FROM metadata")?; + let mut metadata: HashMap = metadata.into_iter().collect(); + let expected: [(&str, &str); 3] = [ + ("algorithm_version", CACHE_ALGORITHM_VERSION), + ("input_fingerprint", fingerprint), + ("schema_version", CACHE_SCHEMA_VERSION), + ]; + let matches: bool = metadata.len() == expected.len() + && expected + .iter() + .all(|(key, value)| metadata.remove(*key).as_deref() == Some(*value)); + if !matches { + return Ok(None); + } + read_index(&connection) + })(); + connection.call_method0("close")?; + result + }; + match read() { + Ok(index) => Ok(index), + Err(error) if skips_cache(py, &error)? => Ok(None), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/register_column_lineage_graph.rs b/crates/sqlbuild-python-compile/src/lineage/main/register_column_lineage_graph.rs new file mode 100644 index 0000000000..813da74318 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/register_column_lineage_graph.rs @@ -0,0 +1,9 @@ +//! Register the column lineage graph functions and classes. + +use crate::lineage::models::ProjectColumnLineage; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the column lineage graph functions and classes. +pub fn register_column_lineage_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/register_project_lineage.rs b/crates/sqlbuild-python-compile/src/lineage/main/register_project_lineage.rs new file mode 100644 index 0000000000..8a6d125763 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/register_project_lineage.rs @@ -0,0 +1,13 @@ +//! Register the project lineage functions and classes. + +use crate::lineage::main::build_direct_semantic_uses::build_direct_semantic_uses; +use crate::lineage::main::build_project_column_lineage::build_project_column_lineage; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the project lineage functions and classes. +pub fn register_project_lineage(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_project_column_lineage, module)?)?; + module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/register_relation_fingerprint.rs b/crates/sqlbuild-python-compile/src/lineage/main/register_relation_fingerprint.rs new file mode 100644 index 0000000000..561a4e51b8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/register_relation_fingerprint.rs @@ -0,0 +1,11 @@ +//! Register the relation fingerprint functions and classes. + +use crate::lineage::_helpers::graph::relation_fingerprint::relation_lineage_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the relation fingerprint functions and classes. +pub fn register_relation_fingerprint(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(relation_lineage_fingerprint, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/lineage/main/write_lineage_cache.rs b/crates/sqlbuild-python-compile/src/lineage/main/write_lineage_cache.rs new file mode 100644 index 0000000000..0ab3c60e52 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/main/write_lineage_cache.rs @@ -0,0 +1,53 @@ +//! Publish `index` atomically; a failure only skips the cache. + +use crate::lineage::_helpers::lineage::lineage_cache::cache_path; +use crate::lineage::_helpers::lineage::lineage_cache::connect; +use crate::lineage::_helpers::lineage::lineage_cache::skips_cache; +use crate::lineage::_helpers::lineage::lineage_cache::write_index; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::lineage_command::models::LineageIndex; +use std::path::{Path, PathBuf}; + +/// Publish `index` atomically; a failure only skips the cache. +pub fn write_lineage_cache( + py: Python<'_>, + project_dir: &Path, + fingerprint: &str, + index: &LineageIndex, +) -> PyResult<()> { + let path: PathBuf = cache_path(project_dir); + let Some(directory) = path.parent() else { + return Ok(()); + }; + if std::fs::create_dir_all(directory).is_err() { + return Ok(()); + } + let mut temporary: Option = None; + let mut write = || -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("prefix", ".structural-graph-")?; + options.set_item("dir", directory)?; + let (descriptor, name): (i64, PathBuf) = py + .import("tempfile")? + .getattr("mkstemp")? + .call((), Some(&options))? + .extract()?; + py.import("os")?.getattr("close")?.call1((descriptor,))?; + temporary = Some(name.clone()); + let connection: Bound<'_, PyAny> = connect(py, &name.to_string_lossy(), false)?; + let written: PyResult<()> = write_index(&connection, fingerprint, index); + connection.call_method0("close")?; + written?; + std::fs::rename(&name, &path)?; + Ok(()) + }; + let outcome: PyResult<()> = write(); + if let Some(name) = temporary { + let _ = std::fs::remove_file(name); + } + match outcome { + Err(error) if !skips_cache(py, &error)? => Err(error), + _ => Ok(()), + } +} diff --git a/crates/sqlbuild-python-compile/src/lineage/mod.rs b/crates/sqlbuild-python-compile/src/lineage/mod.rs new file mode 100644 index 0000000000..308e320c60 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/mod.rs @@ -0,0 +1,5 @@ +//! Project column lineage: the lineage graph, the relation lineage index and cache, and the lineage command. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/lineage/models.rs b/crates/sqlbuild-python-compile/src/lineage/models.rs new file mode 100644 index 0000000000..d4b09dcd55 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/lineage/models.rs @@ -0,0 +1,520 @@ +//! Public records of lineage. + +use crate::lineage::_helpers::graph::column_lineage_graph::EdgeParts; +use crate::lineage::_helpers::graph::column_lineage_graph::EdgeRecords; +use crate::lineage::_helpers::graph::column_lineage_graph::RecordBuilder; +use crate::lineage::_helpers::graph::column_lineage_graph::empty_key; +use crate::lineage::_helpers::graph::column_lineage_graph::fact_edge_count; +use crate::lineage::_helpers::graph::column_lineage_graph::model_from_facts; +use crate::lineage::_helpers::graph::column_lineage_graph::upstream_count; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRefMut; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use pyo3::types::PyType; +use sqlbuild_analysis::lineage::models::LineageColumn; +use sqlbuild_analysis::lineage::models::LineageSource; +use sqlbuild_analysis::lineage_graph::models::EdgeIndex; +use sqlbuild_analysis::lineage_graph::models::EdgeKey; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::types::RecordId; +use std::collections::HashMap; + +/// Where a compact model's facts are: given, or projected from its compiled model on first read. +#[derive(Debug)] +pub enum CompactColumns { + Facts(Py), + /// A compiled model whose `fast_lineage_columns` are the facts, and their upstream count. + Model { + model: Py, + edges: usize, + }, +} + +/// One compact model's facts and root-star flag. +#[derive(Debug)] +pub struct CompactModel { + pub columns: CompactColumns, + pub has_star: bool, +} + +impl CompactModel { + /// The model's lineage facts as Python objects. + pub(crate) fn facts<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.columns { + CompactColumns::Facts(columns) => Ok(columns.bind(py).clone()), + CompactColumns::Model { model, .. } => { + native_field(model.bind(py), "fast_lineage_columns") + } + } + } +} + +#[pyclass(module = "sqlbuild._native", name = "ProjectColumnLineage")] +#[derive(Debug)] +pub struct ProjectColumnLineage { + pub(crate) models: Py, + pub(crate) compact: HashMap, + pub(crate) model_order: Vec, + pub(crate) edges: Option>, + pub(crate) records: Option, + pub(crate) edge_counts: HashMap, +} + +#[pymethods] +impl ProjectColumnLineage { + #[new] + #[pyo3(signature = (*, models, edges))] + pub(crate) fn new(models: &Bound<'_, PyAny>, edges: &Bound<'_, PyAny>) -> PyResult { + let mut keys: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); + for edge in edges.try_iter()? { + let edge: Bound<'_, PyAny> = edge?; + keys.push(EdgeKey { + source_name: native_field(&edge, "source")? + .getattr("resource_name")? + .extract()?, + target_model: native_field(&edge, "target")? + .getattr("resource_name")? + .extract()?, + ..empty_key() + }); + parts.push(EdgeParts::Given(edge.unbind())); + } + Ok(Self { + model_order: models + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + models: models.clone().unbind(), + compact: HashMap::new(), + edges: Some(edges.clone().unbind()), + records: Some(EdgeRecords::new(keys, parts)), + edge_counts: HashMap::new(), + }) + } + + /// Build indexed project lineage without expanding compact analysis facts. + #[classmethod] + #[pyo3(signature = (*, models, compact_models, model_order))] + pub(crate) fn from_fast_facts( + cls: &Bound<'_, PyType>, + models: &Bound<'_, PyAny>, + compact_models: &Bound<'_, PyDict>, + model_order: Vec, + ) -> PyResult { + let classes: LineageClasses<'_> = LineageClasses::new(cls.py())?; + let mut compact: HashMap = HashMap::new(); + for (name, entry) in compact_models.iter() { + let (columns, has_star): (Bound<'_, PyAny>, bool) = entry.extract()?; + let _ = compact.insert( + name.extract()?, + CompactModel { + columns: CompactColumns::Facts(columns.unbind()), + has_star, + }, + ); + } + Self::compacted(&classes, models, compact, model_order) + } + + /// Return model lineage, expanding compact model facts on first access. + #[getter] + pub fn models(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !slf.compact.is_empty() { + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + let models: Bound<'_, PyAny> = slf.models.bind(py).clone(); + for name in &slf.model_order { + let Some(compact) = slf.compact.get(name) else { + continue; + }; + if models.contains(name)? { + continue; + } + let lineage: Bound<'_, PyAny> = + model_from_facts(&classes, name, &compact.facts(py)?, compact.has_star)?; + models.set_item(name, lineage)?; + } + slf.compact.clear(); + } + Ok(slf.models.clone_ref(py)) + } + + /// Return all graph edges, expanding compact edge records on first access. + #[getter] + pub fn edges(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let py: Python<'_> = slf.py(); + if let Some(edges) = &slf.edges { + return Ok(edges.clone_ref(py)); + } + let classes: LineageClasses<'_> = LineageClasses::new(py)?; + slf.ensure_records(&classes)?; + let count: usize = slf.records.as_ref().map_or(0, |records| records.keys.len()); + let edges: Vec> = slf.materialised(&classes, 0..count)?; + let edges: Py = PyTuple::new(py, edges)?.into_any().unbind(); + slf.edges = Some(edges.clone_ref(py)); + Ok(edges) + } + + /// Return whether model lineage is available without expanding its columns. + #[pyo3(name = "has_model")] + pub(crate) fn contains_model( + &self, + py: Python<'_>, + model_name: &Bound<'_, PyAny>, + ) -> PyResult { + if self.models.bind(py).contains(model_name)? { + return Ok(true); + } + Ok(model_name + .extract::() + .is_ok_and(|name| self.compact.contains_key(&name))) + } + + /// Return whether model lineage retains unresolved root-star uncertainty. + pub(crate) fn model_has_star( + &self, + py: Python<'_>, + model_name: &Bound<'_, PyAny>, + ) -> PyResult { + if let Ok(name) = model_name.extract::() + && let Some(compact) = self.compact.get(&name) + { + return Ok(compact.has_star); + } + let model: Bound<'_, PyAny> = self.models.bind(py).call_method1("get", (model_name,))?; + if model.is_none() { + return Ok(false); + } + native_field(&model, "has_star")?.extract() + } + + /// Return a target model's direct edge count without expanding edge objects. + pub(crate) fn edge_count_targeting(&self, model_name: &str) -> usize { + match &self.records { + Some(records) => records + .index + .by_target_model + .get(model_name) + .map_or(0, Vec::len), + None => self.edge_counts.get(model_name).copied().unwrap_or(0), + } + } + + /// Return edges whose target model is `model_name`. + pub(crate) fn edges_targeting<'py>( + mut slf: PyRefMut<'py, Self>, + model_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = slf.positions(|index| index.by_target_model.get(model_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return edges whose source resource is `resource_name`. + pub(crate) fn edges_sourced_from<'py>( + mut slf: PyRefMut<'py, Self>, + resource_name: &str, + ) -> PyResult> { + let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; + slf.ensure_records(&classes)?; + let positions: Vec = + slf.positions(|index| index.by_source_resource.get(resource_name)); + let edges: Vec> = slf.materialised(&classes, positions)?; + PyTuple::new(slf.py(), edges) + } + + /// Return direct downstream consumers of `resource_name.column_name`. + #[pyo3(signature = (*, resource_name, column_name))] + pub fn column_consumers<'py>( + slf: &Bound<'py, Self>, + resource_name: &str, + column_name: &Bound<'py, PyAny>, + ) -> PyResult> { + let sourced: Bound<'py, PyAny> = + slf.call_method1("edges_sourced_from", (resource_name,))?; + let mut consumers: Vec> = Vec::new(); + for edge in sourced.try_iter()? { + let edge: Bound<'py, PyAny> = edge?; + if native_field(&edge, "source")? + .getattr("column_name")? + .eq(column_name)? + { + consumers.push(edge); + } + } + PyTuple::new(slf.py(), consumers) + } + + pub(crate) fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let models: Bound<'_, PyString> = native_field(slf, "models")?.repr()?; + let edges: Bound<'_, PyString> = native_field(slf, "edges")?.repr()?; + Ok(format!( + "ProjectColumnLineage(models={models}, edges={edges})" + )) + } + + pub(crate) fn __eq__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !other.is_instance_of::() { + return Ok(py.NotImplemented()); + } + let equal: bool = native_field(slf, "models")?.eq(native_field(other, "models")?)? + && native_field(slf, "edges")?.eq(native_field(other, "edges")?)?; + Ok(pyo3::types::PyBool::new(py, equal) + .to_owned() + .into_any() + .unbind()) + } +} + +impl ProjectColumnLineage { + /// Indexed project lineage of built `models` and `compact` facts, in `model_order`. + pub fn compacted( + classes: &LineageClasses<'_>, + models: &Bound<'_, PyAny>, + compact: HashMap, + model_order: Vec, + ) -> PyResult { + let py: Python<'_> = models.py(); + let mut edge_counts: HashMap = HashMap::new(); + for name in &model_order { + if let Some(model) = compact.get(name) { + let count: usize = match &model.columns { + CompactColumns::Facts(columns) => fact_edge_count(classes, columns.bind(py))?, + CompactColumns::Model { edges, .. } => *edges, + }; + let _ = edge_counts.insert(name.clone(), count); + } + } + for item in mapping_items(models)?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); + } + Ok(Self { + models: models.clone().unbind(), + compact, + model_order, + edges: None, + records: None, + edge_counts, + }) + } + + /// Build fast edge records from the compact facts, or the expanded models, in model order. + pub(crate) fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { + if self.records.is_some() { + return Ok(()); + } + let py: Python<'_> = classes.py(); + let mut builder = RecordBuilder::default(); + let models: Bound<'_, PyAny> = self.models.bind(py).clone(); + for name in &self.model_order { + if let Some(compact) = self.compact.get(name) { + builder.push_facts(classes, name, &compact.facts(py)?)?; + continue; + } + let model: Bound<'_, PyAny> = mapping_get(&models, name)?; + if model.is_none() { + continue; + } + builder.push_columns(name, &native_field(&model, "columns")?)?; + } + self.records = Some(EdgeRecords::new(builder.keys, builder.parts)); + Ok(()) + } + + /// Every edge's fields in model order, from the compact records, without building Python edges. + pub fn edge_keys(&mut self, py: Python<'_>) -> PyResult> { + self.ensure_records(&LineageClasses::new(py)?)?; + Ok(self + .records + .as_ref() + .map(|records| records.keys.clone()) + .unwrap_or_default()) + } + + pub(crate) fn positions( + &self, + select: impl Fn(&EdgeIndex) -> Option<&Vec>, + ) -> Vec { + self.records + .as_ref() + .and_then(|records| select(&records.index)) + .cloned() + .unwrap_or_default() + } + + pub(crate) fn materialised( + &mut self, + classes: &LineageClasses<'_>, + positions: impl IntoIterator, + ) -> PyResult>> { + let Some(records) = self.records.as_mut() else { + return Ok(Vec::new()); + }; + positions + .into_iter() + .map(|position| records.edge(classes, position)) + .collect() + } +} + +/// The lineage model classes and enums, imported once per call. +#[derive(Debug)] +pub struct LineageClasses<'py> { + pub qualified_column: Bound<'py, PyAny>, + pub lineage_source: Bound<'py, PyAny>, + pub lineage_edge: Bound<'py, PyAny>, + pub column_lineage: Bound<'py, PyAny>, + pub model_lineage: Bound<'py, PyAny>, + pub semantic_use: Bound<'py, PyAny>, + pub column_fact: Bound<'py, PyAny>, + pub source_fact: Bound<'py, PyAny>, + pub compact_facts: Bound<'py, PyAny>, + pub transform_kind: Bound<'py, PyAny>, + pub confidence: Bound<'py, PyAny>, + pub nullability: Bound<'py, PyAny>, + pub analysis_dialect: Bound<'py, PyAny>, + pub resource_type: Bound<'py, PyAny>, + pub model_resource: Bound<'py, PyAny>, + pub unknown_nullability: Bound<'py, PyAny>, +} + +impl<'py> LineageClasses<'py> { + pub fn new(py: Python<'py>) -> PyResult { + let resource_type: Bound<'py, PyAny> = record_class(py, RecordId::CompiledResourceType)?; + let nullability: Bound<'py, PyAny> = record_class(py, RecordId::InferredNullability)?; + Ok(Self { + qualified_column: record_class(py, RecordId::QualifiedLineageColumn)?, + lineage_source: record_class(py, RecordId::ColumnLineageSource)?, + lineage_edge: record_class(py, RecordId::ColumnLineageEdge)?, + column_lineage: record_class(py, RecordId::ColumnLineage)?, + model_lineage: record_class(py, RecordId::ModelColumnLineage)?, + semantic_use: record_class(py, RecordId::DirectSemanticColumnUse)?, + column_fact: record_class(py, RecordId::CompiledLineageColumnFact)?, + source_fact: record_class(py, RecordId::CompiledLineageSourceFact)?, + compact_facts: record_class(py, RecordId::CompactLineageFacts)?, + transform_kind: record_class(py, RecordId::ColumnTransformKind)?, + confidence: record_class(py, RecordId::ColumnLineageConfidence)?, + analysis_dialect: record_class(py, RecordId::PolyglotAnalysisDialect)?, + model_resource: native_field(&resource_type, "MODEL")?, + unknown_nullability: native_field(&nullability, "UNKNOWN")?, + nullability, + resource_type, + }) + } + + pub fn py(&self) -> Python<'py> { + self.qualified_column.py() + } + + /// `QualifiedLineageColumn(resource_type, resource_name, column_name)`. + pub fn qualified( + &self, + resource_type: &Bound<'py, PyAny>, + resource_name: &str, + column_name: &str, + ) -> PyResult> { + self.qualified_column.call1(( + self.resource_type.call1((resource_type,))?, + resource_name, + column_name, + )) + } + + /// `ColumnLineageEdge(source, target, transform_kind, confidence)`. + pub fn edge( + &self, + source: Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, + transform_kind: &Bound<'py, PyAny>, + confidence: &Bound<'py, PyAny>, + ) -> PyResult> { + self.lineage_edge + .call1((source, target, transform_kind, confidence)) + } + + /// `ModelColumnLineage(model_name, columns, has_star)`. + pub fn model( + &self, + model_name: &str, + columns: Vec>, + has_star: bool, + ) -> PyResult> { + self.model_lineage + .call1((model_name, PyTuple::new(self.py(), columns)?, has_star)) + } + + /// `ColumnLineage` of one native column; `nullability` is the enum member to record. + pub fn column( + &self, + column: &LineageColumn, + nullability: &Bound<'py, PyAny>, + ) -> PyResult> { + let mut sources: Vec> = Vec::with_capacity(column.upstream_columns.len()); + for source in &column.upstream_columns { + let resource_type: Bound<'py, PyAny> = + self.resource_type.call1((source.resource_type.as_str(),))?; + sources.push(self.lineage_source.call1(( + resource_type, + &source.resource_name, + &source.column_name, + ))?); + } + self.column_lineage.call1(( + &column.output_column, + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + self.py().None(), + PyTuple::new(self.py(), sources)?, + nullability, + self.confidence.call1((column.confidence.as_str(),))?, + )) + } + + /// `CompiledLineageColumnFact` of one native star-expansion column. + pub fn column_fact(&self, column: &LineageColumn) -> PyResult> { + let sources: Vec> = column + .upstream_columns + .iter() + .map(|source| self.source_fact(source)) + .collect::>()?; + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("output_column", &column.output_column)?; + fields.set_item("upstream_columns", PyTuple::new(self.py(), sources)?)?; + fields.set_item( + "transform_kind", + self.transform_kind + .call1((column.transform_kind.as_str(),))?, + )?; + fields.set_item( + "confidence", + self.confidence.call1((column.confidence.as_str(),))?, + )?; + self.column_fact.call((), Some(&fields)) + } + + pub(crate) fn source_fact(&self, source: &LineageSource) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("resource_type", source.resource_type.as_str())?; + fields.set_item("resource_name", &source.resource_name)?; + fields.set_item("column_name", &source.column_name)?; + self.source_fact.call((), Some(&fields)) + } +} diff --git a/crates/sqlbuild-python-compile/src/manifest/_helpers/builder.rs b/crates/sqlbuild-python-compile/src/manifest/_helpers/builder.rs new file mode 100644 index 0000000000..58d3f4f755 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/_helpers/builder.rs @@ -0,0 +1,19 @@ +//! `build_manifest`: the dbt-style manifest bytes, whose encoding error is raised on writing. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use sqlbuild_analysis::manifest::errors::ManifestError; + +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +pub(crate) const DATETIME_MODULE: &str = "datetime"; + +pub(crate) fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { + match error { + ManifestError::QueryFingerprint(message) => error_class(py, ErrorId::QueryFingerprint) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value), + ManifestError::MissingParameterType(name) => PyKeyError::new_err(name), + } +} diff --git a/crates/sqlbuild-python-compile/src/manifest/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/manifest/_helpers/mod.rs new file mode 100644 index 0000000000..433cca3fd0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod builder; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs rename to crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python-compile/src/manifest/main/build_manifest.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs rename to crates/sqlbuild-python-compile/src/manifest/main/build_manifest.rs index ec89f1474c..11238eb5b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python-compile/src/manifest/main/build_manifest.rs @@ -1,70 +1,30 @@ -//! `build_manifest`: the dbt-style manifest bytes, whose encoding error is raised on writing. +//! The manifest of `project` and the optional plan, over `graph` or `upstream_deps`/`downstream_deps`. -use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBytes, PyDict}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::manifest::_helpers::builder::DATETIME_MODULE; +use crate::manifest::_helpers::builder::python_error; +use crate::manifest::_helpers::rows::ProjectRows; +use crate::manifest::_helpers::rows::plan_input; +use crate::manifest::_helpers::rows::project_rows; +use crate::manifest::_helpers::rows::python_hooks; +use crate::manifest::models::NativeManifest; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_analysis::graph::models::ProjectGraph; -use sqlbuild_analysis::manifest::errors::{ManifestEncodeError, ManifestError}; use sqlbuild_analysis::manifest::main::build_manifest_json::build_manifest_json; use sqlbuild_analysis::manifest::models::ManifestInput; -use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_host::entries::models::UserCodeHost; - -use crate::bindings::_helpers::manifest::rows::{ - ProjectRows, plan_input, project_rows, python_hooks, -}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_project::graph::main::project::native_graph_from_edges::native_graph_from_edges; use sqlbuild_python_project::graph::models::NativeProjectGraph; -const DATETIME_MODULE: &str = "datetime"; - -/// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeManifest { - contents: Result, ManifestEncodeError>, -} - -#[pymethods] -impl NativeManifest { - /// The bytes `json.dumps(manifest, indent=2) + "\n"` encoded, raising what it raised. - fn contents<'py>(&self, py: Python<'py>) -> PyResult> { - match &self.contents { - Ok(bytes) => Ok(PyBytes::new(py, bytes)), - Err(ManifestEncodeError::Unencodable(message)) => { - Err(PyTypeError::new_err(message.clone())) - } - Err(ManifestEncodeError::Emit(JsonEmitError::NestingTooDeep)) => Err( - PyRecursionError::new_err("maximum recursion depth exceeded"), - ), - Err(ManifestEncodeError::Emit(error)) => { - Err(PyValueError::new_err(format!("{error:?}"))) - } - } - } -} - -fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { - match error { - ManifestError::QueryFingerprint(message) => error_class(py, ErrorId::QueryFingerprint) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value), - ManifestError::MissingParameterType(name) => PyKeyError::new_err(name), - } -} - /// The manifest of `project` and the optional plan, over `graph` or `upstream_deps`/`downstream_deps`. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn build_manifest(options: Option<&Bound<'_, PyDict>>) -> PyResult> { +pub fn build_manifest(options: Option<&Bound<'_, PyDict>>) -> PyResult> { let request = Request::new("build_manifest", options)?; let py: Python<'_> = request.py(); let project: Bound<'_, PyAny> = request.get("project")?; @@ -128,8 +88,3 @@ pub(crate) fn build_manifest(options: Option<&Bound<'_, PyDict>>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_manifest, module)?)?; - module.add_class::() -} diff --git a/crates/sqlbuild-python-compile/src/manifest/main/mod.rs b/crates/sqlbuild-python-compile/src/manifest/main/mod.rs new file mode 100644 index 0000000000..d76f5a4307 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/main/mod.rs @@ -0,0 +1,2 @@ +pub mod build_manifest; +pub mod register_builder; diff --git a/crates/sqlbuild-python-compile/src/manifest/main/register_builder.rs b/crates/sqlbuild-python-compile/src/manifest/main/register_builder.rs new file mode 100644 index 0000000000..ce09fcc603 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/main/register_builder.rs @@ -0,0 +1,12 @@ +//! Register the builder functions and classes. + +use crate::manifest::main::build_manifest::build_manifest; +use crate::manifest::models::NativeManifest; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the builder functions and classes. +pub fn register_builder(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_manifest, module)?)?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-compile/src/manifest/mod.rs b/crates/sqlbuild-python-compile/src/manifest/mod.rs new file mode 100644 index 0000000000..804ab651fd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/mod.rs @@ -0,0 +1,5 @@ +//! The dbt-style manifest built from the compiled project, plan output and hook identities. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/manifest/models.rs b/crates/sqlbuild-python-compile/src/manifest/models.rs new file mode 100644 index 0000000000..8bf2477c93 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/manifest/models.rs @@ -0,0 +1,39 @@ +//! Public records of manifest. + +use pyo3::exceptions::PyRecursionError; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyBytes; +use sqlbuild_analysis::manifest::errors::ManifestEncodeError; +use sqlbuild_core::json::errors::JsonEmitError; + +/// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeManifest { + pub(crate) contents: Result, ManifestEncodeError>, +} + +#[pymethods] +impl NativeManifest { + /// The bytes `json.dumps(manifest, indent=2) + "\n"` encoded, raising what it raised. + pub(crate) fn contents<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.contents { + Ok(bytes) => Ok(PyBytes::new(py, bytes)), + Err(ManifestEncodeError::Unencodable(message)) => { + Err(PyTypeError::new_err(message.clone())) + } + Err(ManifestEncodeError::Emit(JsonEmitError::NestingTooDeep)) => Err( + PyRecursionError::new_err("maximum recursion depth exceeded"), + ), + Err(ManifestEncodeError::Emit(error)) => { + Err(PyValueError::new_err(format!("{error:?}"))) + } + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/declared_relations.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs rename to crates/sqlbuild-python-compile/src/project_assembly/_helpers/declared_relations.rs index 57da7f3ed6..58c21ad13c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/declared_relations.rs @@ -1,11 +1,13 @@ //! What each model, seed, source and table function declares about its columns before analysis. +use crate::project_assembly::models::DeclaredRelations; +use crate::project_assembly::models::DeclaredShapes; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use std::collections::HashMap; use pyo3::types::{PyDict, PyDictMethods, PyString}; -use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -28,39 +30,6 @@ const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// `str(None)`, the text of an exact declaration without a type. const NONE_TEXT: &str = "None"; -/// The three per-relation declaration maps the analysis starts from, and the declared shapes. -pub(crate) struct DeclaredRelations<'py> { - pub(crate) column_types: Bound<'py, PyDict>, - pub(crate) nullability: Bound<'py, PyDict>, - pub(crate) complete_schemas: Bound<'py, PyDict>, - pub(crate) declared: DeclaredShapes, -} - -/// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. -#[derive(Debug, Clone, Default)] -pub(crate) struct ShapeRows { - rows: Shapes, - positions: HashMap, -} - -impl ShapeRows { - /// `shapes[name] = pairs`, keeping an existing name's position as a dict does. - pub(crate) fn set(&mut self, name: String, pairs: Pairs) { - match self.positions.get(&name) { - Some(&position) => self.rows[position].1 = pairs, - None => { - let _ = self.positions.insert(name.clone(), self.rows.len()); - self.rows.push((name, pairs)); - } - } - } - - /// The rows in insertion order. - pub(crate) fn rows(&self) -> &Shapes { - &self.rows - } -} - /// `(column, value)` pairs as a dict keeps them: each column once, at its first position, last value. pub(crate) fn dict_pairs(items: impl IntoIterator) -> Pairs { let mut pairs: Pairs = Vec::new(); @@ -77,14 +46,6 @@ pub(crate) fn dict_pairs(items: impl IntoIterator) -> P pairs } -/// The native rows of the declared types, nullability and authoritative schemas. -#[derive(Debug, Clone, Default)] -pub(crate) struct DeclaredShapes { - pub(crate) types: ShapeRows, - pub(crate) nullability: ShapeRows, - pub(crate) complete: ShapeRows, -} - /// The Python declaration maps being built: declared types, nullability and authoritative schemas. type DeclaredMaps<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); @@ -173,7 +134,7 @@ fn non_null(column: &SchemaColumnFact) -> bool { } /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. -fn column_types<'py>( +pub(crate) fn column_types<'py>( py: Python<'py>, columns: &[SchemaColumnFact], exact: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs rename to crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs index 98e5c64a6f..fec58e8c96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs @@ -3,33 +3,12 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::project_assembly::models::{ - NativeModelFacts, NativeProjectFacts, NativeProjectResources, NativeResourceFailure, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::project_assembly::models::NativeModelFacts; +use crate::project_assembly::models::NativeProjectFacts; +use crate::project_assembly::models::NativeProjectResources; +use crate::project_assembly::models::NativeResourceFailure; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_native_model_facts, - }, - GetterEntry { - class: class_of::, - read: read_native_project_facts, - }, - GetterEntry { - class: class_of::, - read: read_native_project_resources, - }, - GetterEntry { - class: class_of::, - read: read_native_resource_failure, - }, -]; - -fn read_native_model_facts<'py>( +pub(crate) fn read_native_model_facts<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -48,7 +27,7 @@ fn read_native_model_facts<'py>( }) } -fn read_native_project_facts<'py>( +pub(crate) fn read_native_project_facts<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -66,7 +45,7 @@ fn read_native_project_facts<'py>( }) } -fn read_native_project_resources<'py>( +pub(crate) fn read_native_project_resources<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -89,7 +68,7 @@ fn read_native_project_resources<'py>( }) } -fn read_native_resource_failure<'py>( +pub(crate) fn read_native_resource_failure<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-compile/src/project_assembly/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/mod.rs new file mode 100644 index 0000000000..149bd535f5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declared_relations; +pub(crate) mod getters; +pub(crate) mod native_resources; +pub(crate) mod resources; +pub(crate) mod syntax_checks; +pub(crate) mod syntax_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/native_resources.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs rename to crates/sqlbuild-python-compile/src/project_assembly/_helpers/native_resources.rs index 4312270b18..8b17b57773 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/native_resources.rs @@ -1,12 +1,11 @@ //! The native project assembly request built from compile inputs, and its resource facts. +use pyo3::IntoPyObject; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyBoolMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::errors::AssemblyError; -use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names; use sqlbuild_analysis::assembly::project::models::{ AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, @@ -17,9 +16,8 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_model_config::templates::models::Scalar; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::project_assembly::models::{ - NativeProjectResources, NativeResourceFailure, -}; +use crate::project_assembly::models::NativeProjectResources; +use crate::project_assembly::models::NativeResourceFailure; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::main::imported::imported; @@ -30,13 +28,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::authored_values::main::project_var_text::project_var_text; use sqlbuild_python_front::model_config::main::config::record_reads::record_reads; use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; const OS_MODULE: &str = "os"; -const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; +pub(crate) const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const GENERIC_DIALECT: &str = "generic"; const RENDER_LABEL: &str = "\u{0}"; const SOURCE_SLOT: &str = "source"; @@ -243,7 +240,7 @@ fn items<'py>(inputs: &Bound<'py, PyAny>, attribute: &str) -> PyResult( +pub(crate) fn project_request<'py>( inputs: &Bound<'py, PyAny>, dialect: Option, syntax_checks: &Bound<'py, PyAny>, @@ -443,7 +440,7 @@ fn namespace_row(namespace: Namespace) -> NamespaceRow { } /// `NativeProjectResources` for the native result, replaying its template reads first. -fn resources_object<'py>( +pub(crate) fn resources_object<'py>( py: Python<'py>, resources: ProjectResources, held: &[Bound<'py, PyAny>], @@ -478,24 +475,3 @@ fn resources_object<'py>( )? .into_any()) } - -/// Return the resource facts and the resource Python's assembly raises at, if any. -#[pyfunction] -#[pyo3(signature = (*, inputs, dialect, syntax_checks))] -pub(crate) fn assemble_native_project_resources<'py>( - inputs: &Bound<'py, PyAny>, - dialect: Option, - syntax_checks: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let (request, held) = project_request(inputs, dialect, syntax_checks)?; - let resources: ProjectResources = py - .compiler_detach(|| assemble_project_resources(&request)) - .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; - resources_object(py, resources, &held) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_native_project_resources, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/_helpers/resources.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/resources.rs new file mode 100644 index 0000000000..a4114b76c7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/resources.rs @@ -0,0 +1,47 @@ +//! Resource assembly facts for one project, computed natively. + +use crate::project_assembly::main::syntax_error_message::syntax_error_message; +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyString, PyStringMethods}; +use pyo3::{PyErr, pyfunction}; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; + +pub(crate) const SYNTAX_CHECK_CONTEXT: &str = "native SQL syntax check"; + +/// Python's syntax error message for one SQL string, None where it parses. +#[pyfunction] +#[pyo3(signature = (sql, placeholders, dialect, parse_one=false))] +pub(crate) fn sql_syntax_error( + sql: &Bound<'_, PyString>, + placeholders: Option>, + dialect: Option, + parse_one: bool, +) -> PyResult> { + let py: Python<'_> = sql.py(); + let check = SyntaxCheck { + sql: sql.to_str()?.to_owned(), + placeholders: placeholders.unwrap_or_default().into_iter().collect(), + }; + let dialect: String = dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| "generic".to_owned()); + let mode = if parse_one { + SyntaxMode::Parse + } else { + SyntaxMode::Validate + }; + syntax_error_message(py, &check, &dialect, mode) +} + +/// The `ValueError` Python's syntax validation raises before parsing. +pub(crate) fn syntax_value_error(failure: SyntaxFailure) -> PyErr { + match failure { + SyntaxFailure::Normalization(message) => PyValueError::new_err(message), + SyntaxFailure::UnknownDialect(name) => { + PyValueError::new_err(format!("Unknown dialect: {name}")) + } + } +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs new file mode 100644 index 0000000000..10a27af62d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs @@ -0,0 +1,95 @@ +//! The model and hook SQL assembly validates, and the SQL standalone pivot proofs read. + +use crate::project_assembly::main::model_placeholders::model_placeholders; +use crate::project_assembly::main::standalone_pivot_rows::standalone_pivot_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +use crate::project_assembly::_helpers::syntax_validation::hook_sql_statements; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; + +const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; + +/// Native SQL and dynamic family specs indexed by their model's position. +pub(crate) type PivotModelRows = Vec<(usize, (String, Vec))>; + +pub(crate) fn config_value<'py>( + model_input: &Bound<'py, PyAny>, + key: &str, +) -> PyResult> { + native_field(&native_field(model_input, "config")?, "values")?.call_method1("get", (key,)) +} + +pub(crate) fn model_name(model_input: &Bound<'_, PyAny>) -> PyResult { + native_field( + &native_field(&native_field(model_input, "model_file")?, "file_path")?, + "stem", + )? + .extract() +} + +/// The model query with cursor intrinsics replaced by typed literals for static analysis. +pub(crate) fn analysis_sql<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model_input.py(); + Ok(PyString::new( + py, + &analysis_cursor_sql( + py, + &native_field(model_input, "query_sql")?.extract::()?, + &config_value(model_input, "cursor_type")?, + )?, + ) + .into_any()) +} + +pub(crate) fn syntax_checks<'py>( + model_input: &Bound<'py, PyAny>, + analysis_model_names: &Bound<'py, PyAny>, + analysis_succeeded: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model_input.py(); + let name: String = model_name(model_input)?; + if !native_field(model_input, "sql_validation_enabled")?.is_truthy()? + || (!analysis_model_names.is_none() && !analysis_model_names.contains(&name)?) + { + return Ok(PyTuple::empty(py)); + } + let placeholders: Bound<'py, PyAny> = match model_placeholders(model_input)? { + Some(placeholders) => placeholders.into_any(), + None => py.None().into_bound(py), + }; + let mut statements: Vec> = Vec::new(); + for hook_name in HOOK_NAMES { + for statement in hook_sql_statements(config_value(model_input, hook_name)?)? { + statements.push(PyString::new(py, &statement).into_any()); + } + } + if !analysis_succeeded.contains(&name)? { + statements.push(analysis_sql(model_input)?); + } + let checks: Vec> = statements + .into_iter() + .map(|statement| PyTuple::new(py, [statement, placeholders.clone()])) + .collect::>()?; + PyTuple::new(py, checks) +} + +/// `(sql, families)` by model index for each pivot model no analysis covered. +#[pyfunction] +#[pyo3(signature = (*, model_inputs, analysed))] +pub(crate) fn standalone_pivot_models<'py>( + model_inputs: &Bound<'py, PyAny>, + analysed: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model_inputs.py(); + let models: Bound<'py, PyDict> = PyDict::new(py); + for (index, (sql, _)) in standalone_pivot_rows(model_inputs, analysed)? { + let model = model_inputs.get_item(index)?; + let entry = native_field(&model, "schema_entry")?; + models.set_item(index, (sql, native_field(&entry, "dynamic_columns")?))?; + } + Ok(models) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs rename to crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs index 61727bc485..3b1cd25a9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs @@ -1,46 +1,42 @@ //! Reject model and hook SQL the parser cannot read, naming the model, hook and file. +use crate::project_assembly::main::sql_entry_label::sql_entry_label; +use crate::project_assembly::main::syntax_error_message::syntax_error_message; use std::collections::HashMap; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; -use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const GENERIC_DIALECT: &str = "generic"; - -/// `(placeholder defaults, dialect)` a statement parses with. -pub(crate) type ParseOptions = (Option>, Option); +pub(crate) const GENERIC_DIALECT: &str = "generic"; /// What every statement of one model is parsed with. -struct SyntaxContext<'a, 'py> { - py: Python<'py>, - model_name: &'a str, - placeholders: &'a Option>, - dialect: &'a str, - sql_hook_entry: Bound<'py, PyAny>, - python_hook_entry: Bound<'py, PyAny>, +pub(crate) struct SyntaxContext<'a, 'py> { + pub(crate) py: Python<'py>, + pub(crate) model_name: &'a str, + pub(crate) placeholders: &'a Option>, + pub(crate) dialect: &'a str, + pub(crate) sql_hook_entry: Bound<'py, PyAny>, + pub(crate) python_hook_entry: Bound<'py, PyAny>, } /// One hook value to validate: the hook key, its file and the label errors name it by. -struct HookSite<'a, 'py> { - hook_name: &'a str, - file_path: Bound<'py, PyAny>, - label: Option, +pub(crate) struct HookSite<'a, 'py> { + pub(crate) hook_name: &'a str, + pub(crate) file_path: Bound<'py, PyAny>, + pub(crate) label: Option, } /// `CompileInputError(message, help=sql_analysis_opt_out_help(model_owned=True))`. -fn syntax_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn syntax_error(py: Python<'_>, message: String) -> PyErr { let raised = || -> PyResult { let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("help", sql_analysis_opt_out_help(true))?; @@ -65,18 +61,8 @@ fn parse_error(context: &SyntaxContext<'_, '_>, sql: &str) -> PyResult) -> PyResult { - let name: Bound<'_, PyAny> = native_field(entry, "name")?; - Ok(if name.is_none() { - format!("{prefix} inline_sql(\"...\")") - } else { - format!("{prefix} sql(\"{}\")", name.str()?) - }) -} - /// `validate_hook_sql_syntax`: each SQL string inside a hook value, depth first. -fn check_hook_value( +pub(crate) fn check_hook_value( context: &SyntaxContext<'_, '_>, value: &Bound<'_, PyAny>, site: HookSite<'_, '_>, @@ -145,70 +131,6 @@ fn check_hook_value( Ok(()) } -/// Validate the SQL of one model's hook value, raising the first parse failure. -#[pyfunction] -#[pyo3(signature = (value, hook_name, model, parsing=(None, None)))] -pub(crate) fn validate_hook_sql_syntax( - value: Bound<'_, PyAny>, - hook_name: &str, - model: (String, Bound<'_, PyAny>), - parsing: ParseOptions, -) -> Result<(), PyErr> { - let (model_name, file_path) = model; - let (placeholders, dialect) = parsing; - let py: Python<'_> = value.py(); - let dialect: String = dialect - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); - let context: SyntaxContext<'_, '_> = SyntaxContext { - py, - model_name: &model_name, - placeholders: &placeholders, - dialect: &dialect, - sql_hook_entry: record_class(py, RecordId::SqlHookEntry)?, - python_hook_entry: record_class(py, RecordId::PythonHookEntry)?, - }; - check_hook_value( - &context, - &value, - HookSite { - hook_name, - file_path, - label: None, - }, - ) -} - -/// Validate that a model's query parses, raising its syntax error. -#[pyfunction] -#[pyo3(signature = (query_sql, model, parsing=(None, None)))] -pub(crate) fn validate_sql_syntax( - py: Python<'_>, - query_sql: &str, - model: (String, Bound<'_, PyAny>), - parsing: ParseOptions, -) -> Result<(), PyErr> { - let (model_name, file_path) = model; - let (placeholders, dialect) = parsing; - let check: SyntaxCheck = SyntaxCheck { - sql: query_sql.to_owned(), - placeholders: placeholders.unwrap_or_default().into_iter().collect(), - }; - let dialect: String = dialect - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); - match syntax_error_message(py, &check, &dialect, SyntaxMode::Validate)? { - Some(message) => Err(syntax_error( - py, - format!( - "SQL syntax error in model '{model_name}' ({}): {message}", - file_path.str()? - ), - )), - None => Ok(()), - } -} - /// `hook_sql_statements`: the hook SQL strings `validate_hook_sql_syntax` checks, in its order. pub(crate) fn hook_statements( value: &Bound<'_, PyAny>, @@ -239,13 +161,6 @@ pub(crate) fn hook_sql_statements(value: Bound<'_, PyAny>) -> PyResult String { +pub(crate) fn opt_out_help(model_owned: bool) -> String { sql_analysis_opt_out_help(model_owned) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(opt_out_help, module)?)?; - module.add_function(wrap_pyfunction!(validate_hook_sql_syntax, module)?)?; - module.add_function(wrap_pyfunction!(validate_sql_syntax, module)?)?; - module.add_function(wrap_pyfunction!(hook_sql_statements, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/constants.rs b/crates/sqlbuild-python-compile/src/project_assembly/constants.rs new file mode 100644 index 0000000000..8ba90fc3a4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/constants.rs @@ -0,0 +1,32 @@ +//! Public constants of project assembly. + +use crate::project_assembly::_helpers::getters::read_native_model_facts; +use crate::project_assembly::_helpers::getters::read_native_project_facts; +use crate::project_assembly::_helpers::getters::read_native_project_resources; +use crate::project_assembly::_helpers::getters::read_native_resource_failure; +use crate::project_assembly::models::NativeModelFacts; +use crate::project_assembly::models::NativeProjectFacts; +use crate::project_assembly::models::NativeProjectResources; +use crate::project_assembly::models::NativeResourceFailure; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_native_model_facts, + }, + GetterEntry { + class: class_of::, + read: read_native_project_facts, + }, + GetterEntry { + class: class_of::, + read: read_native_project_resources, + }, + GetterEntry { + class: class_of::, + read: read_native_resource_failure, + }, +]; diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/assemble_native_project_resources.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/assemble_native_project_resources.rs new file mode 100644 index 0000000000..4939aa13d8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/assemble_native_project_resources.rs @@ -0,0 +1,28 @@ +//! Return the resource facts and the resource Python's assembly raises at, if any. + +use crate::project_assembly::_helpers::native_resources::PROJECT_ASSEMBLY_CONTEXT; +use crate::project_assembly::_helpers::native_resources::project_request; +use crate::project_assembly::_helpers::native_resources::resources_object; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::models::ProjectResources; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// Return the resource facts and the resource Python's assembly raises at, if any. +#[pyfunction] +#[pyo3(signature = (*, inputs, dialect, syntax_checks))] +pub fn assemble_native_project_resources<'py>( + inputs: &Bound<'py, PyAny>, + dialect: Option, + syntax_checks: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let (request, held) = project_request(inputs, dialect, syntax_checks)?; + let resources: ProjectResources = py + .compiler_detach(|| assemble_project_resources(&request)) + .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; + resources_object(py, resources, &held) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/declared_relations.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/declared_relations.rs new file mode 100644 index 0000000000..baf60fb799 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/declared_relations.rs @@ -0,0 +1,13 @@ +//! Declared column types, declared nullability and the authoritative binding schemas. + +use crate::project_assembly::_helpers::declared_relations; +use crate::project_assembly::models::DeclaredRelations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Declared column types, declared nullability and the authoritative binding schemas. +#[inline] +pub fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { + declared_relations::declared_relations(inputs) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/mod.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/mod.rs new file mode 100644 index 0000000000..6c0b4247ad --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/mod.rs @@ -0,0 +1,15 @@ +pub mod assemble_native_project_resources; +pub mod declared_relations; +pub mod model_placeholders; +pub mod model_syntax_checks; +pub mod register_native_resources; +pub mod register_resources; +pub mod register_syntax_checks; +pub mod register_syntax_validation; +pub mod sql_entry_label; +pub mod standalone_pivot_rows; +pub mod syntax_error_message; +pub mod syntax_error_messages; +pub mod syntax_valid; +pub mod validate_hook_sql_syntax; +pub mod validate_sql_syntax; diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/model_placeholders.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/model_placeholders.rs new file mode 100644 index 0000000000..0d1e92524a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/model_placeholders.rs @@ -0,0 +1,22 @@ +//! The model's `@@@name` placeholder defaults as strings, or `None`. + +use crate::project_assembly::_helpers::syntax_checks::config_value; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; + +/// The model's `@@@name` placeholder defaults as strings, or `None`. +#[pyfunction] +pub fn model_placeholders<'py>( + model_input: &Bound<'py, PyAny>, +) -> PyResult>> { + let raw: Bound<'py, PyAny> = config_value(model_input, "placeholders")?; + let Ok(raw) = raw.downcast::() else { + return Ok(None); + }; + let placeholders: Bound<'py, PyDict> = PyDict::new(model_input.py()); + for (key, value) in raw.iter() { + placeholders.set_item(key.str()?, value.str()?)?; + } + Ok(Some(placeholders)) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/model_syntax_checks.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/model_syntax_checks.rs new file mode 100644 index 0000000000..eec9f44914 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/model_syntax_checks.rs @@ -0,0 +1,21 @@ +//! Each model's `(sql, placeholders)` that assembly passes to syntax validation. + +use crate::project_assembly::_helpers::syntax_checks::syntax_checks; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; + +/// Each model's `(sql, placeholders)` that assembly passes to syntax validation. +#[pyfunction] +#[pyo3(signature = (*, model_inputs, analysis_model_names, analysis_succeeded))] +pub fn model_syntax_checks<'py>( + model_inputs: &Bound<'py, PyAny>, + analysis_model_names: &Bound<'py, PyAny>, + analysis_succeeded: &Bound<'py, PyAny>, +) -> PyResult> { + let checks: Vec> = model_inputs + .try_iter()? + .map(|model_input| syntax_checks(&model_input?, analysis_model_names, analysis_succeeded)) + .collect::>()?; + PyTuple::new(model_inputs.py(), checks) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/register_native_resources.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/register_native_resources.rs new file mode 100644 index 0000000000..97df9bb4c0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/register_native_resources.rs @@ -0,0 +1,11 @@ +//! Register the native resources functions and classes. + +use crate::project_assembly::main::assemble_native_project_resources::assemble_native_project_resources; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the native resources functions and classes. +pub fn register_native_resources(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_native_project_resources, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/register_resources.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/register_resources.rs new file mode 100644 index 0000000000..6c5aabc031 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/register_resources.rs @@ -0,0 +1,11 @@ +//! Register the resources functions and classes. + +use crate::project_assembly::_helpers::resources::sql_syntax_error; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the resources functions and classes. +pub fn register_resources(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_checks.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_checks.rs new file mode 100644 index 0000000000..717c5215bc --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_checks.rs @@ -0,0 +1,15 @@ +//! Register the syntax checks functions and classes. + +use crate::project_assembly::_helpers::syntax_checks::standalone_pivot_models; +use crate::project_assembly::main::model_placeholders::model_placeholders; +use crate::project_assembly::main::model_syntax_checks::model_syntax_checks; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the syntax checks functions and classes. +pub fn register_syntax_checks(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_placeholders, module)?)?; + module.add_function(wrap_pyfunction!(model_syntax_checks, module)?)?; + module.add_function(wrap_pyfunction!(standalone_pivot_models, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs new file mode 100644 index 0000000000..d5de6c4075 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs @@ -0,0 +1,16 @@ +//! Register the syntax validation functions and classes. + +use crate::project_assembly::_helpers::syntax_validation::hook_sql_statements; +use crate::project_assembly::_helpers::syntax_validation::opt_out_help; +use crate::project_assembly::main::validate_hook_sql_syntax::validate_hook_sql_syntax; +use crate::project_assembly::main::validate_sql_syntax::validate_sql_syntax; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the syntax validation functions and classes. +pub fn register_syntax_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(opt_out_help, module)?)?; + module.add_function(wrap_pyfunction!(validate_hook_sql_syntax, module)?)?; + module.add_function(wrap_pyfunction!(validate_sql_syntax, module)?)?; + module.add_function(wrap_pyfunction!(hook_sql_statements, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/sql_entry_label.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/sql_entry_label.rs new file mode 100644 index 0000000000..11931f3ce7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/sql_entry_label.rs @@ -0,0 +1,14 @@ +//! How errors name a SQL hook entry: `{prefix} sql("name")`, or `{prefix} inline_sql("...")`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// How errors name a SQL hook entry: `{prefix} sql("name")`, or `{prefix} inline_sql("...")`. +pub fn sql_entry_label(prefix: &str, entry: &Bound<'_, PyAny>) -> PyResult { + let name: Bound<'_, PyAny> = native_field(entry, "name")?; + Ok(if name.is_none() { + format!("{prefix} inline_sql(\"...\")") + } else { + format!("{prefix} sql(\"{}\")", name.str()?) + }) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/standalone_pivot_rows.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/standalone_pivot_rows.rs new file mode 100644 index 0000000000..345da484f7 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/standalone_pivot_rows.rs @@ -0,0 +1,41 @@ +//! Native-only pivot requests; the public wrapper alone projects contract records. + +use crate::project_assembly::_helpers::syntax_checks::PivotModelRows; +use crate::project_assembly::_helpers::syntax_checks::analysis_sql; +use crate::project_assembly::_helpers::syntax_checks::model_name; +use crate::project_assembly::main::model_placeholders::model_placeholders; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDictMethods; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +/// Native-only pivot requests; the public wrapper alone projects contract records. +pub fn standalone_pivot_rows( + model_inputs: &Bound<'_, PyAny>, + analysed: &Bound<'_, PyAny>, +) -> PyResult { + let mut models: PivotModelRows = Vec::new(); + for (index, model_input) in model_inputs.try_iter()?.enumerate() { + let model_input = model_input?; + let Some(schema) = attached_schema(&model_input)? else { + continue; + }; + let families = schema.dynamic_columns; + if families.is_empty() || analysed.contains(model_name(&model_input)?)? { + continue; + } + let query_sql: String = analysis_sql(&model_input)?.extract()?; + let sql: String = match model_placeholders(&model_input)? { + Some(placeholders) if !placeholders.is_empty() => { + let pairs: Vec<(String, String)> = placeholders + .iter() + .map(|(key, value)| Ok((key.extract()?, value.extract()?))) + .collect::>()?; + placeholder_defaults(&query_sql, &pairs) + } + _ => query_sql, + }; + models.push((index, (sql, families))); + } + Ok(models) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_message.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_message.rs new file mode 100644 index 0000000000..2be5ed0212 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_message.rs @@ -0,0 +1,20 @@ +//! Python's syntax error message for one check, raising what Python's validation raises. + +use crate::project_assembly::_helpers::resources::syntax_value_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// Python's syntax error message for one check, raising what Python's validation raises. +pub fn syntax_error_message( + py: Python<'_>, + check: &SyntaxCheck, + dialect: &str, + mode: SyntaxMode, +) -> PyResult> { + py.compiler_detach(|| Ok(check_syntax_error(check, dialect, mode))) + .map_err(compiler_error)? + .map_err(syntax_value_error) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs new file mode 100644 index 0000000000..69aedb8c97 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs @@ -0,0 +1,26 @@ +//! Each check's `syntax_error_message`, in order, parsed together on the compile pool. + +use crate::input_loops::main::pooled::pooled; +use crate::project_assembly::_helpers::resources::syntax_value_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; + +/// Each check's `syntax_error_message`, in order, parsed together on the compile pool. +pub fn syntax_error_messages( + py: Python<'_>, + checks: &[SyntaxCheck], + dialect: &str, + mode: SyntaxMode, +) -> Vec>> { + py.detach(|| { + pooled(checks, |check| { + catch_compiler_panic(|| Ok(check_syntax_error(check, dialect, mode))) + }) + }) + .into_iter() + .map(|outcome| outcome.map_err(compiler_error)?.map_err(syntax_value_error)) + .collect() +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_valid.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_valid.rs new file mode 100644 index 0000000000..a8f46343f0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_valid.rs @@ -0,0 +1,17 @@ +//! Whether every check parses, raising what Python's validation raises. + +use crate::project_assembly::_helpers::resources::SYNTAX_CHECK_CONTEXT; +use crate::project_assembly::_helpers::resources::syntax_value_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; +use sqlbuild_analysis::assembly::project::models::SyntaxCheck; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// Whether every check parses, raising what Python's validation raises. +pub fn syntax_valid(py: Python<'_>, dialect: &str, checks: &[SyntaxCheck]) -> PyResult { + py.compiler_detach(|| check_sql_syntax(dialect, checks)) + .map_err(|reason| compiler_error(native_failure(SYNTAX_CHECK_CONTEXT, &reason)))? + .map_err(syntax_value_error) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/validate_hook_sql_syntax.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/validate_hook_sql_syntax.rs new file mode 100644 index 0000000000..cfa667327d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/validate_hook_sql_syntax.rs @@ -0,0 +1,45 @@ +//! Validate the SQL of one model's hook value, raising the first parse failure. + +use crate::project_assembly::_helpers::syntax_validation::GENERIC_DIALECT; +use crate::project_assembly::_helpers::syntax_validation::HookSite; +use crate::project_assembly::_helpers::syntax_validation::SyntaxContext; +use crate::project_assembly::_helpers::syntax_validation::check_hook_value; +use crate::project_assembly::types::ParseOptions; +use pyo3::prelude::{Bound, PyAny, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; + +/// Validate the SQL of one model's hook value, raising the first parse failure. +#[pyfunction] +#[pyo3(signature = (value, hook_name, model, parsing=(None, None)))] +pub fn validate_hook_sql_syntax( + value: Bound<'_, PyAny>, + hook_name: &str, + model: (String, Bound<'_, PyAny>), + parsing: ParseOptions, +) -> Result<(), PyErr> { + let (model_name, file_path) = model; + let (placeholders, dialect) = parsing; + let py: Python<'_> = value.py(); + let dialect: String = dialect + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + let context: SyntaxContext<'_, '_> = SyntaxContext { + py, + model_name: &model_name, + placeholders: &placeholders, + dialect: &dialect, + sql_hook_entry: record_class(py, RecordId::SqlHookEntry)?, + python_hook_entry: record_class(py, RecordId::PythonHookEntry)?, + }; + check_hook_value( + &context, + &value, + HookSite { + hook_name, + file_path, + label: None, + }, + ) +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/validate_sql_syntax.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/validate_sql_syntax.rs new file mode 100644 index 0000000000..64e257a468 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/validate_sql_syntax.rs @@ -0,0 +1,39 @@ +//! Validate that a model's query parses, raising its syntax error. + +use crate::project_assembly::_helpers::syntax_validation::GENERIC_DIALECT; +use crate::project_assembly::_helpers::syntax_validation::syntax_error; +use crate::project_assembly::main::syntax_error_message::syntax_error_message; +use crate::project_assembly::types::ParseOptions; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; + +/// Validate that a model's query parses, raising its syntax error. +#[pyfunction] +#[pyo3(signature = (query_sql, model, parsing=(None, None)))] +pub fn validate_sql_syntax( + py: Python<'_>, + query_sql: &str, + model: (String, Bound<'_, PyAny>), + parsing: ParseOptions, +) -> Result<(), PyErr> { + let (model_name, file_path) = model; + let (placeholders, dialect) = parsing; + let check: SyntaxCheck = SyntaxCheck { + sql: query_sql.to_owned(), + placeholders: placeholders.unwrap_or_default().into_iter().collect(), + }; + let dialect: String = dialect + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + match syntax_error_message(py, &check, &dialect, SyntaxMode::Validate)? { + Some(message) => Err(syntax_error( + py, + format!( + "SQL syntax error in model '{model_name}' ({}): {message}", + file_path.str()? + ), + )), + None => Ok(()), + } +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/mod.rs b/crates/sqlbuild-python-compile/src/project_assembly/mod.rs new file mode 100644 index 0000000000..55672052b0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/mod.rs @@ -0,0 +1,7 @@ +//! The native project assembly request built from compile inputs, and the resource and model facts it hands to the compile driver. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-compile/src/project_assembly/models.rs b/crates/sqlbuild-python-compile/src/project_assembly/models.rs new file mode 100644 index 0000000000..385bcf6e80 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/models.rs @@ -0,0 +1,92 @@ +//! Public records of project assembly. + +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::pyclass; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use std::collections::HashMap; + +/// The three per-relation declaration maps the analysis starts from, and the declared shapes. +#[derive(Debug)] +pub struct DeclaredRelations<'py> { + pub column_types: Bound<'py, PyDict>, + pub nullability: Bound<'py, PyDict>, + pub complete_schemas: Bound<'py, PyDict>, + pub declared: DeclaredShapes, +} + +/// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. +#[derive(Debug, Clone, Default)] +pub struct ShapeRows { + pub(crate) rows: Shapes, + pub(crate) positions: HashMap, +} + +impl ShapeRows { + /// `shapes[name] = pairs`, keeping an existing name's position as a dict does. + pub fn set(&mut self, name: String, pairs: Pairs) { + match self.positions.get(&name) { + Some(&position) => self.rows[position].1 = pairs, + None => { + let _ = self.positions.insert(name.clone(), self.rows.len()); + self.rows.push((name, pairs)); + } + } + } + + /// The rows in insertion order. + pub fn rows(&self) -> &Shapes { + &self.rows + } +} + +/// The native rows of the declared types, nullability and authoritative schemas. +#[derive(Debug, Clone, Default)] +pub struct DeclaredShapes { + pub types: ShapeRows, + pub nullability: ShapeRows, + pub complete: ShapeRows, +} + +/// Per-resource facts in input order, and whether each model's SQL passed syntax validation. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct NativeProjectResources { + pub model_deps: Py, + pub model_syntax_valid: Py, + pub source_namespaces: Py, + pub seed_namespaces: Py, + pub function_deps: Py, + pub audit_scope_deps: Py, + /// The `NativeResourceFailure` Python's assembly raises at, or `None`. + pub failure: Py, +} + +/// The resource Python's assembly raises at, and the exception it raises there. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct NativeResourceFailure { + pub slot: Py, + pub index: Py, + pub error: Py, +} + +/// One model's native facts; its dependencies as `(resource type, name)` rows. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct NativeModelFacts { + pub deps: Py, + pub dynamic_contract: Py, + pub syntax_valid: Py, +} + +/// Every model's native facts and the resource facts. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct NativeProjectFacts { + pub models: Py, + pub resources: Py, +} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/types.rs b/crates/sqlbuild-python-compile/src/project_assembly/types.rs new file mode 100644 index 0000000000..e10b686d01 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/project_assembly/types.rs @@ -0,0 +1,6 @@ +//! Public traits and type aliases of project assembly. + +use std::collections::HashMap; + +/// `(placeholder defaults, dialect)` a statement parses with. +pub type ParseOptions = (Option>, Option); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs index 432dd59b15..aa69dbf87b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs @@ -1,18 +1,20 @@ //! DSL header scanning, header lint rules and header formatting fixes over Python lint models. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::rules::main::checking::header_violations::header_violations; +use crate::rules::main::checking::rules_config::rules_config; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_rules::dsl_headers::constants::RULE_HEADER_PARSE; use sqlbuild_rules::dsl_headers::main::format_headers::format_headers; -use sqlbuild_rules::dsl_headers::main::lint_headers::lint_headers; use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges as measurement_ranges; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers as scan; use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges as body_ranges; -use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; +use sqlbuild_rules::dsl_headers::models::{HeaderSpan, HeaderViolation}; -use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; -use crate::bindings::_helpers::rules::native_header_parser::NativeHeaderParser; +use crate::rules::models::LintViolationRow; +use crate::rules::models::NativeHeaderParser; +use crate::rules::models::ViolationClasses; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,13 +39,6 @@ fn text(contents: &Bound<'_, PyAny>) -> PyResult> { Ok(contents.extract::()?.chars().collect()) } -pub(crate) fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { - Ok(HeaderRulesConfig { - line_width: native_field(config, "line_width")?.extract()?, - max_description_lines: native_field(config, "max_description_lines")?.extract()?, - }) -} - fn spans(headers: &Bound<'_, PyAny>) -> PyResult> { headers .try_iter()? @@ -65,7 +60,7 @@ fn ranges<'py>(py: Python<'py>, ranges: Vec<(usize, usize)>) -> PyResult( +pub(crate) fn scan_headers<'py>( contents: &Bound<'py, PyAny>, first_only: bool, ) -> PyResult> { @@ -90,7 +85,7 @@ fn scan_headers<'py>( /// Non-empty SQL body ranges around and between the headers. #[pyfunction] #[pyo3(signature = (*, contents, headers))] -fn sql_body_ranges<'py>( +pub(crate) fn sql_body_ranges<'py>( contents: &Bound<'py, PyAny>, headers: &Bound<'py, PyAny>, ) -> PyResult> { @@ -103,23 +98,16 @@ fn sql_body_ranges<'py>( /// The authored query ranges inside MEASURE and EVIDENCE blocks. #[pyfunction] #[pyo3(signature = (*, contents))] -fn measurement_body_ranges<'py>(contents: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn measurement_body_ranges<'py>( + contents: &Bound<'py, PyAny>, +) -> PyResult> { ranges(contents.py(), measurement_ranges(&text(contents)?)) } -/// The header rule violations of `headers` in `contents`. -pub(crate) fn header_violations( - contents: &[char], - headers: &[HeaderSpan], - config: HeaderRulesConfig, -) -> Result, String> { - lint_headers(contents, headers, config, &NativeHeaderParser) -} - /// Every header rule violation of `headers` in `contents`. #[pyfunction] #[pyo3(signature = (*, contents, file_path, headers, config))] -fn lint_native_headers<'py>( +pub(crate) fn lint_native_headers<'py>( contents: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, headers: &Bound<'py, PyAny>, @@ -218,17 +206,3 @@ pub(crate) fn reject_unparseable_header_rewrites<'py>( } Ok((accepted, faults.clone())) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_headers, module)?)?; - module.add_function(wrap_pyfunction!(sql_body_ranges, module)?)?; - module.add_function(wrap_pyfunction!(measurement_body_ranges, module)?)?; - module.add_function(wrap_pyfunction!(lint_native_headers, module)?)?; - module.add_function(wrap_pyfunction!(format_native_headers, module)?)?; - module.add_function(wrap_pyfunction!(safe_format_files, module)?)?; - module.add_function(wrap_pyfunction!( - reject_unparseable_header_rewrites, - module - )?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/early_builtin_rules.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/early_builtin_rules.rs index 4fbdffac47..8a0a3c8a1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/early_builtin_rules.rs @@ -1,26 +1,25 @@ //! Built-in rule evaluation started speculatively once a compile's model types are recovered. +use crate::rules::main::configuration::load_rules_config_with_cache::load_rules_config_with_cache; +use crate::rules::models::EarlyRulesSettings; use std::cell::RefCell; -use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; -use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; use sqlbuild_host::compile_context::main::record_compile_metric::record_compile_metric; -use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_rules::engine::main::answer_speculative_rows::answer_speculative_rows; use sqlbuild_rules::engine::main::discard_speculative_rows::discard_speculative_rows; use sqlbuild_rules::engine::main::start_speculative_rows::start_speculative_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows}; -use crate::bindings::_helpers::rules::catalogue::references_custom_rules; -use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; -use crate::bindings::_helpers::rules::rule_plan::{rule_run_plan, selects_row_rules}; -use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; +use crate::rules::_helpers::configuration::catalogue::references_custom_rules; +use crate::rules::_helpers::rule::rule_dispatch::speculative_rules_request; +use crate::rules::_helpers::rule::rule_plan::rule_run_plan; +use crate::rules::_helpers::rule::rule_plan::selects_row_rules; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::types::CompilerDetach; @@ -29,29 +28,20 @@ use sqlbuild_python_project::compiled_views::main::project::compiled_project_fac use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; const CANCELLED: &str = "native rules evaluation was cancelled"; -const BUILD_PHASE: &str = "early_rules_build_ms"; +pub(crate) const BUILD_PHASE: &str = "early_rules_build_ms"; const WAIT_PHASE: &str = "early_rules_wait_ms"; const HIT_METRIC: &str = "early_rules_hits"; const MISS_METRIC: &str = "early_rules_misses"; -/// What one compile's early built-in rules start from. -pub(crate) struct EarlyRulesSettings { - pub(crate) project_dir: PathBuf, - pub(crate) dialect: String, - pub(crate) no_sql_analysis: bool, - /// Whether `--no-cache` turned the Rules cache off, as the rules phase does. - pub(crate) no_cache: bool, -} - /// One compile thread's early built-in rules, from arming to the rules phase taking them. -enum Slot { +pub(crate) enum Slot { Armed(EarlyRulesSettings), Started(EarlyBuiltinRules), Spent, } thread_local! { - static SLOT: RefCell> = const { RefCell::new(None) }; + pub(crate) static SLOT: RefCell> = const { RefCell::new(None) }; } /// Whether the rules phase used the early evaluation, and how long it waited for it. @@ -75,63 +65,12 @@ impl EarlyRulesReport { } } -/// Arms early built-in rules for one compile analysis on this thread; dropping it stops them. -pub(crate) struct EarlyRulesScope { - owner: bool, -} - -impl EarlyRulesScope { - /// Claim this thread's slot unarmed; a nested compile analysis on this thread claims nothing. - pub(crate) fn new() -> Self { - let owner: bool = SLOT.with(|slot| { - let mut slot = slot.borrow_mut(); - if slot.is_some() { - return false; - } - *slot = Some(Slot::Spent); - true - }); - Self { owner } - } - - /// Start early rules with `settings` once the compile recovers its model types. - pub(crate) fn arm(&self, settings: EarlyRulesSettings) { - if !self.owner { - return; - } - SLOT.with(|slot| { - if let Some(current @ Slot::Spent) = slot.borrow_mut().as_mut() { - *current = Slot::Armed(settings); - } - }); - } - - /// Cancel and join early rules this scope armed that started and were not taken. - pub(crate) fn stop(&self, py: Python<'_>) { - if self.owner - && let Some(rules) = take_started() - { - rules.discard(py); - } - } -} - -impl Drop for EarlyRulesScope { - fn drop(&mut self) { - if !self.owner { - return; - } - let slot: Option = SLOT.with(|slot| slot.borrow_mut().take()); - drop(slot); - } -} - /// The started early rules, which only the rules phase of the arming compile takes. pub(crate) fn take_early_rules() -> Option { take_started() } -fn take_started() -> Option { +pub(crate) fn take_started() -> Option { SLOT.with(|slot| { let mut slot = slot.borrow_mut(); match slot.as_mut() { @@ -147,41 +86,8 @@ fn take_started() -> Option { }) } -/// Start early built-in rules over `project` when armed; only non-`Exception` errors propagate. -pub(crate) fn start_early_rules(project: &Bound<'_, PyAny>) -> PyResult<()> { - let settings: Option = SLOT.with(|slot| { - let mut slot = slot.borrow_mut(); - match slot.as_mut() { - Some(current @ Slot::Armed(_)) => match std::mem::replace(current, Slot::Spent) { - Slot::Armed(settings) => Some(settings), - _ => None, - }, - _ => None, - } - }); - let Some(settings) = settings else { - return Ok(()); - }; - let py: Python<'_> = project.py(); - let started: Option = - match record_compile_timing(BUILD_PHASE, || early_request(project, &settings)) { - Ok(Some(rows)) => EarlyBuiltinRules::spawn(rows), - Ok(None) => None, - Err(error) if error.is_instance_of::(py) => None, - Err(error) => return Err(error), - }; - if let Some(started) = started { - SLOT.with(|slot| { - if let Some(current) = slot.borrow_mut().as_mut() { - *current = Slot::Started(started); - } - }); - } - Ok(()) -} - /// The built-in rules request the rules phase is expected to build, when it evaluates no custom rules. -fn early_request( +pub(crate) fn early_request( project: &Bound<'_, PyAny>, settings: &EarlyRulesSettings, ) -> PyResult> { @@ -219,7 +125,7 @@ pub(crate) struct EarlyBuiltinRules { impl EarlyBuiltinRules { /// The started speculation, or `None` when its thread cannot start. - fn spawn(rows: RowsRequest) -> Option { + pub(crate) fn spawn(rows: RowsRequest) -> Option { match start_speculative_rows(rows) { Ok(speculative) => Some(Self { speculative, diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/checking/evaluation.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/evaluation.rs new file mode 100644 index 0000000000..aba66deb81 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/evaluation.rs @@ -0,0 +1,15 @@ +//! Register the versioned Python boundary for the native Rules engine. + +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; + +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; + +#[pyfunction] +pub(crate) fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { + py.compiler_detach(|| { + sqlbuild_rules::engine::main::custom_host::run_custom_host_json(spec_json) + }) + .map_err(value_error) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/findings.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/findings.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/getters.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs similarity index 58% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/getters.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs index e3fe8eb100..eea13baabd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/getters.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::rules::models::EarlySqlLint; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_early_sql_lint, -}]; - -fn read_early_sql_lint<'py>( +pub(crate) fn read_early_sql_lint<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/lint.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/lint.rs index 746b7fc8ce..e1002446dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/lint.rs @@ -1,17 +1,16 @@ //! Lint and format SQL with the native rules engine. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::{FromPyObject, pyfunction}; use sqlbuild_rules::lint_text::models::InterpolationSite; use sqlbuild_rules::lint_text::types::PreparedSql; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::models::Positional; -use sqlbuild_python_project::compiled_views::main::compiled_models::register_relation_keys::register_relation_keys; #[pyfunction] -fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { +pub(crate) fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| sqlclean::sql_lint::main::engine::lint_json(request_json)) .map_err(value_error) } @@ -19,7 +18,7 @@ fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { /// One SQL lint preparation request, read from a Python mapping. #[derive(FromPyObject)] #[pyo3(from_item_all)] -struct LintPreparationRequest { +pub(crate) struct LintPreparationRequest { expanded: String, before_expansion: String, prior_sites: Vec, @@ -30,7 +29,7 @@ struct LintPreparationRequest { type PreparedLintSql = (String, Vec>, Vec); #[pyfunction] -fn prepare_lint_sql( +pub(crate) fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, ) -> PyResult> { @@ -54,14 +53,7 @@ fn prepare_lint_sql( } #[pyfunction] -fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { +pub(crate) fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| sqlclean::sql_lint::main::formatter::format_json(request_json)) .map_err(value_error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; - module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; - module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; - register_relation_keys(module) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/checking/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/mod.rs new file mode 100644 index 0000000000..7692ffc619 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod dsl_headers; +pub(crate) mod early_builtin_rules; +pub(crate) mod evaluation; +pub(crate) mod findings; +pub(crate) mod getters; +pub(crate) mod lint; +pub(crate) mod native_header_parser; +pub(crate) mod project_tree_queries; +pub(crate) mod results; +pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs index 2568abb08e..be07a6d854 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs @@ -1,34 +1,26 @@ //! DSL header values parsed natively for header rules, compared as Python compares their objects. +use crate::rules::models::NativeHeaderParser; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; use sqlbuild_rules::dsl_headers::models::{HeaderSpan, ParseFailure, ParseOutcome}; use sqlbuild_rules::dsl_headers::types::HeaderParser; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; -use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; -use sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure; -use sqlbuild_sqltext::header_strings::models::QuoteFailure; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_front::sqltext::main::header_failure::header_failure; /// The path header rules report header syntax errors at. -const SENTINEL_PATH: &str = ""; -const DESCRIPTION_KEY: &str = "description"; +pub(crate) const SENTINEL_PATH: &str = ""; +pub(crate) const DESCRIPTION_KEY: &str = "description"; pub(crate) const FUNCTION_KIND: &str = "FUNCTION"; const ARGUMENTS_KEY: &str = "arguments"; const NEGATIVE_ZERO: &str = "-0"; const ZERO: &str = "0"; -/// Header values parsed natively, compared as their Python projections compare. -#[derive(Default)] -pub(crate) struct NativeHeaderParser; - /// The `ModelSqlParseError` a rejected header parse raises. pub(crate) fn rejection(py: Python<'_>, failure: ParseFailure) -> PyErr { let raised: PyResult = (|| { @@ -41,57 +33,6 @@ pub(crate) fn rejection(py: Python<'_>, failure: ParseFailure) -> PyErr { raised.unwrap_or_else(|error| error) } -impl HeaderParser for NativeHeaderParser { - type Values = AuthoredValue; - /// A native failure's message. - type Error = String; - - fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { - let parsed = match catch_compiler_panic(|| Ok(parse_one(inner))) { - Ok(parsed) => parsed, - Err(message) => { - return ParseOutcome::Raised( - message.clone(), - ParseFailure { - message, - help: None, - position: None, - }, - ); - } - }; - let (value, _, error) = parsed; - if let Some(value) = value { - return ParseOutcome::Parsed(value); - } - let error: String = error.unwrap_or_else(|| "None".to_owned()); - let quote: Option = quote_failure(&error, inner, (1, 1)); - let (detail, help, _) = header_failure(&error, inner, None); - let failure: ParseFailure = ParseFailure { - message: format!( - "{kind}(...) in '{SENTINEL_PATH}' contains invalid SQLBuild header syntax: {detail}" - ), - help, - position: quote.map(|quote| quote.position), - }; - ParseOutcome::Rejected(failure) - } - - fn description(&self, values: &AuthoredValue) -> Result, String> { - Ok(match map_value(values, DESCRIPTION_KEY) { - Some(AuthoredValue::String(text)) => Some(text.clone()), - Some(AuthoredValue::BareWord(word)) if header_word_value(word) == HeaderWord::Text => { - Some(word.clone()) - } - _ => None, - }) - } - - fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> Result { - Ok(python_equal(left, right)) - } -} - /// Why a header could not be read: a header syntax error, or a native failure's message. pub(crate) enum HeaderFailure { Rejected(ParseFailure), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs index d8fa4bf915..e9d744a155 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs @@ -3,14 +3,15 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; -use crate::bindings::_helpers::rules::fact_tree::{Raised, VALUE_ERROR, strerror, text_error}; +use crate::rules::_helpers::fact::fact_tree::Raised; +use crate::rules::_helpers::fact::fact_tree::VALUE_ERROR; +use crate::rules::_helpers::fact::fact_tree::strerror; +use crate::rules::_helpers::fact::fact_tree::text_error; use pyo3::exceptions::PyOSError; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_rules::project_tree::errors::{ ProjectTextError, ProjectTreeFileError, RelativePartsError, }; @@ -21,7 +22,8 @@ use sqlbuild_rules::project_tree::main::project_tree_glob::project_tree_glob; use sqlbuild_rules::project_tree::main::read_project_text::read_project_text; use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_path; -use crate::bindings::_helpers::rules::rule_api::{usage_error_for, validate_project_relative}; +use crate::rules::_helpers::rule::rule_api::usage_error_for; +use crate::rules::_helpers::rule::rule_api::validate_project_relative; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -55,7 +57,7 @@ fn selected<'py>( /// The nodes directly inside the project directory `path`. #[pyfunction] -fn project_tree_child_nodes<'py>( +pub(crate) fn project_tree_child_nodes<'py>( nodes: &Bound<'py, PyAny>, path: &Bound<'py, PyAny>, ) -> PyResult> { @@ -71,7 +73,7 @@ fn project_tree_child_nodes<'py>( /// The nodes below the project directory `path`, or every node for an empty path. #[pyfunction] -fn project_tree_descendant_nodes<'py>( +pub(crate) fn project_tree_descendant_nodes<'py>( nodes: &Bound<'py, PyAny>, path: &Bound<'py, PyAny>, ) -> PyResult> { @@ -91,7 +93,7 @@ fn project_tree_descendant_nodes<'py>( /// The nodes a project-relative shell-style pattern matches. #[pyfunction] -fn project_tree_glob_nodes<'py>( +pub(crate) fn project_tree_glob_nodes<'py>( nodes: &Bound<'py, PyAny>, pattern: &Bound<'py, PyAny>, ) -> PyResult> { @@ -125,7 +127,7 @@ fn not_beneath(path: &Bound<'_, PyAny>, under: &Bound<'_, PyAny>) -> PyErr { /// `path`'s parts relative to the resolved project `root`, then to `under` when given. #[pyfunction] #[pyo3(signature = (*, root, path, under))] -fn project_tree_relative_parts<'py>( +pub(crate) fn project_tree_relative_parts<'py>( root: &Bound<'py, PyAny>, path: &Bound<'py, PyAny>, under: &Bound<'py, PyAny>, @@ -158,7 +160,7 @@ fn project_tree_relative_parts<'py>( /// The UTF-8 text of the tracked project file at `path` under the resolved `root`. #[pyfunction] -fn project_tree_read_text(root: PathBuf, path: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn project_tree_read_text(root: PathBuf, path: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = path.py(); let text: String = path.str()?.to_string_lossy().into_owned(); let raised: Raised = match read_project_text(&root, Path::new(&text)) { @@ -223,12 +225,3 @@ fn python_read_error(py: Python<'_>, file: &Path, text: &str) -> PyErr { }; read().unwrap_or_else(|error| error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(project_tree_child_nodes, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_descendant_nodes, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_glob_nodes, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_relative_parts, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_read_text, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/checking/results.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/results.rs new file mode 100644 index 0000000000..492d3fc632 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/results.rs @@ -0,0 +1,29 @@ +//! Rules results rendered for the CLI, and SQLCLEAN044 help naming files that repeat a literal. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use sqlbuild_rules::rule_results::models::RuleFinding; + +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::path_posix::path_posix; + +pub(crate) fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { + Ok(RuleFinding { + code: field(finding, "code")?.extract()?, + path: path_posix(&field(finding, "path")?)?, + line: field(finding, "line")?.extract()?, + column: field(finding, "column")?.extract()?, + message: field(finding, "message")?.extract()?, + remediation: field(finding, "remediation")?.extract()?, + unevaluated: field(finding, "unevaluated")?.is_truthy()?, + affected_rules: field(finding, "affected_rules")?.extract()?, + fixable: field(finding, "fixable")?.is_truthy()?, + }) +} + +pub(crate) fn rule_findings(findings: &Bound<'_, PyAny>) -> PyResult> { + findings + .try_iter()? + .map(|finding| rule_finding(&finding?)) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs index f7f185d771..09ae2e68bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs @@ -28,13 +28,16 @@ use sqlbuild_rules::models::{ ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; -use crate::bindings::_helpers::rules::config_json::asdict_json; -use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; -use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; -use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; -use crate::bindings::_helpers::rules::rule_run::bulk_cache_file; -use crate::bindings::_helpers::rules::selection::rules_error; -use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; +use crate::rules::_helpers::checking::early_builtin_rules::EarlyBuiltinRules; +use crate::rules::_helpers::checking::findings::FindingClasses; +use crate::rules::_helpers::checking::findings::finding_fault; +use crate::rules::_helpers::configuration::config_json::asdict_json; +use crate::rules::_helpers::configuration::selection::rules_error; +use crate::rules::_helpers::fact::fact_replay::project_attribute; +use crate::rules::_helpers::fact::fact_replay::project_models; +use crate::rules::_helpers::rule::rule_run::bulk_cache_file; +use crate::rules::_helpers::sql::sql_test_objects::scenario_fact; +use crate::rules::_helpers::sql::sql_test_objects::sql_test_fact; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::value_error::value_error; @@ -336,7 +339,9 @@ fn built_request( } request.models = built_models(&rows, &request, model_rows)?; let request: EvaluateRequest = - crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; + crate::compile_delta::main::rules_rows::whole_project_request::whole_project_request( + request, + )?; let delta: bool = sqlbuild_python_base::compile_context::main::delta::injection::injection().is_some(); let memo: Option<(String, PathBuf)> = rows diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/builtin_catalogue.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/builtin_catalogue.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs index 9fe21fe6b7..58819f7235 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs @@ -3,21 +3,23 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_rules::models::RulesConfig; use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSpec, OptionValue}; -use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; -use crate::bindings::_helpers::rules::config_json::asdict_json; -use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules as verify_rule_sources; -use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; -use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_codes}; +use crate::rules::_helpers::configuration::builtin_catalogue::builtin_rules; +use crate::rules::_helpers::configuration::config_json::asdict_json; +use crate::rules::_helpers::configuration::selection::rules_error; +use crate::rules::_helpers::configuration::selection::selected_rule_codes; +use crate::rules::_helpers::custom::custom_rule_evidence::verify_custom_rules as verify_rule_sources; +use crate::rules::_helpers::fact::fact_replay::project_attribute; +use crate::rules::_helpers::fact::fact_replay::project_models; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; @@ -280,24 +282,10 @@ pub(crate) fn verify_custom_rules<'py>( Ok(()) } -/// The validated built-in and custom rule catalogue sorted by code. -#[pyfunction] -#[pyo3(signature = (*, config, project_dir, include_custom = true))] -pub(crate) fn build_rule_catalogue<'py>( - config: &Bound<'py, PyAny>, - project_dir: &Bound<'py, PyAny>, - include_custom: bool, -) -> PyResult> { - PyTuple::new( - config.py(), - rule_catalogue(config, project_dir, include_custom)?, - ) -} - /// The catalogue rules the configuration activates; `custom_payloads` describes custom rules. #[pyfunction] #[pyo3(signature = (*, catalogue, config, project_dir, custom_payloads = None))] -fn select_catalogue_rules<'py>( +pub(crate) fn select_catalogue_rules<'py>( catalogue: &Bound<'py, PyAny>, config: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, @@ -412,16 +400,9 @@ pub(crate) fn resolved_ignore_selectors<'py>( /// `config` with selector-scoped rule ignores resolved against the compiled project. #[pyfunction] #[pyo3(signature = (*, config, project))] -fn resolve_rule_ignore_selectors<'py>( +pub(crate) fn resolve_rule_ignore_selectors<'py>( config: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, ) -> PyResult> { resolved_ignore_selectors(config, project) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_rule_catalogue, module)?)?; - module.add_function(wrap_pyfunction!(select_catalogue_rules, module)?)?; - module.add_function(wrap_pyfunction!(resolve_rule_ignore_selectors, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/config_json.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/config_json.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/config_json.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/config_json.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/guidance.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/guidance.rs index 1036aed53e..cea904579f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/guidance.rs @@ -3,40 +3,36 @@ use std::path::{Path, PathBuf}; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::build_identity::main::native_build_identity::native_build_identity; use sqlbuild_rules::errors::{SkillInstallError, SkillInstallErrorKind}; use sqlbuild_rules::models::RulesConfig; -use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; -use sqlbuild_rules::rule_catalogue::main::install_rendered_skills::install_rendered_skills; use sqlbuild_rules::rule_catalogue::main::render_skills_body::render_skills_body; use sqlbuild_rules::rule_catalogue::main::ruleset_fingerprint::ruleset_fingerprint; use sqlbuild_rules::rule_catalogue::main::skill_lock_name::skill_lock_name; -use sqlbuild_rules::rule_catalogue::models::{FingerprintRule, OptionText, RuleText, SkillTarget}; +use sqlbuild_rules::rule_catalogue::models::{FingerprintRule, OptionText, RuleText}; -use crate::bindings::_helpers::rules::catalogue::{ - native_rules_config, references_custom_rules, rule_catalogue, select_rules, verify_custom_rules, -}; -use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; -use crate::bindings::_helpers::rules::selection::rules_error; -use crate::bindings::_helpers::rules::skills::{owned_skill, skill_state}; +use crate::rules::_helpers::configuration::catalogue::native_rules_config; +use crate::rules::_helpers::configuration::catalogue::references_custom_rules; +use crate::rules::_helpers::configuration::catalogue::rule_catalogue; +use crate::rules::_helpers::configuration::catalogue::select_rules; +use crate::rules::_helpers::configuration::catalogue::verify_custom_rules; +use crate::rules::_helpers::configuration::selection::rules_error; +use crate::rules::_helpers::configuration::skills::owned_skill; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_implementation_fingerprint; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -const FILELOCK_MODULE: &str = "filelock"; -const SKILL_PATHS: [&str; 3] = [ +pub(crate) const FILELOCK_MODULE: &str = "filelock"; +pub(crate) const SKILL_PATHS: [&str; 3] = [ ".agents/skills/sqlbuild-rules/SKILL.md", ".claude/skills/sqlbuild-rules/SKILL.md", ".opencode/skills/sqlbuild-rules/SKILL.md", ]; const SKILL_LOCK_DIRECTORY: &str = "sqlbuild-rules-skill-locks"; -const FRESH_SKILL: &str = "fresh"; +pub(crate) const FRESH_SKILL: &str = "fresh"; fn text_field(object: &Bound<'_, PyAny>, name: &str) -> PyResult { record_field(object, name)?.extract() @@ -47,7 +43,10 @@ fn repr_text(value: &Bound<'_, PyAny>) -> PyResult { } /// The text view of one Python `Rule`, with effective options under `config` when given. -fn rule_text(rule: &Bound<'_, PyAny>, config: Option<&Bound<'_, PyAny>>) -> PyResult { +pub(crate) fn rule_text( + rule: &Bound<'_, PyAny>, + config: Option<&Bound<'_, PyAny>>, +) -> PyResult { let code: String = text_field(rule, "code")?; let configured: Option> = match config { Some(config) => record_field(config, "rule_options")? @@ -90,19 +89,6 @@ fn rule_text(rule: &Bound<'_, PyAny>, config: Option<&Bound<'_, PyAny>>) -> PyRe }) } -/// `sqb rules show` text for one rule, with effective thresholds when a config is given. -#[pyfunction] -#[pyo3(signature = (*, rule, config = None))] -pub(crate) fn format_rule( - rule: &Bound<'_, PyAny>, - config: Option>, -) -> PyResult { - let config: Option> = config.filter(|config| !config.is_none()); - let native: Option = config.as_ref().map(native_rules_config).transpose()?; - let text: RuleText = rule_text(rule, config.as_ref())?; - compiler_guard(|| Ok(format_rule_text(&text, native.as_ref()))) -} - fn fingerprint_rule( rule: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, @@ -124,7 +110,7 @@ fn fingerprint_rule( } /// The generated skill content and the ruleset fingerprint it records. -fn render_skills( +pub(crate) fn render_skills( config: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, ) -> PyResult<(String, String)> { @@ -157,14 +143,14 @@ fn render_skills( Ok((owned_skill(py, &body, &fingerprint)?, fingerprint)) } -fn current_content(path: &Path) -> PyResult> { +pub(crate) fn current_content(path: &Path) -> PyResult> { if !path.is_file() { return Ok(None); } std::fs::read_to_string(path).map(Some).map_err(PyErr::from) } -fn install_error(py: Python<'_>, error: SkillInstallError) -> PyErr { +pub(crate) fn install_error(py: Python<'_>, error: SkillInstallError) -> PyErr { let raised: PyErr = match error.kind { SkillInstallErrorKind::Refused => rules_error(py, &error.message, None), SkillInstallErrorKind::Io => PyOSError::new_err(error.message), @@ -175,7 +161,7 @@ fn install_error(py: Python<'_>, error: SkillInstallError) -> PyErr { raised } -fn lock_path(project_dir: &Path) -> PyResult { +pub(crate) fn lock_path(project_dir: &Path) -> PyResult { let canonical: String = project_dir.canonicalize()?.display().to_string(); let canonical: String = if cfg!(windows) { canonical.to_lowercase().replace('/', "\\") @@ -186,46 +172,3 @@ fn lock_path(project_dir: &Path) -> PyResult { std::fs::create_dir_all(&directory)?; Ok(directory.join(skill_lock_name(&canonical))) } - -/// Install every owned rules skill target, or with `check` report whether all are fresh. -#[pyfunction] -#[pyo3(signature = (*, config, project_dir, check))] -pub(crate) fn install_rules_skills( - config: &Bound<'_, PyAny>, - project_dir: &Bound<'_, PyAny>, - check: bool, -) -> PyResult { - let py: Python<'_> = config.py(); - let (content, fingerprint): (String, String) = render_skills(config, project_dir)?; - let root: PathBuf = project_dir.extract()?; - let targets = || -> PyResult> { - let mut targets: Vec = Vec::with_capacity(SKILL_PATHS.len()); - for relative in SKILL_PATHS { - let path: PathBuf = root.join(relative); - let state: String = skill_state(current_content(&path)?.as_deref(), &fingerprint); - targets.push(SkillTarget { path, state }); - } - Ok(targets) - }; - if check { - return Ok(targets()?.iter().all(|target| target.state == FRESH_SKILL)); - } - let lock: Bound<'_, PyAny> = - imported(py, FILELOCK_MODULE, "FileLock")?.call1((lock_path(&root)?,))?; - let _ = lock.call_method0("acquire")?; - let installed = (|| -> PyResult<()> { - let targets: Vec = targets()?; - compiler_guard(|| Ok(install_rendered_skills(&content, &targets)))? - .map_err(|error| install_error(py, error)) - })(); - let released = lock.call_method0("release"); - installed?; - let _ = released?; - Ok(true) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_rule, module)?)?; - module.add_function(wrap_pyfunction!(install_rules_skills, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/mod.rs new file mode 100644 index 0000000000..30fa76d9f3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod builtin_catalogue; +pub(crate) mod catalogue; +pub(crate) mod config_json; +pub(crate) mod guidance; +pub(crate) mod rules_config; +pub(crate) mod selection; +pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/rules_config.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/rules_config.rs index 38e16e10be..04663e146e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/rules_config.rs @@ -1,18 +1,11 @@ //! Rules configuration loaded natively and returned as the Python configuration models. -use std::path::{Path, PathBuf}; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, Py, pyfunction, wrap_pyfunction}; use serde_json::Value; -use sqlbuild_rules::configuration::errors::RulesConfigLoadError; -use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::json_value_to_python::json_value_to_python; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -95,7 +88,10 @@ fn tables<'py, T>( Ok(PyTuple::new(py, values)?.into_any()) } -fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { +pub(crate) fn config_object<'py>( + py: Python<'py>, + config: &RulesConfig, +) -> PyResult> { let models = Models { py }; let threshold_overrides = tables(py, &config.threshold_overrides, |item| { models.build( @@ -203,7 +199,12 @@ fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult, class: ErrorId, message: String, help: Option) -> PyErr { +pub(crate) fn rules_error( + py: Python<'_>, + class: ErrorId, + message: String, + help: Option, +) -> PyErr { let raised = || -> PyResult { let error_class: Bound<'_, PyAny> = error_class(py, class)?; let error: Bound<'_, PyAny> = match help { @@ -218,67 +219,3 @@ fn rules_error(py: Python<'_>, class: ErrorId, message: String, help: Option, project_dir: PathBuf) -> PyResult> { - load_rules_config_with_cache(py, project_dir, true) -} - -/// The project's rules configuration, with the Rules cache off when `cache` is `false`. -pub(crate) fn load_rules_config_with_cache( - py: Python<'_>, - project_dir: PathBuf, - cache: bool, -) -> PyResult> { - let mut config: RulesConfig = match rules_config_of(&project_dir)? { - Ok(config) => config, - Err(invalid) => return Err(invalid.into_pyerr(py)), - }; - config.cache.enabled &= cache; - Ok(config_object(py, &config)?.unbind()) -} - -/// Why a project's rules configuration is invalid. -pub(crate) enum InvalidRulesConfig { - /// The configuration could not be read: a `RulesError`. - Unreadable(String), - /// It uses a setting SQLBuild rejects: a `RulesConfigError` with help. - Forbidden { message: String, help: String }, -} - -impl InvalidRulesConfig { - fn into_pyerr(self, py: Python<'_>) -> PyErr { - match self { - Self::Unreadable(message) => rules_error(py, ErrorId::Rules, message, None), - Self::Forbidden { message, help } => { - rules_error(py, ErrorId::RulesConfig, message, Some(help)) - } - } - } -} - -/// The project's validated rules configuration, or why it is invalid. -pub(crate) fn rules_config_of( - project_dir: &Path, -) -> PyResult> { - let project_dir: PathBuf = project_dir - .canonicalize() - .or_else(|_| std::path::absolute(project_dir))?; - match compiler_guard(|| Ok(load_config(&project_dir)))? { - Ok(config) => Ok(Ok(config)), - Err(RulesConfigLoadError::Invalid(message)) => { - Ok(Err(InvalidRulesConfig::Unreadable(message))) - } - Err(RulesConfigLoadError::Policy(error)) => Ok(Err(InvalidRulesConfig::Forbidden { - message: error.message, - help: error.help, - })), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/selection.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/selection.rs index 07fab2e972..d0e95f6a49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/selection.rs @@ -1,17 +1,16 @@ //! The active rule codes of a rules configuration and its custom rules, resolved natively. use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; -use crate::bindings::_helpers::rules::config_json::{asdict_json, dumped_json}; -use crate::bindings::_helpers::rules::constants::RULES_NATIVE_API_VERSION; +use crate::rules::_helpers::configuration::config_json::asdict_json; +use crate::rules::_helpers::configuration::config_json::dumped_json; +use crate::rules::constants::RULES_NATIVE_API_VERSION; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::types::ErrorId; @@ -105,8 +104,3 @@ pub(crate) fn rules_error(py: Python<'_>, message: &str, cause: Option) - } error } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(selected_rule_codes, module)?)?; - module.add_function(wrap_pyfunction!(rule_selected, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/skills.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/configuration/skills.rs index c211162397..91d25c519b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/configuration/skills.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{PyResult, Python}; -use crate::bindings::_helpers::rules::selection::rules_error; +use crate::rules::_helpers::configuration::selection::rules_error; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; const SKILL_OWNER: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs index 59efe4c926..4e46d7c6df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs @@ -4,9 +4,9 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use std::time::Instant; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBytes, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonValue, OrjsonOptions}; @@ -16,24 +16,21 @@ use sqlbuild_rules::custom_host_pool::types::{FactKey, HostPlan}; use sqlbuild_rules::custom_rule_cache::models::CustomEvaluation; use sqlbuild_rules::custom_rule_cache::types::ReadSet; -use crate::bindings::_helpers::rules::constants::{ - CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULES_CACHE_FILE, -}; -use crate::bindings::_helpers::rules::custom_host_columns::custom_host_declared_columns; -use crate::bindings::_helpers::rules::custom_host_payload::{ - CUSTOM_HOST_PAYLOAD_MAGIC, custom_host_payload, -}; -use crate::bindings::_helpers::rules::custom_hosts::{HostCancel, HostRequest, run_custom_hosts}; -use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; -use crate::bindings::_helpers::rules::custom_rule_cache::custom_rule_identity; -use crate::bindings::_helpers::rules::custom_rule_cache::stored_rules; -use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; -use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_import_closure; -use crate::bindings::_helpers::rules::fact_encoder::{Encoded, reproducible}; -use crate::bindings::_helpers::rules::fact_replay::{ - FactReplay, delta_safe_fact, is_tree_fact, read_text_path, -}; -use crate::bindings::_helpers::rules::rule_run::{bulk_cache_file, rules_bulk_cache_path}; +use crate::rules::_helpers::custom::custom_hosts::HostCancel; +use crate::rules::_helpers::custom::custom_hosts::HostRequest; +use crate::rules::_helpers::custom::custom_hosts::run_custom_hosts; +use crate::rules::_helpers::custom::custom_rule_cache::CustomRuleCache; +use crate::rules::_helpers::custom::custom_rule_cache::custom_rule_identity; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_implementation_fingerprint; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_import_closure; +use crate::rules::_helpers::fact::fact_encoder::Encoded; +use crate::rules::_helpers::fact::fact_encoder::reproducible; +use crate::rules::_helpers::fact::fact_replay::FactReplay; +use crate::rules::_helpers::fact::fact_replay::delta_safe_fact; +use crate::rules::_helpers::fact::fact_replay::is_tree_fact; +use crate::rules::_helpers::rule::rule_run::bulk_cache_file; +use crate::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; +use crate::rules::constants::CUSTOM_RULES_CACHE_FILE; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; @@ -46,11 +43,12 @@ use sqlbuild_python_base::records::types::RecordId; const PROJECT_SUBJECT: &str = "project"; /// Why a delta compile cannot rerun a custom rule that reads facts of models it does not compile. -const READ_MORE: &str = "a custom rule read more than the changed models"; -const PROJECT_WIDE: &str = "a project-wide custom rule must run again"; -const OUTSIDE_DELTA: &str = "a custom rule must run again for a model the delta does not compile"; +pub(crate) const READ_MORE: &str = "a custom rule read more than the changed models"; +pub(crate) const PROJECT_WIDE: &str = "a project-wide custom rule must run again"; +pub(crate) const OUTSIDE_DELTA: &str = + "a custom rule must run again for a model the delta does not compile"; /// The custom-rule cache's subject key of project-wide rules. -const PROJECT_SUBJECT_KEY: &str = ""; +pub(crate) const PROJECT_SUBJECT_KEY: &str = ""; const MODEL_SUBJECT: &str = "model"; /// The run one custom-rule evaluation describes. pub(crate) struct CustomRun<'a, 'py> { @@ -237,43 +235,6 @@ fn delta_plan_refusal( None } -/// Why a delta of `model_paths` would stop at custom rules, judged from their stored read sets. -pub(crate) fn stored_custom_rule_refusal( - project_dir: &Path, - (model_paths, changed): (&BTreeSet, &BTreeSet), - all_sources: bool, -) -> Option<&'static str> { - let stored = stored_rules(&rules_bulk_cache_path( - project_dir.to_path_buf(), - CUSTOM_RULES_CACHE_FILE, - )); - for rule in stored.values() { - for (subject, entry) in &rule.entries { - let Some(readset) = rule.readsets.get(entry.readset) else { - continue; - }; - if model_paths.contains(subject) { - if !readset - .iter() - .all(|key| delta_safe_fact(key, model_paths, all_sources)) - { - return Some(READ_MORE); - } - } else if readset - .iter() - .any(|key| read_text_path(key).is_some_and(|path| changed.contains(path))) - { - return Some(if subject == PROJECT_SUBJECT_KEY { - PROJECT_WIDE - } else { - OUTSIDE_DELTA - }); - } - } - } - None -} - /// Whether a delta compile evaluated a custom rule from facts it holds in full. fn delta_safe_evaluation(evaluation: &CustomEvaluation, injection: &DeltaInjection) -> bool { let all_sources: bool = injection.source_names.is_none(); @@ -500,20 +461,10 @@ fn cpu_seconds(py: Python<'_>) -> PyResult { /// The digest the custom-rule cache keys one recorded fact on, or `None` when unreproducible. #[pyfunction] #[pyo3(signature = (*, project, project_dir, key))] -fn custom_rule_fact_digest( +pub(crate) fn custom_rule_fact_digest( project: &Bound<'_, PyAny>, project_dir: PathBuf, key: FactKey, ) -> PyResult> { reproducible(FactReplay::new(project, &project_dir)?.digest(&key)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(custom_host_payload, module)?)?; - module.add_function(wrap_pyfunction!(custom_host_declared_columns, module)?)?; - module.add( - "CUSTOM_HOST_PAYLOAD_MAGIC", - PyBytes::new(module.py(), CUSTOM_HOST_PAYLOAD_MAGIC), - )?; - module.add_function(wrap_pyfunction!(custom_rule_fact_digest, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_columns.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_columns.rs index ce808b309e..9ba4b0d683 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_columns.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_columns.rs @@ -11,7 +11,7 @@ use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::models::{ParsedThresholds, ThresholdBound}; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::rules::custom_hosts::rules_error; +use crate::rules::_helpers::custom::custom_hosts::rules_error; use sqlbuild_python_front::model_config::main::headers::native_column_record::native_column_record; /// One native header column with the text of the path its location names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_observation.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_observation.rs index b2a44b684b..80e39956bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_observation.rs @@ -3,9 +3,7 @@ use std::path::Path; use std::sync::{Arc, Mutex, MutexGuard}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{pyclass, pymethods}; @@ -13,7 +11,7 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; /// The `sqlbuild._native` attribute tests set to a `CustomHostObserver` to observe host runs. -const OBSERVER_ATTRIBUTE: &str = "custom_host_observer"; +pub(crate) const OBSERVER_ATTRIBUTE: &str = "custom_host_observer"; /// The requests of every host launched while observed, in launch order. pub(crate) type LaunchLog = Arc>>; @@ -106,8 +104,3 @@ fn locked(launches: &LaunchLog) -> MutexGuard<'_, Vec> { Err(poisoned) => poisoned.into_inner(), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add(OBSERVER_ATTRIBUTE, module.py().None()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs index 372d122c2e..0a6cc63b53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs @@ -12,11 +12,11 @@ use pyo3::types::{ use serde_json::{Map, Value, json}; use sqlbuild_discovery::models::LineColumnSpan; -use crate::bindings::_helpers::rules::custom_host_columns::PayloadColumn; -use crate::bindings::_helpers::rules::custom_hosts::pickled; -use crate::bindings::_helpers::rules::fact_replay::{ - ModelEntry, project_attribute, project_models, -}; +use crate::rules::_helpers::custom::custom_host_columns::PayloadColumn; +use crate::rules::_helpers::custom::custom_hosts::pickled; +use crate::rules::_helpers::fact::fact_replay::ModelEntry; +use crate::rules::_helpers::fact::fact_replay::project_attribute; +use crate::rules::_helpers::fact::fact_replay::project_models; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::compiled_views::main::compiled_models::enum_columns::enum_columns; use sqlbuild_python_project::compiled_views::main::project::compiled_audit_record::compiled_audit_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs index 9ff9c6b1a7..532b3414d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs @@ -29,13 +29,15 @@ use sqlbuild_rules::custom_host_pool::models::{HostRuleSubject, HostRun, HostSli use sqlbuild_rules::custom_host_pool::types::{HostInvocation, HostPlan}; use sqlbuild_rules::engine::main::custom_host::run_custom_host_json; -use crate::bindings::_helpers::rules::constants::{ - CUSTOM_HOST_ABANDONED_SUFFIX, CUSTOM_HOST_CANCELLED_MARKER, CUSTOM_HOST_RUNTIME_VERSION, -}; -use crate::bindings::_helpers::rules::custom_host_observation::{ - LaunchLog, ObservedHosts, before_publish, observed_hosts, record_launch, -}; -use crate::bindings::_helpers::rules::custom_host_payload::write_payload; +use crate::rules::_helpers::custom::custom_host_observation::LaunchLog; +use crate::rules::_helpers::custom::custom_host_observation::ObservedHosts; +use crate::rules::_helpers::custom::custom_host_observation::before_publish; +use crate::rules::_helpers::custom::custom_host_observation::observed_hosts; +use crate::rules::_helpers::custom::custom_host_observation::record_launch; +use crate::rules::_helpers::custom::custom_host_payload::write_payload; +use crate::rules::constants::CUSTOM_HOST_ABANDONED_SUFFIX; +use crate::rules::constants::CUSTOM_HOST_CANCELLED_MARKER; +use crate::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_cache.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_cache.rs index 7a5a3a8d2f..9750f10e37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_cache.rs @@ -3,9 +3,9 @@ use std::collections::BTreeMap; use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; use sqlbuild_rules::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; use sqlbuild_rules::custom_rule_cache::main::ordered_custom_findings::ordered_custom_findings; @@ -21,7 +21,8 @@ use sqlbuild_rules::custom_rule_cache::models::{ }; use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResults}; -use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; +use crate::rules::_helpers::checking::findings::FindingClasses; +use crate::rules::_helpers::checking::findings::invalid_finding; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; @@ -226,8 +227,3 @@ impl CustomRuleCache { } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(custom_rule_identity, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs index 97757d27ab..c10682d98c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs @@ -4,11 +4,9 @@ use sqlbuild_python_base::positional_record; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::path::{Path, PathBuf}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_pysource::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; use sqlbuild_pysource::rule_evidence::main::disallowed_import::disallowed_import; use sqlbuild_pysource::rule_evidence::main::implementation_fingerprint::implementation_fingerprint; @@ -106,7 +104,7 @@ pub(crate) fn custom_rule_implementation_fingerprint( positional_record! { into; /// `(path, line, column, owner)` of one public-harness case. - struct EvidenceRow { + pub(crate) struct EvidenceRow { path: String, line: usize, column: usize, @@ -117,7 +115,7 @@ positional_record! { /// `(path, line, column, owner)` of each public-harness case that statically evaluates the rule. #[pyfunction] #[pyo3(signature = (*, rule, project_dir))] -fn custom_rule_test_evidence<'py>( +pub(crate) fn custom_rule_test_evidence<'py>( rule: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { @@ -157,7 +155,7 @@ fn rules_error(py: Python<'_>, error_id: ErrorId, message: String) -> PyErr { /// Whether custom-rule code may import one absolute module name. #[pyfunction] #[pyo3(name = "allowed_custom_rule_import")] -fn allowed_custom_rule_import_binding(module: &str) -> bool { +pub(crate) fn allowed_custom_rule_import_binding(module: &str) -> bool { allowed_custom_rule_import(module) } @@ -241,18 +239,3 @@ pub(crate) fn verify_custom_rules( } Ok(()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - allowed_custom_rule_import_binding, - module - )?)?; - module.add_function(wrap_pyfunction!(verify_custom_rules, module)?)?; - module.add_function(wrap_pyfunction!(custom_rule_import_closure, module)?)?; - module.add_function(wrap_pyfunction!( - custom_rule_implementation_fingerprint, - module - )?)?; - module.add_function(wrap_pyfunction!(custom_rule_test_evidence, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/mod.rs new file mode 100644 index 0000000000..08bb4eba5a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod custom_evaluation; +pub(crate) mod custom_host_columns; +pub(crate) mod custom_host_observation; +pub(crate) mod custom_host_payload; +pub(crate) mod custom_hosts; +pub(crate) mod custom_rule_cache; +pub(crate) mod custom_rule_evidence; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_encoder.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_encoder.rs index 486fc6c47a..664b0ad572 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_encoder.rs @@ -4,16 +4,13 @@ use std::cell::RefCell; use std::collections::HashMap; use std::collections::hash_map::Entry; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, PyTypeMethods, - Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, PyTypeMethods, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyStringMethods, PyTuple, PyType, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonValue, OrjsonOptions}; @@ -302,13 +299,13 @@ pub(crate) fn error_digest(kind: &str, message: &str) -> Encoded { /// The digest a rule host records for a live filesystem fact's value, or `None`. #[pyfunction] -fn fact_value_digest(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn fact_value_digest(value: &Bound<'_, PyAny>) -> PyResult> { reproducible(FactEncoder::new().encode(value).and_then(value_digest)) } /// The digest a rule host records for a live filesystem fact that raised `error`, or `None`. #[pyfunction] -fn fact_error_digest(error: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn fact_error_digest(error: &Bound<'_, PyAny>) -> PyResult> { let kind: String = error.get_type().name()?.to_string(); let message: String = error.str()?.to_string_lossy().into_owned(); reproducible(error_digest(&kind, &message)) @@ -322,8 +319,3 @@ pub(crate) fn reproducible(encoded: Encoded) -> PyResult> { Err(FactFailure::Python(error)) => Err(error), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fact_value_digest, module)?)?; - module.add_function(wrap_pyfunction!(fact_error_digest, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs index 948466a24e..a12a8f9472 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs @@ -13,13 +13,20 @@ use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_pa use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; -use crate::bindings::_helpers::rules::fact_encoder::{ - Encoded, FactEncoder, FactFailure, error_digest, value_digest, -}; -use crate::bindings::_helpers::rules::fact_tree::{ - Outcome, Raised, TreeQuery, nodes_value, path_string, read_text, relative_parts, - selected_nodes, tree_nodes, -}; +use crate::rules::_helpers::fact::fact_encoder::Encoded; +use crate::rules::_helpers::fact::fact_encoder::FactEncoder; +use crate::rules::_helpers::fact::fact_encoder::FactFailure; +use crate::rules::_helpers::fact::fact_encoder::error_digest; +use crate::rules::_helpers::fact::fact_encoder::value_digest; +use crate::rules::_helpers::fact::fact_tree::Outcome; +use crate::rules::_helpers::fact::fact_tree::Raised; +use crate::rules::_helpers::fact::fact_tree::TreeQuery; +use crate::rules::_helpers::fact::fact_tree::nodes_value; +use crate::rules::_helpers::fact::fact_tree::path_string; +use crate::rules::_helpers::fact::fact_tree::read_text; +use crate::rules::_helpers::fact::fact_tree::relative_parts; +use crate::rules::_helpers::fact::fact_tree::selected_nodes; +use crate::rules::_helpers::fact::fact_tree::tree_nodes; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as record_field_or_attribute; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_tree.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_tree.rs index fa3dec8b4c..5e282d5ba0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_tree.rs @@ -20,7 +20,9 @@ use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_pa use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; -use crate::bindings::_helpers::rules::fact_encoder::{Encoded, FactEncoder, FactFailure}; +use crate::rules::_helpers::fact::fact_encoder::Encoded; +use crate::rules::_helpers::fact::fact_encoder::FactEncoder; +use crate::rules::_helpers::fact::fact_encoder::FactFailure; /// The qualified name of the `ProjectPath` record class. const PROJECT_PATH_RECORD: &str = "sqlbuild._native.ProjectPath"; diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/fact/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/mod.rs new file mode 100644 index 0000000000..25ccbf6ee2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod fact_encoder; +pub(crate) mod fact_replay; +pub(crate) mod fact_tree; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_facts.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_facts.rs index 3fa3378d9d..fecf350999 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_facts.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::rules::fact_replay::project_attribute; +use crate::rules::_helpers::fact::fact_replay::project_attribute; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs index 2132e21dd0..dbc1f8413b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs @@ -4,33 +4,32 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::pyfunction; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, }; use sqlbuild_core::text::main::python_strip::python_strip; -use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_rules::fix_verification::main::refusal::refusal; use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; -use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; -use crate::bindings::_helpers::rules::fix_facts::compile_facts; -use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; -use crate::bindings::_helpers::rules::lint_expansion::{ - LintExpansionSources, lint_expansion_inputs, -}; -use crate::bindings::_helpers::rules::lint_pipeline::{ - LintFile, LintOutcome, LintRequest, LintSettings, ViolationClasses, lint, lint_sources, -}; -use crate::bindings::_helpers::rules::run_lint::given_files; +use crate::compile_attachment::models::ExpansionVars; +use crate::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use crate::rules::_helpers::fix::fix_facts::compile_facts; +use crate::rules::_helpers::fix::fix_verdict::fix_verdict; +use crate::rules::_helpers::lint::lint_expansion::LintExpansionSources; +use crate::rules::_helpers::lint::lint_expansion::lint_expansion_inputs; +use crate::rules::_helpers::lint::lint_pipeline::LintRequest; +use crate::rules::_helpers::lint::lint_pipeline::LintSettings; +use crate::rules::_helpers::lint::lint_pipeline::lint; +use crate::rules::_helpers::lint::lint_pipeline::lint_sources; +use crate::rules::_helpers::lint::run_lint::given_files; +use crate::rules::constants::SAFE_RULE_FIX_CODES; +use crate::rules::constants::SQL_RULE_PREFIX; +use crate::rules::models::LintFile; +use crate::rules::models::LintOutcome; +use crate::rules::models::ViolationClasses; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -48,17 +47,17 @@ const VERIFICATION_FAILURE: &str = const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; const MAX_PASSES: usize = 128; -struct FixSession<'py> { - project_dir: Bound<'py, PyAny>, - original: Bound<'py, PyDict>, - config: Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, - inputs: Bound<'py, PyAny>, - reports: Vec>, - blocked: Bound<'py, PyDict>, - skipped: Bound<'py, PyDict>, - baseline: Option>, - baseline_facts: Option>, +pub(crate) struct FixSession<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) original: Bound<'py, PyDict>, + pub(crate) config: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) inputs: Bound<'py, PyAny>, + pub(crate) reports: Vec>, + pub(crate) blocked: Bound<'py, PyDict>, + pub(crate) skipped: Bound<'py, PyDict>, + pub(crate) baseline: Option>, + pub(crate) baseline_facts: Option>, } type PlannedPass<'py> = ( @@ -68,58 +67,8 @@ type PlannedPass<'py> = ( ); type FixState = (Vec<(PathBuf, String)>, Vec); -/// Plan fixes without exposing any unverified intermediate source on disk. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn plan_rule_fixes<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "plan_rule_fixes", - options, - ( - &[ - "project_dir", - "files", - "config", - "adapter", - "discovered_inputs", - "enabled", - ], - &[], - ), - )?; - let files = args.value("files")?.cast_into::()?; - if !args.flag("enabled")? { - return result(&files, &[], &[]); - } - let adapter = args.value("adapter")?; - let inputs = args.value("discovered_inputs")?; - if !adapter.is_instance(&host_entry(py, AdapterHost::BaseAdapter)?)? || inputs.is_none() { - return Err(PyErr::from_value( - error_class(py, ErrorId::ProjectCompile)? - .call1(("format --fix requires discovered compiler inputs and an adapter",))?, - )); - } - FixSession { - project_dir: args.value("project_dir")?, - original: files, - config: args.value("config")?, - adapter, - inputs, - reports: Vec::new(), - blocked: PyDict::new(py), - skipped: PyDict::new(py), - baseline: None, - baseline_facts: None, - } - .run() -} - impl<'py> FixSession<'py> { - fn run(&mut self) -> PyResult> { + pub(crate) fn run(&mut self) -> PyResult> { let py = self.original.py(); let mut current = self.original.copy()?; let paths: Vec> = field(&self.inputs, "model_files")? @@ -742,7 +691,7 @@ fn sorted_items<'py>( sorted.extract() } -fn result<'py>( +pub(crate) fn result<'py>( files: &Bound<'py, PyDict>, reports: &[Bound<'py, PyAny>], faults: &[Bound<'py, PyAny>], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_verdict.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_verdict.rs diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/fix/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/mod.rs new file mode 100644 index 0000000000..60a13a03f1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fix/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod fix_facts; +pub(crate) mod fix_session; +pub(crate) mod fix_verdict; diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/lint/early_sql_lint.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/early_sql_lint.rs new file mode 100644 index 0000000000..b77db3d6cc --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/early_sql_lint.rs @@ -0,0 +1,231 @@ +//! Speculative SQL checks started once compile inputs are ready, and the lint settings they use. + +use crate::rules::main::configuration::rules_config_of::rules_config_of; +use crate::rules::models::EarlyLintResult; +use crate::rules::models::ExpansionReuse; +use crate::rules::models::PreparedSqlLint; +use std::path::PathBuf; + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread::JoinHandle; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_rules::lint_text::models::LintSqlOptions; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; + +use crate::compile_attachment::main::present::present; +use crate::rules::_helpers::lint::lint_expansion::LintExpansionSources; +use crate::rules::_helpers::lint::lint_expansion::lint_expansion_inputs; +use crate::rules::_helpers::lint::lint_inputs::model_lint_expansion; +use crate::rules::_helpers::lint::lint_pipeline::CompiledSource; +use crate::rules::_helpers::lint::lint_pipeline::LintRequest; +use crate::rules::_helpers::lint::lint_pipeline::LintSettings; +use crate::rules::_helpers::lint::lint_pipeline::lint; +use crate::rules::_helpers::lint::lint_pipeline::lint_sources; +use crate::rules::_helpers::rule::rule_cache::stored_entries; +use crate::rules::models::LintFile; +use crate::rules::models::LintOutcome; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as record_field; +use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key_row::model_key_row; +use sqlbuild_python_project::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::relation_keys::relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::source_key_input::source_key_input; +use sqlbuild_python_project::compiled_views::models::RelationKeyRow; +use sqlbuild_python_project::compiled_views::types::RelationKeys; + +/// The lint thresholds used when the project leaves them unset. +const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; +const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; +const MAX_LITERAL_LENGTH: &str = "max_literal_length"; +const MAX_RANKING_ORDER_BY: &str = "max_ranking_order_by"; +const LINT_THREAD: &str = "sqlbuild-lint"; +const SQL_RULES_CACHE_FILE: &str = "sql.json"; + +/// The SQL rule result cache file of `project_dir`. +pub(crate) fn sql_rule_cache_path(project_dir: &Path) -> PathBuf { + rules_bulk_cache_path(project_dir, SQL_RULES_CACHE_FILE) +} + +/// The native SQL lint settings that SQL rules run with. +pub(crate) fn sql_lint_settings( + dialect: &str, + codes: &[String], + threshold: impl Fn(&str) -> PyResult>, + relation_keys: RelationKeys, +) -> PyResult { + let defaults: LintSettings = LintSettings::default(); + Ok(LintSettings { + header_rules_enabled: false, + sql: LintSqlOptions { + dialect: dialect.to_owned(), + enabled_rules: Some(codes.to_vec()), + max_literal_length: threshold(MAX_LITERAL_LENGTH)? + .unwrap_or(DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by: threshold(MAX_RANKING_ORDER_BY)? + .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), + ..defaults.sql + }, + relation_keys, + ..defaults + }) +} + +/// What early SQL lint runs with: the project, its selected SQL rule codes and lint settings. +struct EarlyLintPlan { + project_dir: PathBuf, + codes: Vec, + settings: LintSettings, +} + +/// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. +fn early_lint_plan( + inputs: &Bound<'_, PyAny>, + dialect: &str, + no_cache: bool, +) -> PyResult> { + let py: Python<'_> = inputs.py(); + let project_dir: Bound<'_, PyAny> = + record_field(&record_field(inputs, "discovered_inputs")?, "project_dir")?; + if project_dir.is_none() { + return Ok(None); + } + let project_dir: PathBuf = project_dir.extract()?; + for diagnostic in record_field(inputs, "diagnostics")?.try_iter()? { + if record_field(&diagnostic?, "is_error")?.is_truthy()? { + return Ok(None); + } + } + let Ok(mut config) = rules_config_of(&project_dir)? else { + return Ok(None); + }; + config.cache.enabled &= !no_cache; + if config.cache.enabled && !stored_entries(&sql_rule_cache_path(&project_dir))?.is_empty() { + return Ok(None); + } + let codes: Vec = + compiler_guard(|| Ok(selected_sql_codes(config.select.as_deref(), &config.ignore)))?; + if codes.is_empty() { + return Ok(None); + } + let mut rows: Vec = Vec::new(); + for model in record_field(inputs, "model_inputs")?.try_iter()? { + rows.push(model_key_row(&model?)?); + } + let seed_inputs: Vec> = record_field(inputs, "seed_inputs")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&seed_inputs)?; + let mut seeds: Vec> = Vec::new(); + for seed in &seed_inputs { + seeds.push(record_field(seed, "schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in record_field(inputs, "source_inputs")?.try_iter()? { + sources.push(source_key_input(&source?)?); + } + let keys: RelationKeys = relation_keys( + py, + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), + &one_row_seeds, + )?; + let settings: LintSettings = sql_lint_settings( + dialect, + &codes, + |key| Ok(config.thresholds.get(key).map(|value| i64::from(*value))), + keys, + )?; + Ok(Some(EarlyLintPlan { + project_dir, + codes, + settings, + })) +} + +pub(crate) fn same_codes(left: &[String], right: &[String]) -> bool { + left.iter().collect::>() == right.iter().collect::>() +} + +/// Start early SQL lint on a native thread when the project selects SQL rules without a cache. +pub(crate) fn start_early_lint( + (inputs, dialect): (&Bound<'_, PyAny>, &str), + reuse: ExpansionReuse, + no_cache: bool, + stop: &Arc, +) -> PyResult> { + let Some(plan) = early_lint_plan(inputs, dialect, no_cache)? else { + return Ok(None); + }; + let mut compiled: HashMap = HashMap::new(); + for model in record_field(inputs, "model_inputs")?.try_iter()? { + if let Some((path, expansion)) = model_lint_expansion(&model?)? { + compiled.insert(path, expansion); + } + } + let work: EarlyLintWork = EarlyLintWork { + project_dir: plan.project_dir.clone(), + settings: plan.settings, + reuse, + compiled, + stop: Arc::clone(stop), + }; + let thread: JoinHandle> = std::thread::Builder::new() + .name(LINT_THREAD.to_owned()) + .spawn(move || Python::attach(|py| early_lint(py, &work))) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + Ok(Some(PreparedSqlLint { + codes: plan.codes, + project_dir: plan.project_dir, + thread: Mutex::new(Some(thread)), + result: OnceLock::new(), + stop: Arc::clone(stop), + })) +} + +/// What one early SQL lint thread lints: the project, its settings and the compile's expansions. +struct EarlyLintWork { + project_dir: PathBuf, + settings: LintSettings, + reuse: ExpansionReuse, + compiled: HashMap, + stop: Arc, +} + +/// Expand and lint the project's SQL early, stopping between files once `stop` is set. +fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult { + let project_dir: &Path = &work.project_dir; + let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { + project_dir: project_dir.into_pyobject(py)?.into_any(), + value_renderer: Some(work.reuse.value_renderer(py)), + vars: work.reuse.expansion_vars(py)?, + discovered_inputs: Some(work.reuse.discovered_inputs(py)?), + declaration_scope: None, + static_declaration_scope: present(work.reuse.declaration_scope(py)?), + })?; + let files: Vec = lint_sources(project_dir, None, None)?; + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir, + settings: &work.settings, + dynamic_output_paths: &HashSet::new(), + compiled_expansions: &work.compiled, + context: Some(&context), + skip_unexpandable: false, + stop: Some(&work.stop), + }, + files, + )?; + Ok(EarlyLintResult { outcome, context }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_expansion.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_expansion.rs index 6c625c4144..7b8b8de8ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_expansion.rs @@ -1,16 +1,15 @@ //! The SQL expansion context compiler-integrated lint expands authored bodies with. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ - ExpansionSources, ExpansionVars, present, sql_expansion_inputs, -}; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::compile_attachment::main::present::present; +use crate::compile_attachment::main::sql_expansion_inputs::sql_expansion_inputs; +use crate::compile_attachment::models::ExpansionSources; +use crate::compile_attachment::models::ExpansionVars; +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -137,7 +136,3 @@ fn compile_first_error(py: Python<'_>, error: PyErr) -> PyResult { raised.set_cause(py, Some(error)); Ok(raised) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_lint_expansion_context, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_headers.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_headers.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_inputs.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_inputs.rs index 314baf9875..9b3559e122 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_inputs.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledExpansion, CompiledSource, span_pass, -}; +use crate::rules::_helpers::lint::lint_pipeline::CompiledExpansion; +use crate::rules::_helpers::lint::lint_pipeline::CompiledSource; +use crate::rules::_helpers::lint::lint_pipeline::span_pass; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::types::SpanRow; @@ -91,7 +91,7 @@ pub(crate) fn project_sql_expansions( } return Ok(expansions); } - crate::bindings::_helpers::rules::run_lint::compiled_expansion_records(&native_field( + crate::rules::_helpers::lint::run_lint::compiled_expansion_records(&native_field( project, "sql_expansions", )?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs index 2f9ddf87c6..bde1fee2f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs @@ -1,5 +1,10 @@ //! The lint pipeline over native data: header rules, expanded bodies, native SQL lint and suppressions. +use crate::rules::main::checking::header_violations::header_violations; +use crate::rules::main::lint::project_relative_path::project_relative_path; +use crate::rules::models::LintFile; +use crate::rules::models::LintOutcome; +use crate::rules::models::LintViolationRow; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -7,7 +12,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; use sqlbuild_analysis::sql_macros::main::sql_macro_body_form::sql_macro_body_form; use sqlbuild_analysis::sql_macros::models::{SqlMacroForm, SqlMacroParameter}; @@ -19,10 +23,7 @@ use sqlbuild_rules::dsl_headers::main::declares_table_returns::declares_table_re use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges; -use sqlbuild_rules::dsl_headers::models::{ - HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, -}; -use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; use sqlbuild_rules::lint_text::main::defined_cte_names::defined_cte_names; use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; @@ -44,18 +45,14 @@ use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::project_tree::main::project_real_paths::project_real_paths; use sqlclean::sql_lint::main::fill_finding_path::fill_finding_path; -use crate::bindings::_helpers::rules::dsl_headers::header_violations; -use crate::bindings::_helpers::rules::native_header_parser::{ - FUNCTION_KIND, HeaderFailure, NativeHeaderParser, external_identifiers, -}; -use sqlbuild_python_base::boundary::main::call_with::call_with; +use crate::rules::_helpers::checking::native_header_parser::FUNCTION_KIND; +use crate::rules::_helpers::checking::native_header_parser::HeaderFailure; +use crate::rules::_helpers::checking::native_header_parser::external_identifiers; +use crate::rules::models::NativeHeaderParser; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::render::main::declarations::expand_with_inputs::expand_with_inputs; use sqlbuild_python_front::render::models::AuthoredExpansion; @@ -63,7 +60,7 @@ use sqlbuild_python_front::render::models::SqlExpansionInputs; use sqlbuild_python_front::render::types::SpanRow; pub(crate) const NATIVE_ENGINE: &str = "native"; -const HEADER_ENGINE: &str = "sqlbuild"; +pub(crate) const HEADER_ENGINE: &str = "sqlbuild"; const HOOKS_FOLDER: &str = "hooks"; const AUDIT_KIND: &str = "audit"; const SINGULAR_AUDIT_KIND: &str = "singular_audit"; @@ -72,8 +69,8 @@ const CEREMONIAL_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; const SQL_MACRO_KIND: &str = "MACRO"; const SQL_TEST_KIND: &str = "TEST"; const INTERPOLATION_MARKERS: [&str; 2] = ["@", "${"]; -const FAULT_SEVERITY: &str = "FAULT"; -const WARNING_SEVERITY: &str = "WARNING"; +pub(crate) const FAULT_SEVERITY: &str = "FAULT"; +pub(crate) const WARNING_SEVERITY: &str = "WARNING"; /// Resolved lint settings; the defaults are `LintConfig`'s. #[derive(Clone, Debug)] @@ -145,13 +142,6 @@ impl LintSettings { } } -/// One authored SQL file and its text. -#[derive(Clone, Debug)] -pub(crate) struct LintFile { - pub(crate) path: PathBuf, - pub(crate) contents: String, -} - /// A compile's authored-to-expanded SQL for one file, reused instead of expanding again. #[derive(Clone, Debug)] pub(crate) struct CompiledExpansion { @@ -233,218 +223,10 @@ pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { .collect() } -fn offset(value: usize) -> i64 { +pub(crate) fn offset(value: usize) -> i64 { i64::try_from(value).unwrap_or(i64::MAX) } -/// A proposed authored-source replacement in the violation's file. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct LintEditRow { - pub(crate) start: i64, - pub(crate) end: i64, - pub(crate) replacement: String, -} - -/// One lint violation against the file at index `file` of its run. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct LintViolationRow { - pub(crate) file: usize, - pub(crate) line: i64, - pub(crate) column: i64, - pub(crate) code: String, - pub(crate) message: String, - pub(crate) fault: bool, - pub(crate) engine: &'static str, - pub(crate) end: Option<(i64, i64)>, - pub(crate) remediation: Option, - pub(crate) fix: Option, - pub(crate) fix_unavailable_reason: Option, -} - -impl LintViolationRow { - pub(crate) fn header(file: usize, violation: HeaderViolation) -> Self { - Self { - file, - line: offset(violation.line), - column: offset(violation.column), - code: violation.code.to_owned(), - message: violation.message, - fault: violation.severity == HeaderSeverity::Fault, - engine: HEADER_ENGINE, - end: None, - remediation: Some(violation.remediation), - fix: None, - fix_unavailable_reason: None, - } - } - - pub(crate) fn diagnostic(diagnostic: AuthoredDiagnostic) -> Self { - Self { - file: diagnostic.file, - line: diagnostic.line, - column: diagnostic.column, - code: diagnostic.code, - message: diagnostic.message, - fault: diagnostic.fault, - engine: NATIVE_ENGINE, - end: diagnostic.end, - remediation: Some(diagnostic.remediation), - fix: diagnostic.fix.map(|edit| LintEditRow { - start: edit.start, - end: edit.end, - replacement: edit.replacement, - }), - fix_unavailable_reason: diagnostic.fix_unavailable_reason, - } - } - - pub(crate) fn suppression(finding: SuppressionFinding) -> Self { - Self { - file: finding.file, - line: offset(finding.line), - column: 1, - code: SUPPRESSION_CODE.to_owned(), - message: finding.message, - fault: finding.fault, - engine: NATIVE_ENGINE, - end: None, - remediation: Some(finding.remediation.to_owned()), - fix: finding.fix.map(|(start, end)| LintEditRow { - start: offset(start), - end: offset(end), - replacement: String::new(), - }), - fix_unavailable_reason: finding.fix_unavailable_reason.map(str::to_owned), - } - } -} - -/// The lint model classes violations are projected as. -pub(crate) struct ViolationClasses<'py> { - violation: Bound<'py, PyAny>, - edit: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> ViolationClasses<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: record_class(py, RecordId::LintViolation)?, - edit: record_class(py, RecordId::LintEdit)?, - severity: record_class(py, RecordId::LintSeverity)?, - }) - } - - /// `row` as a `LintViolation` against `file_path`. - pub(crate) fn violation( - &self, - file_path: &Bound<'py, PyAny>, - row: &LintViolationRow, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let none = || py.None().into_bound(py); - let text = |value: &Option| { - value - .as_deref() - .map_or_else(none, |text| PyString::new(py, text).into_any()) - }; - let code: Bound<'py, PyAny> = PyString::new(py, &row.code).into_any(); - let fix: Bound<'py, PyAny> = match &row.fix { - Some(edit) => call_with( - &self.edit, - &[ - ("file_path", file_path.clone()), - ("code", code.clone()), - ("start", edit.start.into_pyobject(py)?.into_any()), - ("end", edit.end.into_pyobject(py)?.into_any()), - ( - "replacement", - PyString::new(py, &edit.replacement).into_any(), - ), - ], - )?, - None => none(), - }; - let (end_line, end_column) = match row.end { - Some((line, column)) => ( - line.into_pyobject(py)?.into_any(), - column.into_pyobject(py)?.into_any(), - ), - None => (none(), none()), - }; - let severity: &str = if row.fault { - FAULT_SEVERITY - } else { - WARNING_SEVERITY - }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", row.line.into_pyobject(py)?.into_any()), - ("column", row.column.into_pyobject(py)?.into_any()), - ("code", code), - ("message", PyString::new(py, &row.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, row.engine).into_any()), - ("end_line", end_line), - ("end_column", end_column), - ("remediation", text(&row.remediation)), - ("fix", fix), - ("fix_unavailable_reason", text(&row.fix_unavailable_reason)), - ], - ) - } -} - -/// Every linted file, its sorted violations and the files expansion skipped. -pub(crate) struct LintOutcome { - /// The linted files in the order they were given. - pub(crate) files: Vec, - pub(crate) violations: Vec, - /// Skipped files by index, with why they could not be expanded. - pub(crate) unexpandable: Vec<(usize, String)>, -} - -impl LintOutcome { - /// This outcome as a `LintRunResult`. - pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { - let paths: Vec> = self - .files - .iter() - .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) - .collect::>()?; - let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; - let violations: Vec> = self - .violations - .iter() - .map(|row| classes.violation(&paths[row.file], row)) - .collect::>()?; - let texts: Bound<'py, PyDict> = PyDict::new(py); - for (path, file) in paths.iter().zip(&self.files) { - texts.set_item(path, &file.contents)?; - } - let unexpandable: Bound<'py, PyDict> = PyDict::new(py); - for (file, message) in &self.unexpandable { - unexpandable.set_item(&paths[*file], message)?; - } - new_record_with( - py, - RecordId::LintRunResult, - &[ - ( - "files_checked", - self.files.len().into_pyobject(py)?.into_any(), - ), - ("violations", PyTuple::new(py, violations)?.into_any()), - ("formatted_files", PyTuple::empty(py).into_any()), - ("source_texts", texts.into_any()), - ("unexpandable", unexpandable.into_any()), - ], - ) - } -} - /// What one lint run reads besides its files. pub(crate) struct LintRequest<'a> { pub(crate) project_dir: &'a Path, @@ -1186,16 +968,6 @@ fn lint_body( Ok((body, dependencies.relations)) } -/// `path` relative to `project_dir` with `/` separators, or as given outside the project. -pub(crate) fn project_relative_path(path: &Path, project_dir: &Path) -> String { - let relative: &Path = path.strip_prefix(project_dir).unwrap_or(path); - relative - .components() - .map(|part| part.as_os_str().to_string_lossy().into_owned()) - .collect::>() - .join("/") -} - /// The relation name each dependency sentinel stands for, without its `kind:` prefix. pub(crate) fn relation_names(relations: &[(String, String)]) -> Vec<(String, String)> { relations diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_preparation.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_preparation.rs index c22c853333..f97a4de2ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_preparation.rs @@ -5,9 +5,10 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyTuple}; use sqlbuild_rules::lint_text::models::{ExpansionSpan, LintSqlBody}; -use crate::bindings::_helpers::rules::lint_pipeline::{ - BodyRole, SingleBody, SingleBodyRequest, prepare_single_body, -}; +use crate::rules::_helpers::lint::lint_pipeline::BodyRole; +use crate::rules::_helpers::lint::lint_pipeline::SingleBody; +use crate::rules::_helpers::lint::lint_pipeline::SingleBodyRequest; +use crate::rules::_helpers::lint::lint_pipeline::prepare_single_body; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies.rs index 4a2f3876b4..349307f19c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies.rs @@ -1,16 +1,16 @@ //! Native SQL lint of prepared lint bodies, as Python lint violations. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; use sqlbuild_rules::lint_text::models::{ AuthoredDiagnostic, ExpansionSpan, LintSqlBody, LintSqlOptions, }; -use crate::bindings::_helpers::rules::lint_pipeline::{ - LintViolationRow, ViolationClasses, relation_names as sentinel_relation_names, -}; +use crate::rules::_helpers::lint::lint_pipeline::relation_names as sentinel_relation_names; +use crate::rules::models::LintViolationRow; +use crate::rules::models::ViolationClasses; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -97,7 +97,7 @@ fn options(config: &Bound<'_, PyAny>) -> PyResult { /// Lint expanded bodies natively and map every diagnostic onto authored source. #[pyfunction] -fn lint_native_sql_bodies<'py>( +pub(crate) fn lint_native_sql_bodies<'py>( py: Python<'py>, bodies: Vec>, contents_by_path: Bound<'py, PyDict>, @@ -132,7 +132,3 @@ fn lint_native_sql_bodies<'py>( }) .collect() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lint_native_sql_bodies, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_text.rs similarity index 60% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_text.rs index 145f1d66f3..be784235e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_text.rs @@ -1,10 +1,13 @@ //! Interpolation sentinels, inline directive findings and fixture-null candidates over lint models. +use crate::rules::main::lint::project_relative_path::project_relative_path; +use crate::rules::main::lint::relative_parts::relative_parts; +use crate::rules::main::lint::unsafe_fault::unsafe_fault; use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_rules::lint_text::errors::LintTextError; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind as role_kind; @@ -12,29 +15,26 @@ use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interp use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolation; use sqlbuild_rules::lint_text::models::{ - FormatFault, FormatFile, FormattedBodies, InterpolationSite, Neutralized, SuppressionFinding, + FormatFile, FormattedBodies, InterpolationSite, Neutralized, SuppressionFinding, }; -use crate::bindings::_helpers::rules::lint_pipeline::{ - LintViolationRow, NATIVE_ENGINE, ViolationClasses, project_relative_path, -}; +use crate::rules::models::LintViolationRow; +use crate::rules::models::ViolationClasses; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_project::analysis::models::Positional; -const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; -const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; -const FORMAT_UNAVAILABLE_CODE: &str = "format-unavailable"; -const FORMAT_UNAVAILABLE_REMEDIATION: &str = "No change is needed. The file was left as written and is formatted once SQLClean formats this dialect."; -const FORMAT_UNSAFE_PREFIX: &str = +pub(crate) const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; +pub(crate) const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; +pub(crate) const FORMAT_UNAVAILABLE_CODE: &str = "format-unavailable"; +pub(crate) const FORMAT_UNAVAILABLE_REMEDIATION: &str = "No change is needed. The file was left as written and is formatted once SQLClean formats this dialect."; +pub(crate) const FORMAT_UNSAFE_PREFIX: &str = "File left unchanged because its SQL body could not be formatted"; -const FORMAT_UNAVAILABLE_PREFIX: &str = "File left as written"; +pub(crate) const FORMAT_UNAVAILABLE_PREFIX: &str = "File left as written"; fn site_tuples(neutralized: Neutralized) -> (String, Vec>) { (neutralized.text, Positional::wrap_all(neutralized.sites)) @@ -42,7 +42,7 @@ fn site_tuples(neutralized: Neutralized) -> (String, Vec>, ) -> PyResult<(bool, String)> { @@ -71,7 +71,7 @@ fn restore_lint_interpolation( /// The named declaration kind of a project-relative path, as `DeclarationKind`. #[pyfunction] #[pyo3(signature = (*, relative_path))] -fn named_declaration_role_kind<'py>( +pub(crate) fn named_declaration_role_kind<'py>( relative_path: &Bound<'py, PyAny>, ) -> PyResult>> { let parts: Vec = native_field(relative_path, "parts")?.extract()?; @@ -120,60 +120,6 @@ pub(crate) fn reject_lint_suppressions<'py>( Ok(retained) } -/// The parts of `path` below `project_dir`, or `None` outside the project. -pub(crate) fn relative_parts( - path: &std::path::Path, - project_dir: &std::path::Path, -) -> Option> { - match path.strip_prefix(project_dir) { - Ok(relative) => Some( - relative - .components() - .map(|part| part.as_os_str().to_string_lossy().into_owned()) - .collect(), - ), - Err(_) => None, - } -} - -/// A declined body's `format-unsafe` fault, or a `format-unavailable` warning for an unformatted dialect. -pub(crate) fn unsafe_fault<'py>( - file_path: &Bound<'py, PyAny>, - fault: &FormatFault, -) -> PyResult> { - let py: Python<'py> = file_path.py(); - let (severity, code, prefix): (&str, &str, &str) = if fault.unavailable { - ( - "WARNING", - FORMAT_UNAVAILABLE_CODE, - FORMAT_UNAVAILABLE_PREFIX, - ) - } else { - ("FAULT", FORMAT_UNSAFE_CODE, FORMAT_UNSAFE_PREFIX) - }; - let remediation: &str = if fault.unavailable { - FORMAT_UNAVAILABLE_REMEDIATION - } else { - FORMAT_UNSAFE_REMEDIATION - }; - let message: String = format!("{prefix}: {}", fault.reason); - let severity: Bound<'py, PyAny> = enum_member(py, RecordId::LintSeverity, severity)?; - new_record_with( - py, - RecordId::LintViolation, - &[ - ("file_path", file_path.clone()), - ("line", fault.line.into_pyobject(py)?.into_any()), - ("column", 1_usize.into_pyobject(py)?.into_any()), - ("code", PyString::new(py, code).into_any()), - ("message", PyString::new(py, &message).into_any()), - ("severity", severity), - ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), - ("remediation", PyString::new(py, remediation).into_any()), - ], - ) -} - /// Format the SQL bodies of `files`; return changed texts by path and declined-body faults. #[pyfunction] pub(crate) fn format_lint_sql_bodies<'py>( @@ -209,11 +155,3 @@ pub(crate) fn format_lint_sql_bodies<'py>( .collect::>()?; Ok((formatted, faults)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(neutralize_lint_interpolation, module)?)?; - module.add_function(wrap_pyfunction!(restore_lint_interpolation, module)?)?; - module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; - module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; - module.add_function(wrap_pyfunction!(reject_lint_suppressions, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/lint/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/mod.rs new file mode 100644 index 0000000000..63e902f433 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod early_sql_lint; +pub(crate) mod lint_expansion; +pub(crate) mod lint_headers; +pub(crate) mod lint_inputs; +pub(crate) mod lint_pipeline; +pub(crate) mod lint_preparation; +pub(crate) mod lint_sql_bodies; +pub(crate) mod lint_text; +pub(crate) mod run_lint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/run_lint.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/lint/run_lint.rs index 673c166fd2..9500b215ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/lint/run_lint.rs @@ -3,19 +3,22 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; -use crate::bindings::_helpers::rules::lint_expansion::{ - LintExpansionSources, lint_expansion_inputs, -}; -use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledExpansion, CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, - lint_sources, -}; +use crate::compile_attachment::main::present::present; +use crate::compile_attachment::models::ExpansionVars; +use crate::rules::_helpers::lint::lint_expansion::LintExpansionSources; +use crate::rules::_helpers::lint::lint_expansion::lint_expansion_inputs; +use crate::rules::_helpers::lint::lint_pipeline::CompiledExpansion; +use crate::rules::_helpers::lint::lint_pipeline::CompiledSource; +use crate::rules::_helpers::lint::lint_pipeline::LintRequest; +use crate::rules::_helpers::lint::lint_pipeline::LintSettings; +use crate::rules::_helpers::lint::lint_pipeline::lint; +use crate::rules::_helpers::lint::lint_pipeline::lint_sources; +use crate::rules::models::LintFile; +use crate::rules::models::LintOutcome; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_front::render::models::SqlExpansionInputs; @@ -71,7 +74,7 @@ pub(crate) fn compiled_expansion_records( /// Collect supported SQL-analysis inputs of a project. #[pyfunction] #[pyo3(signature = (*, project_dir, selected_paths = None))] -fn collect_lint_project_files<'py>( +pub(crate) fn collect_lint_project_files<'py>( py: Python<'py>, project_dir: PathBuf, selected_paths: Option>, @@ -90,7 +93,7 @@ fn collect_lint_project_files<'py>( /// Lint all DSL files without modifying anything; optionally skip unexpandable SQL. #[pyfunction] #[pyo3(signature = (**options))] -fn run_lint<'py>( +pub(crate) fn run_lint<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -160,9 +163,3 @@ fn run_lint<'py>( )?; outcome.record(py) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_lint, module)?)?; - module.add_function(wrap_pyfunction!(collect_lint_project_files, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/mod.rs new file mode 100644 index 0000000000..78b6fae967 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod checking; +pub(crate) mod configuration; +pub(crate) mod custom; +pub(crate) mod fact; +pub(crate) mod fix; +pub(crate) mod lint; +pub(crate) mod rule; +pub(crate) mod sql; diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/rule/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/mod.rs new file mode 100644 index 0000000000..ae531d6d1a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod rule_api; +pub(crate) mod rule_cache; +pub(crate) mod rule_dispatch; +pub(crate) mod rule_plan; +pub(crate) mod rule_run; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs index 61478d0317..d0c5f6aefe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs @@ -4,13 +4,11 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_e use std::path::{Path, PathBuf}; use pyo3::exceptions::PyAttributeError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, }; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_config::toml::main::toml_literal::toml_literal; use sqlbuild_config::toml::models::TomlLiteral; use sqlbuild_rules::project_tree::main::project_path_escapes::project_path_escapes; @@ -74,7 +72,7 @@ fn position<'py>( /// The `(line, column)` of a finding at the project-relative `path`, from `node` when unset. #[pyfunction] #[pyo3(signature = (*, path, line, column, node))] -fn finding_position<'py>( +pub(crate) fn finding_position<'py>( path: &Bound<'py, PyAny>, line: &Bound<'py, PyAny>, column: &Bound<'py, PyAny>, @@ -128,7 +126,7 @@ fn option_literal(value: &Bound<'_, PyAny>) -> PyResult { /// The TOML text the rule harness writes for one `RuleCase` option value. #[pyfunction] -fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { Ok(toml_literal(&option_literal(value)?)) } @@ -150,7 +148,7 @@ fn escapes(path: &Bound<'_, PyAny>) -> PyResult { /// Every supported path under the resolved project `root`, as sorted `ProjectPath`s. #[pyfunction] -fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = root.py(); let root: PathBuf = root.extract()?; let mut paths: Vec<(Bound<'py, PyAny>, bool)> = project_tree_entries(&root) @@ -184,11 +182,3 @@ fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult>()?; PyTuple::new(py, nodes) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(finding_position, module)?)?; - module.add_function(wrap_pyfunction!(rule_option_toml_value, module)?)?; - module.add_function(wrap_pyfunction!(validate_project_relative, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_paths, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs index 6a4a5b3cd8..4e4bfae325 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs @@ -5,11 +5,9 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use serde_json::{Value, json}; use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; @@ -24,7 +22,7 @@ use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, use sqlbuild_rules::rule_cache::types::CachedEntries; use sqlbuild_rules::rule_results::models::RuleFinding; -use crate::bindings::_helpers::rules::lint_pipeline::LintSettings; +use crate::rules::_helpers::lint::lint_pipeline::LintSettings; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Stored entries as `{path: (key, findings JSON)}`. @@ -81,7 +79,7 @@ pub(crate) fn model_rule_keys( /// One non-model SQL file's cache key, or `None` when its findings depend on expansion. #[pyfunction] #[pyo3(signature = (**request))] -fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult> { +pub(crate) fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult> { let request = Request::new("sql_file_rule_key", request)?; let lint_key: String = request.get("lint_key")?.extract()?; let codes: Vec = request.get("codes")?.extract()?; @@ -99,7 +97,7 @@ fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult PyResult { +pub(crate) fn load_sql_rule_cache(path: PathBuf) -> PyResult { stored_entries(&path) } @@ -213,7 +211,7 @@ fn finding_payload(finding: &Bound<'_, PyAny>) -> PyResult { /// The stored JSON of one path's findings. #[pyfunction] #[pyo3(signature = (*, findings))] -fn encode_sql_rule_findings(findings: Vec>) -> PyResult { +pub(crate) fn encode_sql_rule_findings(findings: Vec>) -> PyResult { let payloads: Vec = findings .iter() .map(finding_payload) @@ -232,10 +230,3 @@ pub(crate) fn write_rule_cache(path: &Path, stored: &StoredEntries) -> PyResult< std::fs::rename(&temporary, path)?; Ok(()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_file_rule_key, module)?)?; - module.add_function(wrap_pyfunction!(load_sql_rule_cache, module)?)?; - module.add_function(wrap_pyfunction!(encode_sql_rule_findings, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs index c664257c77..55a89143bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs @@ -8,40 +8,41 @@ use std::thread::JoinHandle; use std::time::Duration; use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::rules::catalogue::{ - resolved_ignore_selectors, rule_catalogue, select_rules, -}; -use crate::bindings::_helpers::rules::constants::RULES_MEMO_REUSES; -use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; -use crate::bindings::_helpers::rules::custom_evaluation::{ - CustomRun, evaluate_custom_rules_cached, outcome, -}; -use crate::bindings::_helpers::rules::custom_hosts::HostCancel; -use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; -use crate::bindings::_helpers::rules::early_builtin_rules::{ - EarlyBuiltinRules, EarlyRulesReport, take_early_rules, -}; -use crate::bindings::_helpers::rules::fact_replay::project_models; -use crate::bindings::_helpers::rules::results::rule_finding; -use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; -use crate::bindings::_helpers::rules::rows::{ - NativeRulesRequest, build_rules_request, evaluate_rules_request, evaluate_rules_request_with, - rules_request_over, selectionless_rules_request, -}; -use crate::bindings::_helpers::rules::rule_plan::{ - plan_rule_run, selects_row_rules, whole_project_codes, -}; -use crate::bindings::_helpers::rules::rule_run::finish_rule_run; -use crate::bindings::_helpers::rules::selection::{rule_selected, selected_rule_codes}; -use crate::bindings::_helpers::rules::sql_rules::run_planned_sql_rules; +use crate::rules::_helpers::checking::early_builtin_rules::EarlyBuiltinRules; +use crate::rules::_helpers::checking::early_builtin_rules::EarlyRulesReport; +use crate::rules::_helpers::checking::early_builtin_rules::take_early_rules; +use crate::rules::_helpers::checking::results::rule_finding; +use crate::rules::_helpers::checking::rows::NativeRulesRequest; +use crate::rules::_helpers::checking::rows::build_rules_request; +use crate::rules::_helpers::checking::rows::evaluate_rules_request; +use crate::rules::_helpers::checking::rows::evaluate_rules_request_with; +use crate::rules::_helpers::checking::rows::finalize_rule_findings_rows; +use crate::rules::_helpers::checking::rows::rules_request_over; +use crate::rules::_helpers::checking::rows::selectionless_rules_request; +use crate::rules::_helpers::configuration::catalogue::resolved_ignore_selectors; +use crate::rules::_helpers::configuration::catalogue::rule_catalogue; +use crate::rules::_helpers::configuration::catalogue::select_rules; +use crate::rules::_helpers::configuration::selection::rule_selected; +use crate::rules::_helpers::configuration::selection::selected_rule_codes; +use crate::rules::_helpers::custom::custom_evaluation::CustomRun; +use crate::rules::_helpers::custom::custom_evaluation::evaluate_custom_rules_cached; +use crate::rules::_helpers::custom::custom_evaluation::outcome; +use crate::rules::_helpers::custom::custom_hosts::HostCancel; +use crate::rules::_helpers::custom::custom_rule_evidence::verify_custom_rules; +use crate::rules::_helpers::fact::fact_replay::project_models; +use crate::rules::_helpers::rule::rule_plan::plan_rule_run; +use crate::rules::_helpers::rule::rule_plan::selects_row_rules; +use crate::rules::_helpers::rule::rule_plan::whole_project_codes; +use crate::rules::_helpers::rule::rule_run::finish_rule_run; +use crate::rules::_helpers::sql::sql_rules::run_planned_sql_rules; +use crate::rules::constants::RULES_MEMO_REUSES; +use crate::rules::constants::TYPE_PROOF_RULE_CODES; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -139,7 +140,7 @@ enum CustomSource { /// Evaluate the selected compiler rules over compiled models. #[pyfunction] #[pyo3(signature = (**options))] -fn evaluate_project_rules<'py>( +pub(crate) fn evaluate_project_rules<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -178,7 +179,7 @@ fn evaluate_project_rules<'py>( /// Evaluate built-in rules natively with the catalogue's selected custom rules beside them. #[pyfunction] #[pyo3(signature = (**options))] -fn evaluate_native_rules<'py>( +pub(crate) fn evaluate_native_rules<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -207,7 +208,7 @@ fn evaluate_native_rules<'py>( /// Evaluate custom rules through their incremental cache. #[pyfunction] #[pyo3(signature = (**options))] -fn evaluate_custom_rules<'py>( +pub(crate) fn evaluate_custom_rules<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -745,10 +746,3 @@ fn cancel_on_interrupt(py: Python<'_>, result: &PyResult>, canc cancel.store(true, Ordering::Release); } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_project_rules, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_native_rules, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_custom_rules, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_rules, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_plan.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_plan.rs index 7077efccbb..0a51a764dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_plan.rs @@ -1,19 +1,22 @@ //! Plan one rules run: the catalogue, selected rules, project slice and settings it evaluates. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyFrozenSet, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use std::path::Path; -use crate::bindings::_helpers::rules::catalogue::{ - references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, - verify_custom_rules, -}; -use crate::bindings::_helpers::rules::constants::{SQL_RULE_PREFIX, TYPE_PROOF_RULE_CODES}; -use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; -use crate::bindings::_helpers::rules::rule_run::freed_memory_released; +use crate::rules::_helpers::configuration::catalogue::references_custom_rules; +use crate::rules::_helpers::configuration::catalogue::resolved_ignore_selectors; +use crate::rules::_helpers::configuration::catalogue::rule_catalogue; +use crate::rules::_helpers::configuration::catalogue::select_rules; +use crate::rules::_helpers::configuration::catalogue::verify_custom_rules; +use crate::rules::_helpers::fact::fact_replay::project_attribute; +use crate::rules::_helpers::fact::fact_replay::project_models; +use crate::rules::_helpers::rule::rule_run::freed_memory_released; +use crate::rules::constants::SQL_RULE_PREFIX; +use crate::rules::constants::TYPE_PROOF_RULE_CODES; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; @@ -273,8 +276,3 @@ fn selection_rules_config<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_rule_run, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_run.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_run.rs index 390b37eb47..eabf1f0eb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_run.rs @@ -4,17 +4,17 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path as bulk_cache_path; use sqlbuild_rules::rule_results::main::group_unevaluated::group_unevaluated; use sqlbuild_rules::rule_results::models::RuleFinding; -use crate::bindings::_helpers::rules::fact_replay::project_models; -use crate::bindings::_helpers::rules::results::rule_finding; -use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; -use crate::bindings::_helpers::rules::rule_plan::whole_project_codes; +use crate::rules::_helpers::checking::results::rule_finding; +use crate::rules::_helpers::checking::rows::finalize_rule_findings_rows; +use crate::rules::_helpers::fact::fact_replay::project_models; +use crate::rules::_helpers::rule::rule_plan::whole_project_codes; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -142,16 +142,6 @@ pub(crate) fn finish_rule_run<'py>( ) } -/// Run the selected built-in, SQL and custom rules; custom rules run on their worker threads. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn run_rules<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules(py, options) -} - /// One bulk Rules cache file owned by the native compiler engine. #[pyfunction] #[pyo3(signature = (*, project_dir, file_name))] @@ -186,10 +176,3 @@ pub(crate) fn freed_memory_released(py: Python<'_>) -> PyResult<()> { _ => Ok(()), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(finish_rule_run, module)?)?; - module.add_function(wrap_pyfunction!(run_rules, module)?)?; - module.add_function(wrap_pyfunction!(rules_bulk_cache_path, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/fixture_schemas.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/sql/fixture_schemas.rs diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/mod.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/mod.rs new file mode 100644 index 0000000000..3993b6425c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod fixture_schemas; +pub(crate) mod sql_document; +pub(crate) mod sql_rules; +pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/sql_document.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index f25e82792e..2d6cef8b39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyclass, pymethods}; @@ -206,9 +206,3 @@ fn tree_node(py: Python<'_>, tree: &Arc, index: usize) -> PyResult, error: &SqlTreeError) -> PyErr { new_annotated_error(py, ErrorId::RuleUsage, error.to_string(), &[]) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_rules.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_rules.rs index 6f6cd2a808..cbe64b8fb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_rules.rs @@ -6,38 +6,46 @@ use std::path::{Path, PathBuf}; use std::time::Instant; use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, - Python, -}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyRef, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::text::main::python_splitlines::python_splitlines; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; use sqlbuild_rules::rule_results::models::RuleFinding; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; -use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; -use crate::bindings::_helpers::rules::early_sql_lint::{ - EarlyLintResult, EarlySqlLint, ExpansionReuse, PreparedSqlLint, sql_lint_settings, - sql_rule_cache_path, -}; -use crate::bindings::_helpers::rules::fact_replay::project_attribute; -use crate::bindings::_helpers::rules::lint_expansion::{ - LintExpansionSources, lint_expansion_inputs, -}; -use crate::bindings::_helpers::rules::lint_inputs::project_sql_expansions; -use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, LintViolationRow, lint, - lint_sources, outside_project, resolved, -}; -use crate::bindings::_helpers::rules::rule_cache::{ - ModelRuleRow, StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, - settings_lint_key, stored_entries, write_rule_cache, -}; +use crate::compile_attachment::models::ExpansionVars; +use crate::rules::_helpers::fact::fact_replay::project_attribute; +use crate::rules::_helpers::lint::early_sql_lint::sql_lint_settings; +use crate::rules::_helpers::lint::early_sql_lint::sql_rule_cache_path; +use crate::rules::_helpers::lint::lint_expansion::LintExpansionSources; +use crate::rules::_helpers::lint::lint_expansion::lint_expansion_inputs; +use crate::rules::_helpers::lint::lint_inputs::project_sql_expansions; +use crate::rules::_helpers::lint::lint_pipeline::CompiledSource; +use crate::rules::_helpers::lint::lint_pipeline::LintRequest; +use crate::rules::_helpers::lint::lint_pipeline::LintSettings; +use crate::rules::_helpers::lint::lint_pipeline::lint; +use crate::rules::_helpers::lint::lint_pipeline::lint_sources; +use crate::rules::_helpers::lint::lint_pipeline::outside_project; +use crate::rules::_helpers::lint::lint_pipeline::resolved; +use crate::rules::_helpers::rule::rule_cache::ModelRuleRow; +use crate::rules::_helpers::rule::rule_cache::StoredEntries; +use crate::rules::_helpers::rule::rule_cache::cached_rule_findings; +use crate::rules::_helpers::rule::rule_cache::encoded_findings; +use crate::rules::_helpers::rule::rule_cache::model_rule_keys; +use crate::rules::_helpers::rule::rule_cache::settings_lint_key; +use crate::rules::_helpers::rule::rule_cache::stored_entries; +use crate::rules::_helpers::rule::rule_cache::write_rule_cache; +use crate::rules::constants::SAFE_RULE_FIX_CODES; +use crate::rules::constants::SQL_RULE_PREFIX; +use crate::rules::models::EarlyLintResult; +use crate::rules::models::EarlySqlLint; +use crate::rules::models::ExpansionReuse; +use crate::rules::models::LintFile; +use crate::rules::models::LintOutcome; +use crate::rules::models::LintViolationRow; +use crate::rules::models::PreparedSqlLint; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; @@ -345,7 +353,7 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult result.outcome.files.clone(), @@ -466,10 +474,7 @@ fn evaluated_sql_rules( file_keys.push((file.path.clone(), relative_path, key)); } let file_keys: Vec<(PathBuf, String, Option)> = - crate::bindings::_helpers::compile_delta::lint_hooks::file_keys( - project_dir, - file_keys, - )?; + crate::compile_delta::main::lint_hooks::file_keys::file_keys(project_dir, file_keys)?; let mut keyed: Vec<(String, String)> = Vec::new(); for (_, path, key) in &file_keys { if let Some(key) = key { @@ -740,8 +745,3 @@ fn lint_finding( fixable, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_planned_sql_rules, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_test_objects.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs rename to crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_test_objects.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python-compile/src/rules/constants.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs rename to crates/sqlbuild-python-compile/src/rules/constants.rs index 14f0b42a54..daae896d8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python-compile/src/rules/constants.rs @@ -1,6 +1,12 @@ -//! Authored SQL-test mode identifiers. +//! Public constants of rules. + +use crate::rules::_helpers::checking::getters::read_early_sql_lint; +use crate::rules::models::EarlySqlLint; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; pub(crate) const MODEL_TEST_MODE: &str = "model"; + /// The rule codes whose fixes `sqb format --fix` may apply after compiler verification. pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ "SQLCLEAN002", @@ -12,24 +18,39 @@ pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ "SQLCLEAN025", "SQLCLEAN042", ]; + /// The note attached to a rule finding that has an available fix. -pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = +pub const RULE_FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; + pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; + /// File name of the custom-rule result cache under the project cache directory. pub(crate) const CUSTOM_RULES_CACHE_FILE: &str = "custom-rules.json"; + /// Code prefix of the SQL lint rules, which check each selected file on its own. pub(crate) const SQL_RULE_PREFIX: &str = "SQLCLEAN"; + /// Suffix of a custom host scratch directory abandoned by a cancelled run. pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; + /// Marker file whose presence tells split custom hosts the run was cancelled. pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; + /// Rules that need proven column types and are skipped without SQL analysis. pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 2] = ["SQBRCONTRACT105", "SQBRPERFORMANCE102"]; + /// Runtime identity custom-rule hosts and their cache entries record. -pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; +pub const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; /// Native rules evaluations answered from the response memo. pub(crate) const RULES_MEMO_REUSES: &str = "rules_memo_reuses"; + /// The `RulesError` attribute holding how to turn off default Rules, printed only with human error text. -pub(crate) const DEFAULT_RULES_NOTICE_ATTR: &str = "default_rules_notice"; +pub const DEFAULT_RULES_NOTICE_ATTR: &str = "default_rules_notice"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_early_sql_lint, +}]; diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/format_rules_result.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/format_rules_result.rs new file mode 100644 index 0000000000..a07dd4f5d0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/format_rules_result.rs @@ -0,0 +1,25 @@ +//! One rules result for the requested CLI surface. + +use crate::rules::_helpers::checking::results::rule_findings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_rules::rule_results::main::format_result::format_result; +use sqlbuild_rules::rule_results::models::ResultCounts; + +/// One rules result for the requested CLI surface. +#[pyfunction] +#[pyo3(signature = (*, result, json_output))] +pub fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult { + let counts = ResultCounts { + evaluated_models: field(result, "evaluated_models")?.extract()?, + unevaluated_resources: field(result, "unevaluated_resources")?.extract()?, + cache_hits: field(result, "cache_hits")?.extract()?, + cache_misses: field(result, "cache_misses")?.extract()?, + }; + Ok(format_result( + &rule_findings(&field(result, "findings")?)?, + counts, + json_output, + )) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/format_skipped_type_proof_rules.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/format_skipped_type_proof_rules.rs new file mode 100644 index 0000000000..84af1842a1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/format_skipped_type_proof_rules.rs @@ -0,0 +1,11 @@ +//! Describe type-proof rules skipped because SQL analysis is disabled, if any. + +use pyo3::pyfunction; +use sqlbuild_rules::rule_results::main::format_skipped_rules::format_skipped_rules; + +/// Describe type-proof rules skipped because SQL analysis is disabled, if any. +#[pyfunction] +#[pyo3(signature = (*, codes))] +pub fn format_skipped_type_proof_rules(codes: Vec) -> Option { + format_skipped_rules(&codes) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/header_violations.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/header_violations.rs new file mode 100644 index 0000000000..1ef81744f5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/header_violations.rs @@ -0,0 +1,14 @@ +//! The header rule violations of `headers` in `contents`. + +use crate::rules::models::NativeHeaderParser; +use sqlbuild_rules::dsl_headers::main::lint_headers::lint_headers; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; + +/// The header rule violations of `headers` in `contents`. +pub fn header_violations( + contents: &[char], + headers: &[HeaderSpan], + config: HeaderRulesConfig, +) -> Result, String> { + lint_headers(contents, headers, config, &NativeHeaderParser) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/mod.rs new file mode 100644 index 0000000000..bf69771dcd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/mod.rs @@ -0,0 +1,11 @@ +pub mod format_rules_result; +pub mod format_skipped_type_proof_rules; +pub mod header_violations; +pub mod register_dsl_headers; +pub mod register_evaluation; +pub mod register_lint; +pub mod register_project_tree_queries; +pub mod register_results; +pub mod rules_config; +pub mod rules_progress; +pub mod start_early_rules; diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/register_dsl_headers.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/register_dsl_headers.rs new file mode 100644 index 0000000000..40e80040df --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/register_dsl_headers.rs @@ -0,0 +1,26 @@ +//! Register the dsl headers functions and classes. + +use crate::rules::_helpers::checking::dsl_headers::format_native_headers; +use crate::rules::_helpers::checking::dsl_headers::lint_native_headers; +use crate::rules::_helpers::checking::dsl_headers::measurement_body_ranges; +use crate::rules::_helpers::checking::dsl_headers::reject_unparseable_header_rewrites; +use crate::rules::_helpers::checking::dsl_headers::safe_format_files; +use crate::rules::_helpers::checking::dsl_headers::scan_headers; +use crate::rules::_helpers::checking::dsl_headers::sql_body_ranges; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the dsl headers functions and classes. +pub fn register_dsl_headers(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_headers, module)?)?; + module.add_function(wrap_pyfunction!(sql_body_ranges, module)?)?; + module.add_function(wrap_pyfunction!(measurement_body_ranges, module)?)?; + module.add_function(wrap_pyfunction!(lint_native_headers, module)?)?; + module.add_function(wrap_pyfunction!(format_native_headers, module)?)?; + module.add_function(wrap_pyfunction!(safe_format_files, module)?)?; + module.add_function(wrap_pyfunction!( + reject_unparseable_header_rewrites, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/register_evaluation.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/register_evaluation.rs new file mode 100644 index 0000000000..1bec50a7fb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/register_evaluation.rs @@ -0,0 +1,11 @@ +//! Register the evaluation functions and classes. + +use crate::rules::_helpers::checking::evaluation::run_custom_host_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the evaluation functions and classes. +pub fn register_evaluation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; + crate::rules::_helpers::checking::rows::register(module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/register_lint.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/register_lint.rs new file mode 100644 index 0000000000..39ecea98c5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/register_lint.rs @@ -0,0 +1,16 @@ +//! Register the lint functions and classes. + +use crate::rules::_helpers::checking::lint::format_sql_json; +use crate::rules::_helpers::checking::lint::lint_sql_json; +use crate::rules::_helpers::checking::lint::prepare_lint_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_project::compiled_views::main::compiled_models::register_relation_keys::register_relation_keys; + +/// Register the lint functions and classes. +pub fn register_lint(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; + module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; + module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; + register_relation_keys(module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs new file mode 100644 index 0000000000..9bb9bde1ef --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs @@ -0,0 +1,19 @@ +//! Register the project tree queries functions and classes. + +use crate::rules::_helpers::checking::project_tree_queries::project_tree_child_nodes; +use crate::rules::_helpers::checking::project_tree_queries::project_tree_descendant_nodes; +use crate::rules::_helpers::checking::project_tree_queries::project_tree_glob_nodes; +use crate::rules::_helpers::checking::project_tree_queries::project_tree_read_text; +use crate::rules::_helpers::checking::project_tree_queries::project_tree_relative_parts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the project tree queries functions and classes. +pub fn register_project_tree_queries(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(project_tree_child_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_descendant_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_glob_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_relative_parts, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_read_text, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/register_results.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/register_results.rs new file mode 100644 index 0000000000..29bbf29d2c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/register_results.rs @@ -0,0 +1,13 @@ +//! Register the results functions and classes. + +use crate::rules::main::checking::format_rules_result::format_rules_result; +use crate::rules::main::checking::format_skipped_type_proof_rules::format_skipped_type_proof_rules; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the results functions and classes. +pub fn register_results(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_rules_result, module)?)?; + module.add_function(wrap_pyfunction!(format_skipped_type_proof_rules, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/rules_config.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/rules_config.rs new file mode 100644 index 0000000000..fae0a455c8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/rules_config.rs @@ -0,0 +1,13 @@ +//! Rules config. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_rules::dsl_headers::models::HeaderRulesConfig; + +/// Rules config. +pub fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { + Ok(HeaderRulesConfig { + line_width: native_field(config, "line_width")?.extract()?, + max_description_lines: native_field(config, "max_description_lines")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/rules_progress.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/rules_progress.rs new file mode 100644 index 0000000000..a66ebed1e0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/rules_progress.rs @@ -0,0 +1,26 @@ +//! The rules phase completion line for one `RulesRunResult`, optionally noting skipped rules. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_rules::rule_results::main::format_rules_progress::format_rules_progress; + +/// The rules phase completion line for one `RulesRunResult`, optionally noting skipped rules. +pub fn rules_progress( + elapsed_seconds: f64, + result: &Bound<'_, PyAny>, + note_skipped_rules: bool, +) -> PyResult { + let skipped: Option> = if note_skipped_rules { + Some(field(result, "skipped_type_proof_rules")?.extract()?) + } else { + None + }; + Ok(format_rules_progress( + elapsed_seconds, + ( + field(result, "built_in_ms")?.extract()?, + field(result, "custom_ms")?.extract()?, + ), + skipped.as_deref(), + )) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/checking/start_early_rules.rs b/crates/sqlbuild-python-compile/src/rules/main/checking/start_early_rules.rs new file mode 100644 index 0000000000..57b24360c4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/checking/start_early_rules.rs @@ -0,0 +1,44 @@ +//! Start early built-in rules over `project` when armed; only non-`Exception` errors propagate. + +use crate::rules::_helpers::checking::early_builtin_rules::BUILD_PHASE; +use crate::rules::_helpers::checking::early_builtin_rules::EarlyBuiltinRules; +use crate::rules::_helpers::checking::early_builtin_rules::SLOT; +use crate::rules::_helpers::checking::early_builtin_rules::Slot; +use crate::rules::_helpers::checking::early_builtin_rules::early_request; +use crate::rules::models::EarlyRulesSettings; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; + +/// Start early built-in rules over `project` when armed; only non-`Exception` errors propagate. +pub fn start_early_rules(project: &Bound<'_, PyAny>) -> PyResult<()> { + let settings: Option = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + match slot.as_mut() { + Some(current @ Slot::Armed(_)) => match std::mem::replace(current, Slot::Spent) { + Slot::Armed(settings) => Some(settings), + _ => None, + }, + _ => None, + } + }); + let Some(settings) = settings else { + return Ok(()); + }; + let py: Python<'_> = project.py(); + let started: Option = + match record_compile_timing(BUILD_PHASE, || early_request(project, &settings)) { + Ok(Some(rows)) => EarlyBuiltinRules::spawn(rows), + Ok(None) => None, + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }; + if let Some(started) = started { + SLOT.with(|slot| { + if let Some(current) = slot.borrow_mut().as_mut() { + *current = Slot::Started(started); + } + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/build_rule_catalogue.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/build_rule_catalogue.rs new file mode 100644 index 0000000000..8bdcb123fb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/build_rule_catalogue.rs @@ -0,0 +1,20 @@ +//! The validated built-in and custom rule catalogue sorted by code. + +use crate::rules::_helpers::configuration::catalogue::rule_catalogue; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; + +/// The validated built-in and custom rule catalogue sorted by code. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, include_custom = true))] +pub fn build_rule_catalogue<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + include_custom: bool, +) -> PyResult> { + PyTuple::new( + config.py(), + rule_catalogue(config, project_dir, include_custom)?, + ) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/format_rule.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/format_rule.rs new file mode 100644 index 0000000000..1ecb3709ca --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/format_rule.rs @@ -0,0 +1,20 @@ +//! `sqb rules show` text for one rule, with effective thresholds when a config is given. + +use crate::rules::_helpers::configuration::catalogue::native_rules_config; +use crate::rules::_helpers::configuration::guidance::rule_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_rules::models::RulesConfig; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::models::RuleText; + +/// `sqb rules show` text for one rule, with effective thresholds when a config is given. +#[pyfunction] +#[pyo3(signature = (*, rule, config = None))] +pub fn format_rule(rule: &Bound<'_, PyAny>, config: Option>) -> PyResult { + let config: Option> = config.filter(|config| !config.is_none()); + let native: Option = config.as_ref().map(native_rules_config).transpose()?; + let text: RuleText = rule_text(rule, config.as_ref())?; + compiler_guard(|| Ok(format_rule_text(&text, native.as_ref()))) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/install_rules_skills.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/install_rules_skills.rs new file mode 100644 index 0000000000..c4cc3fb43a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/install_rules_skills.rs @@ -0,0 +1,54 @@ +//! Install every owned rules skill target, or with `check` report whether all are fresh. + +use crate::rules::_helpers::configuration::guidance::FILELOCK_MODULE; +use crate::rules::_helpers::configuration::guidance::FRESH_SKILL; +use crate::rules::_helpers::configuration::guidance::SKILL_PATHS; +use crate::rules::_helpers::configuration::guidance::current_content; +use crate::rules::_helpers::configuration::guidance::install_error; +use crate::rules::_helpers::configuration::guidance::lock_path; +use crate::rules::_helpers::configuration::guidance::render_skills; +use crate::rules::_helpers::configuration::skills::skill_state; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_rules::rule_catalogue::main::install_rendered_skills::install_rendered_skills; +use sqlbuild_rules::rule_catalogue::models::SkillTarget; +use std::path::PathBuf; + +/// Install every owned rules skill target, or with `check` report whether all are fresh. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, check))] +pub fn install_rules_skills( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + check: bool, +) -> PyResult { + let py: Python<'_> = config.py(); + let (content, fingerprint): (String, String) = render_skills(config, project_dir)?; + let root: PathBuf = project_dir.extract()?; + let targets = || -> PyResult> { + let mut targets: Vec = Vec::with_capacity(SKILL_PATHS.len()); + for relative in SKILL_PATHS { + let path: PathBuf = root.join(relative); + let state: String = skill_state(current_content(&path)?.as_deref(), &fingerprint); + targets.push(SkillTarget { path, state }); + } + Ok(targets) + }; + if check { + return Ok(targets()?.iter().all(|target| target.state == FRESH_SKILL)); + } + let lock: Bound<'_, PyAny> = + imported(py, FILELOCK_MODULE, "FileLock")?.call1((lock_path(&root)?,))?; + let _ = lock.call_method0("acquire")?; + let installed = (|| -> PyResult<()> { + let targets: Vec = targets()?; + compiler_guard(|| Ok(install_rendered_skills(&content, &targets)))? + .map_err(|error| install_error(py, error)) + })(); + let released = lock.call_method0("release"); + installed?; + let _ = released?; + Ok(true) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config.rs new file mode 100644 index 0000000000..851e40124c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config.rs @@ -0,0 +1,13 @@ +//! The project's validated rules configuration as `RulesConfig`. + +use crate::rules::main::configuration::load_rules_config_with_cache::load_rules_config_with_cache; +use pyo3::prelude::{PyAny, PyResult, Python}; +use pyo3::{Py, pyfunction}; +use std::path::PathBuf; + +/// The project's validated rules configuration as `RulesConfig`. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + load_rules_config_with_cache(py, project_dir, true) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config_with_cache.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config_with_cache.rs new file mode 100644 index 0000000000..10819f8077 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/load_rules_config_with_cache.rs @@ -0,0 +1,22 @@ +//! The project's rules configuration, with the Rules cache off when `cache` is `false`. + +use crate::rules::_helpers::configuration::rules_config::config_object; +use crate::rules::main::configuration::rules_config_of::rules_config_of; +use pyo3::Py; +use pyo3::prelude::{PyAny, PyResult, Python}; +use sqlbuild_rules::models::RulesConfig; +use std::path::PathBuf; + +/// The project's rules configuration, with the Rules cache off when `cache` is `false`. +pub fn load_rules_config_with_cache( + py: Python<'_>, + project_dir: PathBuf, + cache: bool, +) -> PyResult> { + let mut config: RulesConfig = match rules_config_of(&project_dir)? { + Ok(config) => config, + Err(invalid) => return Err(invalid.into_pyerr(py)), + }; + config.cache.enabled &= cache; + Ok(config_object(py, &config)?.unbind()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/mod.rs new file mode 100644 index 0000000000..51b1ecb683 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/mod.rs @@ -0,0 +1,10 @@ +pub mod build_rule_catalogue; +pub mod format_rule; +pub mod install_rules_skills; +pub mod load_rules_config; +pub mod load_rules_config_with_cache; +pub mod register_catalogue; +pub mod register_guidance; +pub mod register_rules_config; +pub mod register_selection; +pub mod rules_config_of; diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/register_catalogue.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_catalogue.rs new file mode 100644 index 0000000000..fd76118a67 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_catalogue.rs @@ -0,0 +1,15 @@ +//! Register the catalogue functions and classes. + +use crate::rules::_helpers::configuration::catalogue::resolve_rule_ignore_selectors; +use crate::rules::_helpers::configuration::catalogue::select_catalogue_rules; +use crate::rules::main::configuration::build_rule_catalogue::build_rule_catalogue; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the catalogue functions and classes. +pub fn register_catalogue(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_rule_catalogue, module)?)?; + module.add_function(wrap_pyfunction!(select_catalogue_rules, module)?)?; + module.add_function(wrap_pyfunction!(resolve_rule_ignore_selectors, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/register_guidance.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_guidance.rs new file mode 100644 index 0000000000..493d95d7fd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_guidance.rs @@ -0,0 +1,13 @@ +//! Register the guidance functions and classes. + +use crate::rules::main::configuration::format_rule::format_rule; +use crate::rules::main::configuration::install_rules_skills::install_rules_skills; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the guidance functions and classes. +pub fn register_guidance(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_rule, module)?)?; + module.add_function(wrap_pyfunction!(install_rules_skills, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/register_rules_config.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_rules_config.rs new file mode 100644 index 0000000000..44ad809e91 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_rules_config.rs @@ -0,0 +1,11 @@ +//! Register the rules config functions and classes. + +use crate::rules::main::configuration::load_rules_config::load_rules_config; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rules config functions and classes. +pub fn register_rules_config(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/register_selection.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_selection.rs new file mode 100644 index 0000000000..fc483b0415 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/register_selection.rs @@ -0,0 +1,12 @@ +//! Register the selection functions and classes. + +use crate::rules::_helpers::configuration::selection::rule_selected; +use crate::rules::_helpers::configuration::selection::selected_rule_codes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the selection functions and classes. +pub fn register_selection(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(selected_rule_codes, module)?)?; + module.add_function(wrap_pyfunction!(rule_selected, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/configuration/rules_config_of.rs b/crates/sqlbuild-python-compile/src/rules/main/configuration/rules_config_of.rs new file mode 100644 index 0000000000..8793f920ce --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/configuration/rules_config_of.rs @@ -0,0 +1,26 @@ +//! The project's validated rules configuration, or why it is invalid. + +use crate::rules::models::InvalidRulesConfig; +use pyo3::prelude::PyResult; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_rules::configuration::errors::RulesConfigLoadError; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; +use sqlbuild_rules::models::RulesConfig; +use std::path::{Path, PathBuf}; + +/// The project's validated rules configuration, or why it is invalid. +pub fn rules_config_of(project_dir: &Path) -> PyResult> { + let project_dir: PathBuf = project_dir + .canonicalize() + .or_else(|_| std::path::absolute(project_dir))?; + match compiler_guard(|| Ok(load_config(&project_dir)))? { + Ok(config) => Ok(Ok(config)), + Err(RulesConfigLoadError::Invalid(message)) => { + Ok(Err(InvalidRulesConfig::Unreadable(message))) + } + Err(RulesConfigLoadError::Policy(error)) => Ok(Err(InvalidRulesConfig::Forbidden { + message: error.message, + help: error.help, + })), + } +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/mod.rs new file mode 100644 index 0000000000..b76974c53d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/mod.rs @@ -0,0 +1,5 @@ +pub mod register_custom_evaluation; +pub mod register_custom_host_observation; +pub mod register_custom_rule_cache; +pub mod register_custom_rule_evidence; +pub mod stored_custom_rule_refusal; diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_evaluation.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_evaluation.rs new file mode 100644 index 0000000000..a043f491e6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_evaluation.rs @@ -0,0 +1,20 @@ +//! Register the custom evaluation functions and classes. + +use crate::rules::_helpers::custom::custom_evaluation::custom_rule_fact_digest; +use crate::rules::_helpers::custom::custom_host_columns::custom_host_declared_columns; +use crate::rules::_helpers::custom::custom_host_payload::CUSTOM_HOST_PAYLOAD_MAGIC; +use crate::rules::_helpers::custom::custom_host_payload::custom_host_payload; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyBytes; +use pyo3::wrap_pyfunction; + +/// Register the custom evaluation functions and classes. +pub fn register_custom_evaluation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(custom_host_payload, module)?)?; + module.add_function(wrap_pyfunction!(custom_host_declared_columns, module)?)?; + module.add( + "CUSTOM_HOST_PAYLOAD_MAGIC", + PyBytes::new(module.py(), CUSTOM_HOST_PAYLOAD_MAGIC), + )?; + module.add_function(wrap_pyfunction!(custom_rule_fact_digest, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_host_observation.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_host_observation.rs new file mode 100644 index 0000000000..9c49695828 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_host_observation.rs @@ -0,0 +1,11 @@ +//! Register the custom host observation functions and classes. + +use crate::rules::_helpers::custom::custom_host_observation::CustomHostObserver; +use crate::rules::_helpers::custom::custom_host_observation::OBSERVER_ATTRIBUTE; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the custom host observation functions and classes. +pub fn register_custom_host_observation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add(OBSERVER_ATTRIBUTE, module.py().None()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_cache.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_cache.rs new file mode 100644 index 0000000000..46199ff626 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_cache.rs @@ -0,0 +1,11 @@ +//! Register the custom rule cache functions and classes. + +use crate::rules::_helpers::custom::custom_rule_cache::custom_rule_identity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the custom rule cache functions and classes. +pub fn register_custom_rule_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(custom_rule_identity, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs new file mode 100644 index 0000000000..20c834f09c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs @@ -0,0 +1,25 @@ +//! Register the custom rule evidence functions and classes. + +use crate::rules::_helpers::custom::custom_rule_evidence::allowed_custom_rule_import_binding; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_implementation_fingerprint; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_import_closure; +use crate::rules::_helpers::custom::custom_rule_evidence::custom_rule_test_evidence; +use crate::rules::_helpers::custom::custom_rule_evidence::verify_custom_rules; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the custom rule evidence functions and classes. +pub fn register_custom_rule_evidence(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + allowed_custom_rule_import_binding, + module + )?)?; + module.add_function(wrap_pyfunction!(verify_custom_rules, module)?)?; + module.add_function(wrap_pyfunction!(custom_rule_import_closure, module)?)?; + module.add_function(wrap_pyfunction!( + custom_rule_implementation_fingerprint, + module + )?)?; + module.add_function(wrap_pyfunction!(custom_rule_test_evidence, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/custom/stored_custom_rule_refusal.rs b/crates/sqlbuild-python-compile/src/rules/main/custom/stored_custom_rule_refusal.rs new file mode 100644 index 0000000000..f360027a62 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/custom/stored_custom_rule_refusal.rs @@ -0,0 +1,50 @@ +//! Why a delta of `model_paths` would stop at custom rules, judged from their stored read sets. + +use crate::rules::_helpers::custom::custom_evaluation::OUTSIDE_DELTA; +use crate::rules::_helpers::custom::custom_evaluation::PROJECT_SUBJECT_KEY; +use crate::rules::_helpers::custom::custom_evaluation::PROJECT_WIDE; +use crate::rules::_helpers::custom::custom_evaluation::READ_MORE; +use crate::rules::_helpers::custom::custom_rule_cache::stored_rules; +use crate::rules::_helpers::fact::fact_replay::delta_safe_fact; +use crate::rules::_helpers::fact::fact_replay::read_text_path; +use crate::rules::_helpers::rule::rule_run::rules_bulk_cache_path; +use crate::rules::constants::CUSTOM_RULES_CACHE_FILE; +use std::collections::BTreeSet; +use std::path::Path; + +/// Why a delta of `model_paths` would stop at custom rules, judged from their stored read sets. +pub fn stored_custom_rule_refusal( + project_dir: &Path, + (model_paths, changed): (&BTreeSet, &BTreeSet), + all_sources: bool, +) -> Option<&'static str> { + let stored = stored_rules(&rules_bulk_cache_path( + project_dir.to_path_buf(), + CUSTOM_RULES_CACHE_FILE, + )); + for rule in stored.values() { + for (subject, entry) in &rule.entries { + let Some(readset) = rule.readsets.get(entry.readset) else { + continue; + }; + if model_paths.contains(subject) { + if !readset + .iter() + .all(|key| delta_safe_fact(key, model_paths, all_sources)) + { + return Some(READ_MORE); + } + } else if readset + .iter() + .any(|key| read_text_path(key).is_some_and(|path| changed.contains(path))) + { + return Some(if subject == PROJECT_SUBJECT_KEY { + PROJECT_WIDE + } else { + OUTSIDE_DELTA + }); + } + } + } + None +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/fact/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/fact/mod.rs new file mode 100644 index 0000000000..416a8942cb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fact/mod.rs @@ -0,0 +1 @@ +pub mod register_fact_encoder; diff --git a/crates/sqlbuild-python-compile/src/rules/main/fact/register_fact_encoder.rs b/crates/sqlbuild-python-compile/src/rules/main/fact/register_fact_encoder.rs new file mode 100644 index 0000000000..5b61a0c93c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fact/register_fact_encoder.rs @@ -0,0 +1,12 @@ +//! Register the fact encoder functions and classes. + +use crate::rules::_helpers::fact::fact_encoder::fact_error_digest; +use crate::rules::_helpers::fact::fact_encoder::fact_value_digest; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fact encoder functions and classes. +pub fn register_fact_encoder(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fact_value_digest, module)?)?; + module.add_function(wrap_pyfunction!(fact_error_digest, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/fix/compile_facts_function.rs b/crates/sqlbuild-python-compile/src/rules/main/fix/compile_facts_function.rs new file mode 100644 index 0000000000..f1d8907fb4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fix/compile_facts_function.rs @@ -0,0 +1,10 @@ +//! `compile_facts` as a function of `module`. + +use crate::rules::_helpers::fix::fix_facts::compile_facts; + +/// `compile_facts` as a function of `module`. +pub fn compile_facts_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(compile_facts, module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/fix/fix_verdict_function.rs b/crates/sqlbuild-python-compile/src/rules/main/fix/fix_verdict_function.rs new file mode 100644 index 0000000000..01a4fbd883 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fix/fix_verdict_function.rs @@ -0,0 +1,10 @@ +//! `fix_verdict` as a function of `module`. + +use crate::rules::_helpers::fix::fix_verdict::fix_verdict; + +/// `fix_verdict` as a function of `module`. +pub fn fix_verdict_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(fix_verdict, module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/fix/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/fix/mod.rs new file mode 100644 index 0000000000..31010ee1e5 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fix/mod.rs @@ -0,0 +1,4 @@ +pub mod compile_facts_function; +pub mod fix_verdict_function; +pub mod plan_rule_fixes; +pub mod plan_rule_fixes_function; diff --git a/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes.rs b/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes.rs new file mode 100644 index 0000000000..87776e1622 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes.rs @@ -0,0 +1,62 @@ +//! Plan fixes without exposing any unverified intermediate source on disk. + +use crate::rules::_helpers::fix::fix_session::FixSession; +use crate::rules::_helpers::fix::fix_session::result; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +/// Plan fixes without exposing any unverified intermediate source on disk. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn plan_rule_fixes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_rule_fixes", + options, + ( + &[ + "project_dir", + "files", + "config", + "adapter", + "discovered_inputs", + "enabled", + ], + &[], + ), + )?; + let files = args.value("files")?.cast_into::()?; + if !args.flag("enabled")? { + return result(&files, &[], &[]); + } + let adapter = args.value("adapter")?; + let inputs = args.value("discovered_inputs")?; + if !adapter.is_instance(&host_entry(py, AdapterHost::BaseAdapter)?)? || inputs.is_none() { + return Err(PyErr::from_value( + error_class(py, ErrorId::ProjectCompile)? + .call1(("format --fix requires discovered compiler inputs and an adapter",))?, + )); + } + FixSession { + project_dir: args.value("project_dir")?, + original: files, + config: args.value("config")?, + adapter, + inputs, + reports: Vec::new(), + blocked: PyDict::new(py), + skipped: PyDict::new(py), + baseline: None, + baseline_facts: None, + } + .run() +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes_function.rs b/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes_function.rs new file mode 100644 index 0000000000..82cfa596f3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes_function.rs @@ -0,0 +1,10 @@ +//! `plan_rule_fixes` as a function of `module`. + +use crate::rules::main::fix::plan_rule_fixes::plan_rule_fixes; + +/// `plan_rule_fixes` as a function of `module`. +pub fn plan_rule_fixes_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_rule_fixes, module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/lint_file_role_function.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/lint_file_role_function.rs new file mode 100644 index 0000000000..ce99c8a784 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/lint_file_role_function.rs @@ -0,0 +1,10 @@ +//! `lint_file_role` as a function of `module`. + +use crate::rules::_helpers::lint::lint_headers::lint_file_role; + +/// `lint_file_role` as a function of `module`. +pub fn lint_file_role_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(lint_file_role, module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/mod.rs new file mode 100644 index 0000000000..015b24f280 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/mod.rs @@ -0,0 +1,10 @@ +pub mod lint_file_role_function; +pub mod prepare_lint_body_function; +pub mod project_relative_path; +pub mod register_early_sql_lint; +pub mod register_lint_expansion; +pub mod register_lint_sql_bodies; +pub mod register_lint_text; +pub mod register_run_lint; +pub mod relative_parts; +pub mod unsafe_fault; diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/prepare_lint_body_function.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/prepare_lint_body_function.rs new file mode 100644 index 0000000000..e034b3eb85 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/prepare_lint_body_function.rs @@ -0,0 +1,10 @@ +//! `prepare_lint_body` as a function of `module`. + +use crate::rules::_helpers::lint::lint_preparation::prepare_lint_body; + +/// `prepare_lint_body` as a function of `module`. +pub fn prepare_lint_body_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(prepare_lint_body, module) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/project_relative_path.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/project_relative_path.rs new file mode 100644 index 0000000000..89cca7555b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/project_relative_path.rs @@ -0,0 +1,13 @@ +//! `path` relative to `project_dir` with `/` separators, or as given outside the project. + +use std::path::Path; + +/// `path` relative to `project_dir` with `/` separators, or as given outside the project. +pub fn project_relative_path(path: &Path, project_dir: &Path) -> String { + let relative: &Path = path.strip_prefix(project_dir).unwrap_or(path); + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/register_early_sql_lint.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/register_early_sql_lint.rs new file mode 100644 index 0000000000..1bdbc3449b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/register_early_sql_lint.rs @@ -0,0 +1,12 @@ +//! Register the early sql lint functions and classes. + +use crate::rules::models::EarlySqlLint; +use crate::rules::models::PreparedSqlLint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the early sql lint functions and classes. +pub fn register_early_sql_lint(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_expansion.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_expansion.rs new file mode 100644 index 0000000000..6a353a40ef --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_expansion.rs @@ -0,0 +1,10 @@ +//! Register the lint expansion functions and classes. + +use crate::rules::_helpers::lint::lint_expansion::build_lint_expansion_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the lint expansion functions and classes. +pub fn register_lint_expansion(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_lint_expansion_context, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_sql_bodies.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_sql_bodies.rs new file mode 100644 index 0000000000..4a368ffc9d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_sql_bodies.rs @@ -0,0 +1,10 @@ +//! Register the lint sql bodies functions and classes. + +use crate::rules::_helpers::lint::lint_sql_bodies::lint_native_sql_bodies; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the lint sql bodies functions and classes. +pub fn register_lint_sql_bodies(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lint_native_sql_bodies, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_text.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_text.rs new file mode 100644 index 0000000000..ef1f58a473 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_text.rs @@ -0,0 +1,18 @@ +//! Register the lint text functions and classes. + +use crate::rules::_helpers::lint::lint_text::format_lint_sql_bodies; +use crate::rules::_helpers::lint::lint_text::named_declaration_role_kind; +use crate::rules::_helpers::lint::lint_text::neutralize_lint_interpolation; +use crate::rules::_helpers::lint::lint_text::reject_lint_suppressions; +use crate::rules::_helpers::lint::lint_text::restore_lint_interpolation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the lint text functions and classes. +pub fn register_lint_text(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(neutralize_lint_interpolation, module)?)?; + module.add_function(wrap_pyfunction!(restore_lint_interpolation, module)?)?; + module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; + module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; + module.add_function(wrap_pyfunction!(reject_lint_suppressions, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/register_run_lint.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/register_run_lint.rs new file mode 100644 index 0000000000..010c737c21 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/register_run_lint.rs @@ -0,0 +1,13 @@ +//! Register the run lint functions and classes. + +use crate::rules::_helpers::lint::run_lint::collect_lint_project_files; +use crate::rules::_helpers::lint::run_lint::run_lint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the run lint functions and classes. +pub fn register_run_lint(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_lint, module)?)?; + module.add_function(wrap_pyfunction!(collect_lint_project_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/relative_parts.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/relative_parts.rs new file mode 100644 index 0000000000..9ea90bf6b0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/relative_parts.rs @@ -0,0 +1,17 @@ +//! The parts of `path` below `project_dir`, or `None` outside the project. + +/// The parts of `path` below `project_dir`, or `None` outside the project. +pub fn relative_parts( + path: &std::path::Path, + project_dir: &std::path::Path, +) -> Option> { + match path.strip_prefix(project_dir) { + Ok(relative) => Some( + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect(), + ), + Err(_) => None, + } +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/lint/unsafe_fault.rs b/crates/sqlbuild-python-compile/src/rules/main/lint/unsafe_fault.rs new file mode 100644 index 0000000000..f3dde1e3d0 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/lint/unsafe_fault.rs @@ -0,0 +1,54 @@ +//! A declined body's `format-unsafe` fault, or a `format-unavailable` warning for an unformatted dialect. + +use crate::rules::_helpers::lint::lint_pipeline::NATIVE_ENGINE; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNAVAILABLE_CODE; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNAVAILABLE_PREFIX; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNAVAILABLE_REMEDIATION; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNSAFE_CODE; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNSAFE_PREFIX; +use crate::rules::_helpers::lint::lint_text::FORMAT_UNSAFE_REMEDIATION; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_rules::lint_text::models::FormatFault; + +/// A declined body's `format-unsafe` fault, or a `format-unavailable` warning for an unformatted dialect. +pub fn unsafe_fault<'py>( + file_path: &Bound<'py, PyAny>, + fault: &FormatFault, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let (severity, code, prefix): (&str, &str, &str) = if fault.unavailable { + ( + "WARNING", + FORMAT_UNAVAILABLE_CODE, + FORMAT_UNAVAILABLE_PREFIX, + ) + } else { + ("FAULT", FORMAT_UNSAFE_CODE, FORMAT_UNSAFE_PREFIX) + }; + let remediation: &str = if fault.unavailable { + FORMAT_UNAVAILABLE_REMEDIATION + } else { + FORMAT_UNSAFE_REMEDIATION + }; + let message: String = format!("{prefix}: {}", fault.reason); + let severity: Bound<'py, PyAny> = enum_member(py, RecordId::LintSeverity, severity)?; + new_record_with( + py, + RecordId::LintViolation, + &[ + ("file_path", file_path.clone()), + ("line", fault.line.into_pyobject(py)?.into_any()), + ("column", 1_usize.into_pyobject(py)?.into_any()), + ("code", PyString::new(py, code).into_any()), + ("message", PyString::new(py, &message).into_any()), + ("severity", severity), + ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), + ("remediation", PyString::new(py, remediation).into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/mod.rs new file mode 100644 index 0000000000..4abb858480 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/mod.rs @@ -0,0 +1,8 @@ +pub mod checking; +pub mod configuration; +pub mod custom; +pub mod fact; +pub mod fix; +pub mod lint; +pub mod rule; +pub mod sql; diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/evaluate_rules.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/evaluate_rules.rs new file mode 100644 index 0000000000..3e9401ba4b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/evaluate_rules.rs @@ -0,0 +1,17 @@ +//! Evaluate independent rule phases concurrently, then finalize their combined findings. + +use crate::rules::_helpers::rule::rule_dispatch; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Evaluate independent rule phases concurrently, then finalize their combined findings. +#[inline] +pub fn evaluate_rules<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + rule_dispatch::evaluate_rules(py, options) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/mod.rs new file mode 100644 index 0000000000..e17fa8784b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/mod.rs @@ -0,0 +1,7 @@ +pub mod evaluate_rules; +pub mod register_rule_api; +pub mod register_rule_cache; +pub mod register_rule_dispatch; +pub mod register_rule_plan; +pub mod register_rule_run; +pub mod run_rules; diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_api.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_api.rs new file mode 100644 index 0000000000..11d63fb96e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_api.rs @@ -0,0 +1,17 @@ +//! Register the rule api functions and classes. + +use crate::rules::_helpers::rule::rule_api::finding_position; +use crate::rules::_helpers::rule::rule_api::project_tree_paths; +use crate::rules::_helpers::rule::rule_api::rule_option_toml_value; +use crate::rules::_helpers::rule::rule_api::validate_project_relative; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rule api functions and classes. +pub fn register_rule_api(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finding_position, module)?)?; + module.add_function(wrap_pyfunction!(rule_option_toml_value, module)?)?; + module.add_function(wrap_pyfunction!(validate_project_relative, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_paths, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_cache.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_cache.rs new file mode 100644 index 0000000000..9561300f87 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_cache.rs @@ -0,0 +1,15 @@ +//! Register the rule cache functions and classes. + +use crate::rules::_helpers::rule::rule_cache::encode_sql_rule_findings; +use crate::rules::_helpers::rule::rule_cache::load_sql_rule_cache; +use crate::rules::_helpers::rule::rule_cache::sql_file_rule_key; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rule cache functions and classes. +pub fn register_rule_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_file_rule_key, module)?)?; + module.add_function(wrap_pyfunction!(load_sql_rule_cache, module)?)?; + module.add_function(wrap_pyfunction!(encode_sql_rule_findings, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs new file mode 100644 index 0000000000..007fd7b14d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs @@ -0,0 +1,16 @@ +//! Register the rule dispatch functions and classes. + +use crate::rules::_helpers::rule::rule_dispatch::evaluate_custom_rules; +use crate::rules::_helpers::rule::rule_dispatch::evaluate_native_rules; +use crate::rules::_helpers::rule::rule_dispatch::evaluate_project_rules; +use crate::rules::_helpers::rule::rule_dispatch::evaluate_rules; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rule dispatch functions and classes. +pub fn register_rule_dispatch(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(evaluate_project_rules, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_native_rules, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_custom_rules, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_rules, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_plan.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_plan.rs new file mode 100644 index 0000000000..38f8753c60 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_plan.rs @@ -0,0 +1,11 @@ +//! Register the rule plan functions and classes. + +use crate::rules::_helpers::rule::rule_plan::plan_rule_run; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rule plan functions and classes. +pub fn register_rule_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_rule_run, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_run.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_run.rs new file mode 100644 index 0000000000..8b7989085c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_run.rs @@ -0,0 +1,15 @@ +//! Register the rule run functions and classes. + +use crate::rules::_helpers::rule::rule_run::finish_rule_run; +use crate::rules::_helpers::rule::rule_run::rules_bulk_cache_path; +use crate::rules::main::rule::run_rules::run_rules; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the rule run functions and classes. +pub fn register_rule_run(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finish_rule_run, module)?)?; + module.add_function(wrap_pyfunction!(run_rules, module)?)?; + module.add_function(wrap_pyfunction!(rules_bulk_cache_path, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/rule/run_rules.rs b/crates/sqlbuild-python-compile/src/rules/main/rule/run_rules.rs new file mode 100644 index 0000000000..a4cef5f348 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/rule/run_rules.rs @@ -0,0 +1,15 @@ +//! Run the selected built-in, SQL and custom rules; custom rules run on their worker threads. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +/// Run the selected built-in, SQL and custom rules; custom rules run on their worker threads. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn run_rules<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + crate::rules::_helpers::rule::rule_dispatch::evaluate_rules(py, options) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs b/crates/sqlbuild-python-compile/src/rules/main/sql/format_redundant_fixture_nulls.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs rename to crates/sqlbuild-python-compile/src/rules/main/sql/format_redundant_fixture_nulls.rs index 67026835f7..c0e1f1b888 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs +++ b/crates/sqlbuild-python-compile/src/rules/main/sql/format_redundant_fixture_nulls.rs @@ -1,24 +1,22 @@ -//! Fixture schema conversion at the Python model boundary. +//! Format redundant fixture nulls. +use crate::rules::constants::MODEL_TEST_MODE; use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::lint_text::main::format_fixture_blocks::format_fixture_blocks; -use sqlbuild_rules::lint_text::models::FixtureFormatBlock; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; - -use crate::bindings::_helpers::rules::constants::MODEL_TEST_MODE; -use crate::bindings::_helpers::rules::fixture_schemas; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::records::parse_sql_test_file::parse_sql_test_file; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_rules::lint_text::main::format_fixture_blocks::format_fixture_blocks; +use sqlbuild_rules::lint_text::models::FixtureFormatBlock; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; #[pyfunction] #[pyo3(signature = (*, files, project_dir, discovered_inputs, sql_lexical_syntax))] -pub(crate) fn format_redundant_fixture_nulls<'py>( +pub fn format_redundant_fixture_nulls<'py>( files: &Bound<'py, PyDict>, project_dir: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, @@ -26,7 +24,7 @@ pub(crate) fn format_redundant_fixture_nulls<'py>( ) -> PyResult> { compiler_guard(|| { let py = files.py(); - let relations = fixture_schemas::relations(discovered_inputs)?; + let relations = crate::rules::_helpers::sql::fixture_schemas::relations(discovered_inputs)?; let syntax: LexicalSyntax = native_field(sql_lexical_syntax, "native_mapping")? .extract::()? .into(); @@ -76,7 +74,3 @@ pub(crate) fn format_redundant_fixture_nulls<'py>( Ok(updated) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_redundant_fixture_nulls, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/rules/main/sql/mod.rs b/crates/sqlbuild-python-compile/src/rules/main/sql/mod.rs new file mode 100644 index 0000000000..d397ff0dc1 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/sql/mod.rs @@ -0,0 +1,4 @@ +pub mod format_redundant_fixture_nulls; +pub mod register_fixture_formatting; +pub mod register_sql_document; +pub mod register_sql_rules; diff --git a/crates/sqlbuild-python-compile/src/rules/main/sql/register_fixture_formatting.rs b/crates/sqlbuild-python-compile/src/rules/main/sql/register_fixture_formatting.rs new file mode 100644 index 0000000000..16e6d9a21f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/sql/register_fixture_formatting.rs @@ -0,0 +1,10 @@ +//! Register the fixture formatting functions and classes. + +use crate::rules::main::sql::format_redundant_fixture_nulls::format_redundant_fixture_nulls; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fixture formatting functions and classes. +pub fn register_fixture_formatting(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_redundant_fixture_nulls, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_document.rs b/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_document.rs new file mode 100644 index 0000000000..b3a257d6ba --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_document.rs @@ -0,0 +1,12 @@ +//! Register the sql document functions and classes. + +use crate::rules::_helpers::sql::sql_document::SqlDocument; +use crate::rules::_helpers::sql::sql_document::SqlTreeNode; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the sql document functions and classes. +pub fn register_sql_document(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_rules.rs b/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_rules.rs new file mode 100644 index 0000000000..ab2fe43b06 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/main/sql/register_sql_rules.rs @@ -0,0 +1,11 @@ +//! Register the sql rules functions and classes. + +use crate::rules::_helpers::sql::sql_rules::run_planned_sql_rules; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the sql rules functions and classes. +pub fn register_sql_rules(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_planned_sql_rules, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/rules/mod.rs b/crates/sqlbuild-python-compile/src/rules/mod.rs new file mode 100644 index 0000000000..51abf36174 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/mod.rs @@ -0,0 +1,6 @@ +//! Rules: the rule catalogue and active rules, built-in and custom rule evaluation, and rule results. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/rules/models.rs b/crates/sqlbuild-python-compile/src/rules/models.rs new file mode 100644 index 0000000000..e9db21aa36 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/rules/models.rs @@ -0,0 +1,648 @@ +//! Public records of rules. + +use crate::compile_attachment::models::ExpansionVars; +use crate::rules::_helpers::checking::early_builtin_rules::take_started; +use crate::rules::_helpers::checking::early_builtin_rules::{SLOT, Slot}; +use crate::rules::_helpers::checking::native_header_parser::DESCRIPTION_KEY; +use crate::rules::_helpers::checking::native_header_parser::SENTINEL_PATH; +use crate::rules::_helpers::checking::native_header_parser::map_value; +use crate::rules::_helpers::checking::native_header_parser::python_equal; +use crate::rules::_helpers::configuration::rules_config::rules_error; +use crate::rules::_helpers::lint::early_sql_lint::same_codes; +use crate::rules::_helpers::lint::early_sql_lint::start_early_lint; +use crate::rules::_helpers::lint::lint_pipeline::FAULT_SEVERITY; +use crate::rules::_helpers::lint::lint_pipeline::HEADER_ENGINE; +use crate::rules::_helpers::lint::lint_pipeline::NATIVE_ENGINE; +use crate::rules::_helpers::lint::lint_pipeline::WARNING_SEVERITY; +use crate::rules::_helpers::lint::lint_pipeline::offset; +use pyo3::exceptions::PyException; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::IntoPyObject; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyRefMut; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interruptibly; +use sqlbuild_python_base::native_threads::models::Joined; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as record_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_front::sqltext::main::header_failure::header_failure; +use sqlbuild_rules::dsl_headers::models::HeaderSeverity; +use sqlbuild_rules::dsl_headers::models::HeaderViolation; +use sqlbuild_rules::dsl_headers::models::ParseFailure; +use sqlbuild_rules::dsl_headers::models::ParseOutcome; +use sqlbuild_rules::dsl_headers::types::HeaderParser; +use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::lint_text::models::AuthoredDiagnostic; +use sqlbuild_rules::lint_text::models::SuppressionFinding; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use sqlbuild_sqltext::compiler::models::HeaderWord; +use sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure; +use sqlbuild_sqltext::header_strings::models::QuoteFailure; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::OnceLock; +use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread::JoinHandle; +use std::time::Instant; + +/// What one compile's early built-in rules start from. +#[derive(Debug)] +pub struct EarlyRulesSettings { + pub project_dir: PathBuf, + pub dialect: String, + pub no_sql_analysis: bool, + /// Whether `--no-cache` turned the Rules cache off, as the rules phase does. + pub no_cache: bool, +} + +/// Arms early built-in rules for one compile analysis on this thread; dropping it stops them. +#[derive(Debug)] +pub struct EarlyRulesScope { + pub(crate) owner: bool, +} + +impl EarlyRulesScope { + /// Claim this thread's slot unarmed; a nested compile analysis on this thread claims nothing. + pub(crate) fn new() -> Self { + let owner: bool = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_some() { + return false; + } + *slot = Some(Slot::Spent); + true + }); + Self { owner } + } + + /// Start early rules with `settings` once the compile recovers its model types. + pub fn arm(&self, settings: EarlyRulesSettings) { + if !self.owner { + return; + } + SLOT.with(|slot| { + if let Some(current @ Slot::Spent) = slot.borrow_mut().as_mut() { + *current = Slot::Armed(settings); + } + }); + } + + /// Cancel and join early rules this scope armed that started and were not taken. + pub fn stop(&self, py: Python<'_>) { + if self.owner + && let Some(rules) = take_started() + { + rules.discard(py); + } + } +} + +impl Drop for EarlyRulesScope { + fn drop(&mut self) { + if !self.owner { + return; + } + let slot: Option = SLOT.with(|slot| slot.borrow_mut().take()); + drop(slot); + } +} + +/// Early SQL lint's outcome and the expansion inputs it linted with. +#[derive(Debug)] +pub struct EarlyLintResult { + pub outcome: LintOutcome, + pub context: SqlExpansionInputs, +} + +/// One early SQL lint running on a native thread, for one project and SQL rule codes. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct PreparedSqlLint { + pub(crate) codes: Vec, + pub(crate) project_dir: PathBuf, + pub(crate) thread: Mutex>>>, + pub(crate) result: OnceLock>, + pub(crate) stop: Arc, +} + +impl PreparedSqlLint { + /// Whether this lint ran for `project_dir` with exactly `codes`. + pub fn matches(&self, project_dir: &Path, codes: &[String]) -> bool { + self.project_dir == project_dir && same_codes(&self.codes, codes) + } + + /// Whether this lint ran with exactly `codes`. + pub fn has_codes(&self, codes: &[String]) -> bool { + same_codes(&self.codes, codes) + } + + /// Whether the lint's result is already known. + pub(crate) fn finished(&self) -> bool { + self.result.get().is_some() + } + + /// The lint's result, waiting without the GIL until it finishes or a signal handler raises. + pub fn result(&self, py: Python<'_>) -> &Result { + self.result.get_or_init(|| { + let thread = self + .thread + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let Some(thread) = thread else { + return Err(PyRuntimeError::new_err("early SQL lint was already joined")); + }; + match join_interruptibly(py, thread, || self.stop.store(true, Ordering::Release)) { + Joined::Finished(joined) => joined.unwrap_or_else(|_| { + Err(PyRuntimeError::new_err("early SQL lint thread panicked")) + }), + Joined::Interrupted(signal) => Err(signal), + } + }) + } +} + +/// The compiler's inputs and adapter, which lint expansion reuses for the same discovery. +#[derive(Debug)] +pub struct ExpansionReuse { + pub(crate) inputs: pyo3::prelude::Py, + pub(crate) value_renderer: pyo3::prelude::Py, +} + +impl ExpansionReuse { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + inputs: self.inputs.clone_ref(py), + value_renderer: self.value_renderer.clone_ref(py), + } + } + + /// The discovery the compile ran on. + pub fn discovered_inputs<'py>(&self, py: Python<'py>) -> PyResult> { + record_field(self.inputs.bind(py), "discovered_inputs") + } + + /// The compile's declaration scope. + pub fn declaration_scope<'py>(&self, py: Python<'py>) -> PyResult> { + record_field(self.inputs.bind(py), "declaration_scope") + } + + /// The compile's target and vars, read from its macro context. + pub fn expansion_vars<'py>(&self, py: Python<'py>) -> PyResult> { + let context: Bound<'py, PyAny> = record_field(self.inputs.bind(py), "macro_context")?; + Ok(ExpansionVars::Compile { + target_name: record_field(&context, "target_name")?, + vars: record_field(&context, "vars")?, + }) + } + + /// The adapter that renders declared values. + pub fn value_renderer<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.value_renderer.bind(py).clone() + } +} + +/// Own speculative SQL checks for one compilation invocation. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct EarlySqlLint { + #[pyo3(get)] + pub enabled: bool, + /// Whether `--no-cache` turned the Rules cache off, so a warm SQL rule cache is ignored. + pub(crate) no_cache: bool, + pub(crate) preparation: Option>, + pub(crate) expansion_reuse: Option, + /// The dialect and adapter the `on_inputs_ready` call passes to `start`. + pub(crate) ready_arguments: Option<(String, pyo3::prelude::Py)>, + pub(crate) stop_requested: Arc, +} + +impl EarlySqlLint { + /// A manager that starts speculative checks only when `enabled`. + pub fn new(enabled: bool, no_cache: bool) -> Self { + Self { + enabled, + no_cache, + preparation: None, + expansion_reuse: None, + ready_arguments: None, + stop_requested: Arc::new(AtomicBool::new(false)), + } + } + + /// The running early SQL lint, if one started. + pub fn preparation(&self) -> Option<&pyo3::prelude::Py> { + self.preparation.as_ref() + } + + /// The compile's discovery and declaration scope, if it had one. + pub fn expansion_reuse(&self) -> Option<&ExpansionReuse> { + self.expansion_reuse.as_ref() + } + + /// This manager as the `on_inputs_ready` hook, which calls `start` with `dialect` and `value_renderer`. + pub fn inputs_ready_hook<'py>( + slf: &Bound<'py, Self>, + dialect: &str, + value_renderer: &Bound<'py, PyAny>, + ) -> Bound<'py, PyAny> { + slf.borrow_mut().ready_arguments = + Some((dialect.to_owned(), value_renderer.clone().unbind())); + slf.clone().into_any() + } + + /// Abandon speculative work, return the milliseconds spent joining it, and raise Ctrl+C caught while waiting. + pub fn stop(&self, py: Python<'_>) -> PyResult { + let started: Instant = Instant::now(); + self.stop_requested.store(true, Ordering::Release); + if let Some(preparation) = &self.preparation { + let preparation: &PreparedSqlLint = preparation.get(); + let waiting: bool = !preparation.finished(); + if let Err(error) = preparation.result(py) + && waiting + && !error.is_instance_of::(py) + { + return Err(error.clone_ref(py)); + } + } + Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) + } +} + +#[pymethods] +impl EarlySqlLint { + #[new] + #[pyo3(signature = (*, enabled, no_cache=false))] + pub(crate) fn py_new(enabled: bool, no_cache: bool) -> Self { + Self::new(enabled, no_cache) + } + + pub(crate) fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { + slf + } + + #[pyo3(signature = (*_exception))] + pub(crate) fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) { + let _ = self.stop(py); + } + + /// Keep the compile's expansion inputs, and start early SQL lint when enabled and SQL rules are selected. + #[pyo3(signature = (*, inputs, dialect, value_renderer))] + pub(crate) fn start( + mut slf: PyRefMut<'_, Self>, + inputs: &Bound<'_, PyAny>, + dialect: &str, + value_renderer: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = inputs.py(); + let reuse: ExpansionReuse = ExpansionReuse { + inputs: inputs.clone().unbind(), + value_renderer: value_renderer.clone().unbind(), + }; + if slf.enabled { + let stop: Arc = Arc::clone(&slf.stop_requested); + slf.preparation = + start_early_lint((inputs, dialect), reuse.clone_ref(py), slf.no_cache, &stop)? + .map(|preparation| pyo3::prelude::Py::new(py, preparation)) + .transpose()?; + } + slf.expansion_reuse = (!reuse.declaration_scope(py)?.is_none()).then_some(reuse); + Ok(()) + } + + /// Start with the arguments `inputs_ready_hook` stored, as the compile's `on_inputs_ready` hook. + #[pyo3(signature = (*, inputs))] + pub(crate) fn __call__(slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = inputs.py(); + let Some((dialect, value_renderer)) = slf + .ready_arguments + .as_ref() + .map(|(dialect, renderer)| (dialect.clone(), renderer.clone_ref(py))) + else { + return Err(PyRuntimeError::new_err( + "EarlySqlLint was called before inputs_ready_hook armed it", + )); + }; + Self::start(slf, inputs, &dialect, value_renderer.bind(py)) + } + + /// Abandon unconsumed speculative work and return the milliseconds spent joining it. + #[pyo3(name = "stop")] + pub(crate) fn py_stop(&self, py: Python<'_>) -> PyResult { + self.stop(py) + } +} + +/// One authored SQL file and its text. +#[derive(Clone, Debug)] +pub struct LintFile { + pub path: PathBuf, + pub contents: String, +} + +/// A proposed authored-source replacement in the violation's file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LintEditRow { + pub start: i64, + pub end: i64, + pub replacement: String, +} + +/// One lint violation against the file at index `file` of its run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LintViolationRow { + pub file: usize, + pub line: i64, + pub column: i64, + pub code: String, + pub message: String, + pub fault: bool, + pub engine: &'static str, + pub end: Option<(i64, i64)>, + pub remediation: Option, + pub fix: Option, + pub fix_unavailable_reason: Option, +} + +impl LintViolationRow { + pub fn header(file: usize, violation: HeaderViolation) -> Self { + Self { + file, + line: offset(violation.line), + column: offset(violation.column), + code: violation.code.to_owned(), + message: violation.message, + fault: violation.severity == HeaderSeverity::Fault, + engine: HEADER_ENGINE, + end: None, + remediation: Some(violation.remediation), + fix: None, + fix_unavailable_reason: None, + } + } + + pub fn diagnostic(diagnostic: AuthoredDiagnostic) -> Self { + Self { + file: diagnostic.file, + line: diagnostic.line, + column: diagnostic.column, + code: diagnostic.code, + message: diagnostic.message, + fault: diagnostic.fault, + engine: NATIVE_ENGINE, + end: diagnostic.end, + remediation: Some(diagnostic.remediation), + fix: diagnostic.fix.map(|edit| LintEditRow { + start: edit.start, + end: edit.end, + replacement: edit.replacement, + }), + fix_unavailable_reason: diagnostic.fix_unavailable_reason, + } + } + + pub fn suppression(finding: SuppressionFinding) -> Self { + Self { + file: finding.file, + line: offset(finding.line), + column: 1, + code: SUPPRESSION_CODE.to_owned(), + message: finding.message, + fault: finding.fault, + engine: NATIVE_ENGINE, + end: None, + remediation: Some(finding.remediation.to_owned()), + fix: finding.fix.map(|(start, end)| LintEditRow { + start: offset(start), + end: offset(end), + replacement: String::new(), + }), + fix_unavailable_reason: finding.fix_unavailable_reason.map(str::to_owned), + } + } +} + +/// The lint model classes violations are projected as. +#[derive(Debug)] +pub struct ViolationClasses<'py> { + pub(crate) violation: Bound<'py, PyAny>, + pub(crate) edit: Bound<'py, PyAny>, + pub(crate) severity: Bound<'py, PyAny>, +} + +impl<'py> ViolationClasses<'py> { + pub fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: record_class(py, RecordId::LintViolation)?, + edit: record_class(py, RecordId::LintEdit)?, + severity: record_class(py, RecordId::LintSeverity)?, + }) + } + + /// `row` as a `LintViolation` against `file_path`. + pub fn violation( + &self, + file_path: &Bound<'py, PyAny>, + row: &LintViolationRow, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let none = || py.None().into_bound(py); + let text = |value: &Option| { + value + .as_deref() + .map_or_else(none, |text| PyString::new(py, text).into_any()) + }; + let code: Bound<'py, PyAny> = PyString::new(py, &row.code).into_any(); + let fix: Bound<'py, PyAny> = match &row.fix { + Some(edit) => call_with( + &self.edit, + &[ + ("file_path", file_path.clone()), + ("code", code.clone()), + ("start", edit.start.into_pyobject(py)?.into_any()), + ("end", edit.end.into_pyobject(py)?.into_any()), + ( + "replacement", + PyString::new(py, &edit.replacement).into_any(), + ), + ], + )?, + None => none(), + }; + let (end_line, end_column) = match row.end { + Some((line, column)) => ( + line.into_pyobject(py)?.into_any(), + column.into_pyobject(py)?.into_any(), + ), + None => (none(), none()), + }; + let severity: &str = if row.fault { + FAULT_SEVERITY + } else { + WARNING_SEVERITY + }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", row.line.into_pyobject(py)?.into_any()), + ("column", row.column.into_pyobject(py)?.into_any()), + ("code", code), + ("message", PyString::new(py, &row.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, row.engine).into_any()), + ("end_line", end_line), + ("end_column", end_column), + ("remediation", text(&row.remediation)), + ("fix", fix), + ("fix_unavailable_reason", text(&row.fix_unavailable_reason)), + ], + ) + } +} + +/// Every linted file, its sorted violations and the files expansion skipped. +#[derive(Debug)] +pub struct LintOutcome { + /// The linted files in the order they were given. + pub files: Vec, + pub violations: Vec, + /// Skipped files by index, with why they could not be expanded. + pub unexpandable: Vec<(usize, String)>, +} + +impl LintOutcome { + /// This outcome as a `LintRunResult`. + pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let paths: Vec> = self + .files + .iter() + .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) + .collect::>()?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + let violations: Vec> = self + .violations + .iter() + .map(|row| classes.violation(&paths[row.file], row)) + .collect::>()?; + let texts: Bound<'py, PyDict> = PyDict::new(py); + for (path, file) in paths.iter().zip(&self.files) { + texts.set_item(path, &file.contents)?; + } + let unexpandable: Bound<'py, PyDict> = PyDict::new(py); + for (file, message) in &self.unexpandable { + unexpandable.set_item(&paths[*file], message)?; + } + new_record_with( + py, + RecordId::LintRunResult, + &[ + ( + "files_checked", + self.files.len().into_pyobject(py)?.into_any(), + ), + ("violations", PyTuple::new(py, violations)?.into_any()), + ("formatted_files", PyTuple::empty(py).into_any()), + ("source_texts", texts.into_any()), + ("unexpandable", unexpandable.into_any()), + ], + ) + } +} + +/// Header values parsed natively, compared as their Python projections compare. +#[derive(Debug, Default)] +pub struct NativeHeaderParser; + +impl HeaderParser for NativeHeaderParser { + type Values = AuthoredValue; + /// A native failure's message. + type Error = String; + + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { + let parsed = match catch_compiler_panic(|| Ok(parse_one(inner))) { + Ok(parsed) => parsed, + Err(message) => { + return ParseOutcome::Raised( + message.clone(), + ParseFailure { + message, + help: None, + position: None, + }, + ); + } + }; + let (value, _, error) = parsed; + if let Some(value) = value { + return ParseOutcome::Parsed(value); + } + let error: String = error.unwrap_or_else(|| "None".to_owned()); + let quote: Option = quote_failure(&error, inner, (1, 1)); + let (detail, help, _) = header_failure(&error, inner, None); + let failure: ParseFailure = ParseFailure { + message: format!( + "{kind}(...) in '{SENTINEL_PATH}' contains invalid SQLBuild header syntax: {detail}" + ), + help, + position: quote.map(|quote| quote.position), + }; + ParseOutcome::Rejected(failure) + } + + fn description(&self, values: &AuthoredValue) -> Result, String> { + Ok(match map_value(values, DESCRIPTION_KEY) { + Some(AuthoredValue::String(text)) => Some(text.clone()), + Some(AuthoredValue::BareWord(word)) if header_word_value(word) == HeaderWord::Text => { + Some(word.clone()) + } + _ => None, + }) + } + + fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> Result { + Ok(python_equal(left, right)) + } +} + +/// Why a project's rules configuration is invalid. +#[derive(Debug)] +pub enum InvalidRulesConfig { + /// The configuration could not be read: a `RulesError`. + Unreadable(String), + /// It uses a setting SQLBuild rejects: a `RulesConfigError` with help. + Forbidden { message: String, help: String }, +} + +impl InvalidRulesConfig { + pub(crate) fn into_pyerr(self, py: Python<'_>) -> PyErr { + match self { + Self::Unreadable(message) => rules_error(py, ErrorId::Rules, message, None), + Self::Forbidden { message, help } => { + rules_error(py, ErrorId::RulesConfig, message, Some(help)) + } + } + } +} diff --git a/crates/sqlbuild-python-compile/src/scopes/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/scopes/_helpers/mod.rs new file mode 100644 index 0000000000..c083a6cf03 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/scopes/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod scope_loading; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs rename to crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs index 73a9a0da16..8d09c0680f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs @@ -3,47 +3,42 @@ use std::path::PathBuf; use pyo3::exceptions::{PyImportError, PyOSError, PyUnicodeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::{AdapterHost, UserCodeHost}; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; -use crate::bindings::_helpers::compile_driver::project::compile_adapter_context; -use crate::bindings::_helpers::frontier::cache_directory::cache_directory; -use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; +use crate::compile_driver::main::setup::attached_inputs::attached_inputs; +use crate::compile_driver::main::setup::compile_adapter_context::compile_adapter_context; +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use crate::compile_driver::models::InputsRequest; +use crate::frontier::main::bridged::bridged; +use crate::frontier::main::cache_directory::cache_directory; +use crate::frontier::main::produced::produced; +use crate::frontier::models::FrontierStage; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::main::snapshots::discover_scope_snapshot::discover_scope_snapshot; use sqlbuild_python_front::scopes::main::checks::scope_index_with_compile_usages::scope_index_with_compile_usages; use sqlbuild_python_front::scopes::main::checks::scope_tolerant_index::scope_tolerant_index; use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_native_scope_index; -use sqlbuild_python_front::scopes::main::index::project_fingerprint::project_fingerprint; -use sqlbuild_python_front::scopes::main::index::read_scope_index_cache::read_scope_index_cache; -use sqlbuild_python_front::scopes::main::index::write_scope_index_cache::write_scope_index_cache; use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; /// Where the scope index cache file of a project lives: `(directory, file name)`. -fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { +pub(crate) fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { let directory: Bound<'_, PyAny> = cache_directory(project_dir)?.div(SCOPE_CACHE_DIRECTORY_NAME)?; Ok((directory.extract()?, SCOPE_CACHE_FILENAME.to_owned())) } /// Whether `error` is a project input error that selects the tolerant index. -fn is_project_input_error(py: Python<'_>, error: &PyErr) -> bool { +pub(crate) fn is_project_input_error(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) || error.is_instance_of::(py) || error.is_instance_of::(py) @@ -51,7 +46,7 @@ fn is_project_input_error(py: Python<'_>, error: &PyErr) -> bool { } /// The discovered project inputs and the project adapter that compiles them. -fn project_adapter<'py>( +pub(crate) fn project_adapter<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = project_dir.py(); @@ -72,7 +67,7 @@ fn project_adapter<'py>( } /// The compile inputs' canonical index with usage and placement facts. -fn compiled_index<'py>( +pub(crate) fn compiled_index<'py>( discovered: Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, syntax: &Bound<'py, PyAny>, @@ -107,7 +102,7 @@ fn compiled_index<'py>( } /// The partial static index of tolerant discovery, with its faults as diagnostics. -fn tolerant_index<'py>( +pub(crate) fn tolerant_index<'py>( project_dir: &Bound<'py, PyAny>, syntax: &Bound<'py, PyAny>, ) -> PyResult> { @@ -151,46 +146,3 @@ fn tolerant_index<'py>( )? .into_bound(py)) } - -/// Load or build scope facts without credentials, a connection, or warehouse access. -#[pyfunction] -#[pyo3(signature = (*, project_dir, no_cache=false))] -pub(crate) fn load_or_build_scope_index<'py>( - project_dir: &Bound<'py, PyAny>, - no_cache: bool, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let resolved: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; - let fingerprint: Option = project_fingerprint(py, resolved.extract()?)?; - let cached_fingerprint: Option<&String> = fingerprint.as_ref().filter(|_| !no_cache); - if let Some(fingerprint) = cached_fingerprint { - let (directory, file_name) = cache_location(&resolved)?; - if let Some(cached) = - read_scope_index_cache(py, directory.join(file_name), fingerprint.clone())? - { - return Ok(cached.into_bound(py)); - } - } - let mut syntax: Bound<'py, PyAny> = new_record(py, RecordId::SqlLexicalSyntax, (), None)?; - let compiled: PyResult> = match project_adapter(&resolved) { - Ok((discovered, adapter)) => { - syntax = native_field(&adapter, "sql_lexical_syntax")?; - compiled_index(discovered, &adapter, &syntax) - } - Err(error) => Err(error), - }; - let index: Bound<'py, PyAny> = match compiled { - Ok(index) => index, - Err(error) if is_project_input_error(py, &error) => tolerant_index(&resolved, &syntax)?, - Err(error) => return Err(error), - }; - if let Some(fingerprint) = cached_fingerprint { - let _ = - write_scope_index_cache(py, cache_location(&resolved)?, fingerprint.clone(), &index)?; - } - Ok(index) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_or_build_scope_index, module)?) -} diff --git a/crates/sqlbuild-python-compile/src/scopes/main/load_or_build_scope_index.rs b/crates/sqlbuild-python-compile/src/scopes/main/load_or_build_scope_index.rs new file mode 100644 index 0000000000..8cbda82ab3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/scopes/main/load_or_build_scope_index.rs @@ -0,0 +1,54 @@ +//! Load or build scope facts without credentials, a connection, or warehouse access. + +use crate::scopes::_helpers::scope_loading::cache_location; +use crate::scopes::_helpers::scope_loading::compiled_index; +use crate::scopes::_helpers::scope_loading::is_project_input_error; +use crate::scopes::_helpers::scope_loading::project_adapter; +use crate::scopes::_helpers::scope_loading::tolerant_index; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::scopes::main::index::project_fingerprint::project_fingerprint; +use sqlbuild_python_front::scopes::main::index::read_scope_index_cache::read_scope_index_cache; +use sqlbuild_python_front::scopes::main::index::write_scope_index_cache::write_scope_index_cache; + +/// Load or build scope facts without credentials, a connection, or warehouse access. +#[pyfunction] +#[pyo3(signature = (*, project_dir, no_cache=false))] +pub fn load_or_build_scope_index<'py>( + project_dir: &Bound<'py, PyAny>, + no_cache: bool, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let resolved: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; + let fingerprint: Option = project_fingerprint(py, resolved.extract()?)?; + let cached_fingerprint: Option<&String> = fingerprint.as_ref().filter(|_| !no_cache); + if let Some(fingerprint) = cached_fingerprint { + let (directory, file_name) = cache_location(&resolved)?; + if let Some(cached) = + read_scope_index_cache(py, directory.join(file_name), fingerprint.clone())? + { + return Ok(cached.into_bound(py)); + } + } + let mut syntax: Bound<'py, PyAny> = new_record(py, RecordId::SqlLexicalSyntax, (), None)?; + let compiled: PyResult> = match project_adapter(&resolved) { + Ok((discovered, adapter)) => { + syntax = native_field(&adapter, "sql_lexical_syntax")?; + compiled_index(discovered, &adapter, &syntax) + } + Err(error) => Err(error), + }; + let index: Bound<'py, PyAny> = match compiled { + Ok(index) => index, + Err(error) if is_project_input_error(py, &error) => tolerant_index(&resolved, &syntax)?, + Err(error) => return Err(error), + }; + if let Some(fingerprint) = cached_fingerprint { + let _ = + write_scope_index_cache(py, cache_location(&resolved)?, fingerprint.clone(), &index)?; + } + Ok(index) +} diff --git a/crates/sqlbuild-python-compile/src/scopes/main/mod.rs b/crates/sqlbuild-python-compile/src/scopes/main/mod.rs new file mode 100644 index 0000000000..a1a6c799e9 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/scopes/main/mod.rs @@ -0,0 +1,2 @@ +pub mod load_or_build_scope_index; +pub mod register_scope_loading; diff --git a/crates/sqlbuild-python-compile/src/scopes/main/register_scope_loading.rs b/crates/sqlbuild-python-compile/src/scopes/main/register_scope_loading.rs new file mode 100644 index 0000000000..167e09cae3 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/scopes/main/register_scope_loading.rs @@ -0,0 +1,10 @@ +//! Register the scope loading functions and classes. + +use crate::scopes::main::load_or_build_scope_index::load_or_build_scope_index; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the scope loading functions and classes. +pub fn register_scope_loading(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_or_build_scope_index, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/scopes/mod.rs b/crates/sqlbuild-python-compile/src/scopes/mod.rs new file mode 100644 index 0000000000..dc3d0a838f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/scopes/mod.rs @@ -0,0 +1,4 @@ +//! Offline scope facts from the persistent cache, a compile-input build or the tolerant index. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/mod.rs new file mode 100644 index 0000000000..a7f323c1b6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod recovery; +pub(crate) mod session; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/completion.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/completion.rs index e39ee991e6..4700f61322 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/completion.rs @@ -8,15 +8,17 @@ use sqlbuild_analysis::semantic_checks::models::{ CompletedDiagnostic, CompletionModel, CompletionRequest, SemanticDiagnostic, SemanticLocation, }; -use crate::bindings::_helpers::semantic_checks::failures::semantic_error; -use crate::bindings::_helpers::semantic_checks::metadata::{ShapeRows, shape_rows}; -use crate::bindings::_helpers::semantic_checks::session_facts::{ - column_names, session_model_facts, -}; -use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; -use crate::bindings::_helpers::semantic_checks::type_recovery::{ - StageShared, diagnostic_ids, id_of, items, lineage_outputs, -}; +use crate::semantic_checks::_helpers::recovery::failures::semantic_error; +use crate::semantic_checks::_helpers::recovery::type_recovery::StageShared; +use crate::semantic_checks::_helpers::recovery::type_recovery::diagnostic_ids; +use crate::semantic_checks::_helpers::recovery::type_recovery::id_of; +use crate::semantic_checks::_helpers::recovery::type_recovery::items; +use crate::semantic_checks::_helpers::recovery::type_recovery::lineage_outputs; +use crate::semantic_checks::_helpers::session::metadata::ShapeRows; +use crate::semantic_checks::_helpers::session::metadata::shape_rows; +use crate::semantic_checks::_helpers::session::session_facts::column_names; +use crate::semantic_checks::_helpers::session::session_facts::session_model_facts; +use crate::semantic_checks::_helpers::session::stage_objects::StageClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/failures.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/failures.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/failures.rs diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/help_catalogue.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/help_catalogue.rs new file mode 100644 index 0000000000..065bb61cc4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/help_catalogue.rs @@ -0,0 +1,10 @@ +//! The semantic diagnostic help catalogue, for Python's tests and corpus tooling. + +use pyo3::pyfunction; +use sqlbuild_analysis::semantic_checks::main::semantic_help_catalogue::semantic_help_catalogue as catalogue; + +/// Each semantic diagnostic code and its help. +#[pyfunction] +pub(crate) fn semantic_help_catalogue() -> Vec<(&'static str, &'static str)> { + catalogue().to_vec() +} diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/mod.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/mod.rs new file mode 100644 index 0000000000..0e2b5abda4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod completion; +pub(crate) mod failures; +pub(crate) mod help_catalogue; +pub(crate) mod resource_sql; +pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs index 4ff5851ff8..618d29e54a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs @@ -2,11 +2,9 @@ use std::sync::Arc; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ AuditSqlInput, HookModelInput, HookSqlInput, ResourceSpan, ResourceSqlDiagnostic, @@ -371,7 +369,3 @@ fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult } }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resource_sql_diagnostics_of, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/type_recovery.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/type_recovery.rs index cf87c9b8d0..64e3366a15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/type_recovery.rs @@ -16,13 +16,13 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_checks::types::RevisedBinding; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::semantic_checks::constants::SCHEMA_REVALIDATIONS; -use crate::bindings::_helpers::semantic_checks::failures::{internal_error, semantic_error}; -use crate::bindings::_helpers::semantic_checks::inputs::revalidation_request; -use crate::bindings::_helpers::semantic_checks::session_facts::{ - column_names, session_model_facts, -}; -use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::semantic_checks::_helpers::recovery::failures::internal_error; +use crate::semantic_checks::_helpers::recovery::failures::semantic_error; +use crate::semantic_checks::_helpers::session::inputs::revalidation_request; +use crate::semantic_checks::_helpers::session::session_facts::column_names; +use crate::semantic_checks::_helpers::session::session_facts::session_model_facts; +use crate::semantic_checks::_helpers::session::stage_objects::StageClasses; +use crate::semantic_checks::constants::SCHEMA_REVALIDATIONS; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/inputs.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/inputs.rs index aa627d60a7..9415645e29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/inputs.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use crate::bindings::_helpers::semantic_checks::resource_sql::resource_sql_diagnostics_of; +use crate::semantic_checks::_helpers::recovery::resource_sql::resource_sql_diagnostics_of; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs index ae16dd8d12..73016ea570 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs @@ -13,12 +13,12 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; -use crate::bindings::_helpers::semantic_checks::failures::semantic_error; -use crate::bindings::_helpers::semantic_checks::inputs::{ - column_names, resource_sql_diagnostics, sql_test_columns, -}; -use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; -use crate::bindings::_helpers::semantic_checks::type_recovery::items; +use crate::semantic_checks::_helpers::recovery::failures::semantic_error; +use crate::semantic_checks::_helpers::recovery::type_recovery::items; +use crate::semantic_checks::_helpers::session::inputs::column_names; +use crate::semantic_checks::_helpers::session::inputs::resource_sql_diagnostics; +use crate::semantic_checks::_helpers::session::inputs::sql_test_columns; +use crate::semantic_checks::_helpers::session::stage_objects::StageClasses; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_project::models::ModelMetadataTexts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/mod.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/mod.rs new file mode 100644 index 0000000000..f72ed55d74 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod inputs; +pub(crate) mod metadata; +pub(crate) mod session_facts; +pub(crate) mod stage; +pub(crate) mod stage_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/session_facts.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/session_facts.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage.rs similarity index 52% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage.rs index ed96fc30a8..85d0754c47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage.rs @@ -1,18 +1,15 @@ //! The native semantic completion stage: type recovery, metadata checks, then completion. +use crate::semantic_checks::main::completed_semantic_project::completed_semantic_project; +use crate::semantic_checks::models::SemanticCompletion; use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::rules::early_builtin_rules::start_early_rules; -use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; -use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; -use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; -use crate::bindings::_helpers::semantic_checks::type_recovery::{ - StageShared, items, recovered_types, -}; +use crate::semantic_checks::_helpers::session::metadata::metadata_diagnostics; +use crate::semantic_checks::_helpers::session::stage_objects::StageClasses; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -26,7 +23,7 @@ const NO_CATALOG_MESSAGE: &str = /// The completed project from keyword arguments; an internal failure raises `NativeCompilerError`. #[pyfunction] #[pyo3(signature = (**request))] -fn complete_semantic_project<'py>( +pub(crate) fn complete_semantic_project<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("complete_semantic_project", request)?; @@ -39,62 +36,10 @@ fn complete_semantic_project<'py>( }) } -/// The semantic completion stage's inputs. -pub(crate) struct SemanticCompletion<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) profile: Bound<'py, PyAny>, - pub(crate) binding_results: Bound<'py, PyAny>, - pub(crate) resource_sql_analysis: bool, - pub(crate) session: Option>, -} - -/// The completed project; an internal failure raises `NativeCompilerError`. -pub(crate) fn completed_semantic_project<'py>( - stage: SemanticCompletion<'py>, -) -> PyResult> { - let SemanticCompletion { - project, - profile, - binding_results, - resource_sql_analysis, - session, - } = stage; - let session: Option> = match session { - Some(session) if !session.is_none() => Some(session.extract()?), - _ => None, - }; - let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; - let classes = StageClasses::new(project.py())?; - let shared = StageShared { - catalog: &catalog.inner, - session: session.as_deref(), - session_models: session_fact_models(&project, session.as_deref())?, - }; - let recovered: Bound<'py, PyAny> = - recovered_types(&classes, &project, &binding_results, &shared)?; - start_early_rules(&recovered)?; - let metadata: Vec> = metadata_diagnostics( - &classes, - (&recovered, &profile), - resource_sql_analysis, - &catalog.inner, - )?; - let mut diagnostics: Vec> = items(&native_field(&recovered, "diagnostics")?)?; - diagnostics.extend(metadata); - let combined: Bound<'py, PyAny> = classes.project_with( - &recovered, - &[( - "diagnostics", - &PyTuple::new(project.py(), diagnostics)?.into_any(), - )], - )?; - completed_diagnostics(&classes, &combined, &shared) -} - /// The metadata and resource SQL diagnostics alone, as the stage appends them. #[pyfunction] #[pyo3(signature = (*, project, profile, resource_sql_analysis))] -fn semantic_metadata_diagnostics<'py>( +pub(crate) fn semantic_metadata_diagnostics<'py>( project: &Bound<'py, PyAny>, profile: &Bound<'py, PyAny>, resource_sql_analysis: bool, @@ -110,7 +55,9 @@ fn semantic_metadata_diagnostics<'py>( PyTuple::new(project.py(), diagnostics) } -fn project_catalog<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn project_catalog<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { let binding_catalog: Bound<'py, PyAny> = native_field(project, "binding_catalog")?; if binding_catalog.is_none() { return Err(native_compiler_error(NO_CATALOG_MESSAGE)); @@ -119,7 +66,7 @@ fn project_catalog<'py>(project: &Bound<'py, PyAny>) -> PyResult, session: Option<&NativeModelAnalysisSession>, ) -> PyResult> { @@ -141,8 +88,3 @@ fn session_fact_models( } Ok(models) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(complete_semantic_project, module)?)?; - module.add_function(wrap_pyfunction!(semantic_metadata_diagnostics, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage_objects.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage_objects.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/constants.rs b/crates/sqlbuild-python-compile/src/semantic_checks/constants.rs similarity index 69% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/constants.rs rename to crates/sqlbuild-python-compile/src/semantic_checks/constants.rs index a134a86130..aeeb8fa303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/constants.rs +++ b/crates/sqlbuild-python-compile/src/semantic_checks/constants.rs @@ -1,4 +1,4 @@ -//! Stage-work names the semantic completion stage counts. +//! Public constants of semantic checks. /// Single-model schema revalidations run after type recovery. pub(crate) const SCHEMA_REVALIDATIONS: &str = "schema_revalidations"; diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/main/completed_semantic_project.rs b/crates/sqlbuild-python-compile/src/semantic_checks/main/completed_semantic_project.rs new file mode 100644 index 0000000000..c800a9898f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/main/completed_semantic_project.rs @@ -0,0 +1,60 @@ +//! The completed project; an internal failure raises `NativeCompilerError`. + +use crate::rules::main::checking::start_early_rules::start_early_rules; +use crate::semantic_checks::_helpers::recovery::completion::completed_diagnostics; +use crate::semantic_checks::_helpers::recovery::type_recovery::StageShared; +use crate::semantic_checks::_helpers::recovery::type_recovery::items; +use crate::semantic_checks::_helpers::recovery::type_recovery::recovered_types; +use crate::semantic_checks::_helpers::session::metadata::metadata_diagnostics; +use crate::semantic_checks::_helpers::session::stage::project_catalog; +use crate::semantic_checks::_helpers::session::stage::session_fact_models; +use crate::semantic_checks::_helpers::session::stage_objects::StageClasses; +use crate::semantic_checks::models::SemanticCompletion; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; + +/// The completed project; an internal failure raises `NativeCompilerError`. +pub fn completed_semantic_project<'py>( + stage: SemanticCompletion<'py>, +) -> PyResult> { + let SemanticCompletion { + project, + profile, + binding_results, + resource_sql_analysis, + session, + } = stage; + let session: Option> = match session { + Some(session) if !session.is_none() => Some(session.extract()?), + _ => None, + }; + let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; + let classes = StageClasses::new(project.py())?; + let shared = StageShared { + catalog: &catalog.inner, + session: session.as_deref(), + session_models: session_fact_models(&project, session.as_deref())?, + }; + let recovered: Bound<'py, PyAny> = + recovered_types(&classes, &project, &binding_results, &shared)?; + start_early_rules(&recovered)?; + let metadata: Vec> = metadata_diagnostics( + &classes, + (&recovered, &profile), + resource_sql_analysis, + &catalog.inner, + )?; + let mut diagnostics: Vec> = items(&native_field(&recovered, "diagnostics")?)?; + diagnostics.extend(metadata); + let combined: Bound<'py, PyAny> = classes.project_with( + &recovered, + &[( + "diagnostics", + &PyTuple::new(project.py(), diagnostics)?.into_any(), + )], + )?; + completed_diagnostics(&classes, &combined, &shared) +} diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/main/mod.rs b/crates/sqlbuild-python-compile/src/semantic_checks/main/mod.rs new file mode 100644 index 0000000000..be1c682321 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/main/mod.rs @@ -0,0 +1,4 @@ +pub mod completed_semantic_project; +pub mod register_help_catalogue; +pub mod register_resource_sql; +pub mod register_stage; diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/main/register_help_catalogue.rs b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_help_catalogue.rs new file mode 100644 index 0000000000..931b509249 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_help_catalogue.rs @@ -0,0 +1,10 @@ +//! Register the help catalogue functions and classes. + +use crate::semantic_checks::_helpers::recovery::help_catalogue::semantic_help_catalogue; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the help catalogue functions and classes. +pub fn register_help_catalogue(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_help_catalogue, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/main/register_resource_sql.rs b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_resource_sql.rs new file mode 100644 index 0000000000..0f95217ca6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_resource_sql.rs @@ -0,0 +1,10 @@ +//! Register the resource sql functions and classes. + +use crate::semantic_checks::_helpers::recovery::resource_sql::resource_sql_diagnostics_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the resource sql functions and classes. +pub fn register_resource_sql(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resource_sql_diagnostics_of, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/main/register_stage.rs b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_stage.rs new file mode 100644 index 0000000000..b399a5b26d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/main/register_stage.rs @@ -0,0 +1,12 @@ +//! Register the stage functions and classes. + +use crate::semantic_checks::_helpers::session::stage::complete_semantic_project; +use crate::semantic_checks::_helpers::session::stage::semantic_metadata_diagnostics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the stage functions and classes. +pub fn register_stage(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(complete_semantic_project, module)?)?; + module.add_function(wrap_pyfunction!(semantic_metadata_diagnostics, module)?) +} diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/mod.rs b/crates/sqlbuild-python-compile/src/semantic_checks/mod.rs new file mode 100644 index 0000000000..a553a660ee --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/mod.rs @@ -0,0 +1,6 @@ +//! Semantic completion: diagnostic recovery, explanations, opt-out rejection and the help catalogue. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-compile/src/semantic_checks/models.rs b/crates/sqlbuild-python-compile/src/semantic_checks/models.rs new file mode 100644 index 0000000000..249a19c63a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/semantic_checks/models.rs @@ -0,0 +1,14 @@ +//! Public records of semantic checks. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// The semantic completion stage's inputs. +#[derive(Debug)] +pub struct SemanticCompletion<'py> { + pub project: Bound<'py, PyAny>, + pub profile: Bound<'py, PyAny>, + pub binding_results: Bound<'py, PyAny>, + pub resource_sql_analysis: bool, + pub session: Option>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/assembly.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs rename to crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/assembly.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs rename to crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs index 0fba97583d..738a54bc47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs @@ -1,14 +1,11 @@ //! Compiled SQL tests, their keyed P013 diagnostics and assembly failures, built natively. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiler::models::{AssembledSqlTestFacts, SqlTestHelperDiagnostic}; -use crate::bindings::_helpers::sql_test_glue::assembly::{ - AssemblyRequest, AssemblyRow, assemble_compiled_sql_tests, -}; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use crate::sql_test_glue::_helpers::assembly::AssemblyRow; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -49,7 +46,7 @@ const MODEL_TEST_FIELDS: [&str; 5] = [ type Fields<'py> = Vec<(&'static str, Bound<'py, PyAny>)>; /// The Python classes a compiled test is built from. -struct Classes<'py> { +pub(crate) struct Classes<'py> { resource_type: Bound<'py, PyAny>, direct_input: Bound<'py, PyAny>, assembly: Bound<'py, PyAny>, @@ -61,7 +58,7 @@ struct Classes<'py> { } impl<'py> Classes<'py> { - fn new(py: Python<'py>) -> PyResult { + pub(crate) fn new(py: Python<'py>) -> PyResult { Ok(Self { direct_input: record_class(py, RecordId::CompileDirectLogicSqlTestInputPayload)?, resource_type: record_class(py, RecordId::CompiledResourceType)?, @@ -280,7 +277,7 @@ fn compiled_test<'py>( Ok((test, PyTuple::new(py, keyed)?)) } -fn assembly<'py>( +pub(crate) fn assembly<'py>( classes: &Classes<'py>, test_input: &Bound<'py, PyAny>, (facts, failure): AssemblyRow, @@ -306,42 +303,3 @@ fn assembly<'py>( ], ) } - -/// One assembly per test input, with the error its assembly raises, if any. -#[pyfunction] -#[pyo3(signature = (*, inputs, sql_analysis_dialect=None))] -pub(crate) fn assemble_native_sql_tests<'py>( - py: Python<'py>, - inputs: &Bound<'py, PyAny>, - sql_analysis_dialect: Option, -) -> PyResult> { - let test_inputs: Vec> = native_field(inputs, "test_inputs")?.extract()?; - let request = AssemblyRequest { - model_inputs: native_field(inputs, "model_inputs")?.extract()?, - test_inputs: test_inputs.clone(), - lexical_syntax: native_field( - &native_field(inputs, "sql_lexical_syntax")?, - "native_mapping", - )? - .extract()?, - sql_analysis_dialect, - }; - let rows: Vec = assemble_compiled_sql_tests(py, request)?; - if rows.len() != test_inputs.len() { - return Err(compiler_error( - "native SQL test assembly returned an invalid batch response", - )); - } - let classes: Classes<'py> = Classes::new(py)?; - let assemblies: Vec> = test_inputs - .iter() - .zip(rows) - .map(|(test_input, row)| assembly(&classes, test_input, row)) - .collect::>()?; - PyTuple::new(py, assemblies) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_native_sql_tests, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/mod.rs new file mode 100644 index 0000000000..1b12f3658c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod assembly; +pub(crate) mod compiled_tests; diff --git a/crates/sqlbuild-python-compile/src/sql_test_glue/main/assemble_native_sql_tests.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/main/assemble_native_sql_tests.rs new file mode 100644 index 0000000000..33e45b2e38 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/main/assemble_native_sql_tests.rs @@ -0,0 +1,46 @@ +//! One assembly per test input, with the error its assembly raises, if any. + +use crate::sql_test_glue::_helpers::assembly::AssemblyRequest; +use crate::sql_test_glue::_helpers::assembly::AssemblyRow; +use crate::sql_test_glue::_helpers::assembly::assemble_compiled_sql_tests; +use crate::sql_test_glue::_helpers::compiled_tests::Classes; +use crate::sql_test_glue::_helpers::compiled_tests::assembly; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// One assembly per test input, with the error its assembly raises, if any. +#[pyfunction] +#[pyo3(signature = (*, inputs, sql_analysis_dialect=None))] +pub fn assemble_native_sql_tests<'py>( + py: Python<'py>, + inputs: &Bound<'py, PyAny>, + sql_analysis_dialect: Option, +) -> PyResult> { + let test_inputs: Vec> = native_field(inputs, "test_inputs")?.extract()?; + let request = AssemblyRequest { + model_inputs: native_field(inputs, "model_inputs")?.extract()?, + test_inputs: test_inputs.clone(), + lexical_syntax: native_field( + &native_field(inputs, "sql_lexical_syntax")?, + "native_mapping", + )? + .extract()?, + sql_analysis_dialect, + }; + let rows: Vec = assemble_compiled_sql_tests(py, request)?; + if rows.len() != test_inputs.len() { + return Err(compiler_error( + "native SQL test assembly returned an invalid batch response", + )); + } + let classes: Classes<'py> = Classes::new(py)?; + let assemblies: Vec> = test_inputs + .iter() + .zip(rows) + .map(|(test_input, row)| assembly(&classes, test_input, row)) + .collect::>()?; + PyTuple::new(py, assemblies) +} diff --git a/crates/sqlbuild-python-compile/src/sql_test_glue/main/mod.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/main/mod.rs new file mode 100644 index 0000000000..74dd90ed32 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/main/mod.rs @@ -0,0 +1,2 @@ +pub mod assemble_native_sql_tests; +pub mod register_compiled_tests; diff --git a/crates/sqlbuild-python-compile/src/sql_test_glue/main/register_compiled_tests.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/main/register_compiled_tests.rs new file mode 100644 index 0000000000..5a1f9cca98 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/main/register_compiled_tests.rs @@ -0,0 +1,11 @@ +//! Register the compiled tests functions and classes. + +use crate::sql_test_glue::main::assemble_native_sql_tests::assemble_native_sql_tests; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the compiled tests functions and classes. +pub fn register_compiled_tests(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_native_sql_tests, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-compile/src/sql_test_glue/mod.rs b/crates/sqlbuild-python-compile/src/sql_test_glue/mod.rs new file mode 100644 index 0000000000..9d6c4c5b1b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/sql_test_glue/mod.rs @@ -0,0 +1,4 @@ +//! Compiled SQL tests and their diagnostics, assembled from compile inputs. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 32d52437ba..9c19bdbcf2 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -22,11 +22,9 @@ pyo3.workspace = true rayon.workspace = true rusqlite.workspace = true rustc-hash.workspace = true -serde.workspace = true serde_json.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true -sqlbuild-attachments.workspace = true sqlbuild-cache.workspace = true sqlbuild-cli.workspace = true sqlbuild-config.workspace = true @@ -35,20 +33,18 @@ sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true sqlbuild-orchestration.workspace = true -sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true sqlbuild-python-project.workspace = true sqlbuild-python-planner.workspace = true +sqlbuild-python-compile.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true -sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true -sqlclean.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs deleted file mode 100644 index e60427317e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! A hand-built model's binding diagnostics at their authored locations. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; -use sqlbuild_analysis::binding_locations::models::{ - AuthoredSpan, BindingSource, LocatedDiagnostic, -}; -use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; -use sqlbuild_python_base::positional_record; - -use sqlbuild_python_project::analysis::models::Positional; - -positional_record! { - into; - /// One located diagnostic as Python reads it, its span `(line, column, end line, end column)`. - pub(crate) struct LocatedRow { - pub(crate) code: String, - pub(crate) message: String, - pub(crate) severity: String, - pub(crate) line: Option, - pub(crate) column: Option, - pub(crate) span: Option>, - } -} - -/// The expanded SQL and the expansion passes Python sends for a located model. -pub(crate) type ExpansionRow = Option<(String, Vec>>)>; - -/// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. -#[pyfunction] -pub(crate) fn locate_model_binding_diagnostics( - rows: Vec>, - texts: (String, String, String), - expansion: ExpansionRow, -) -> PyResult> { - let (authored, query, cleaned) = texts; - let rows: Vec = Positional::unwrap_all(rows); - let expansion: Option<(String, Vec>)> = expansion.map(|(expanded, passes)| { - ( - expanded, - passes.into_iter().map(Positional::unwrap_all).collect(), - ) - }); - let source: BindingSource<'_> = BindingSource { - authored: &authored, - query: &query, - cleaned: &cleaned, - expansion: expansion - .as_ref() - .map(|(expanded, passes)| (expanded.as_str(), passes.as_slice())), - }; - let located: Vec = - locate_binding_diagnostics(&rows, &source).map_err(PyValueError::new_err)?; - Ok(located.into_iter().map(located_row).collect()) -} - -fn located_row(item: LocatedDiagnostic) -> LocatedRow { - LocatedRow { - code: item.code, - message: item.message, - severity: item.severity, - line: item.line, - column: item.column, - span: item.location.map(Positional), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(locate_model_binding_diagnostics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs deleted file mode 100644 index 322bfa51a3..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Validate, fingerprint and analyze SQL queries for the Python compiler. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog; -use sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references; - -#[pyfunction] -fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) - }) - .map_err(value_error) -} - -/// A warehouse type name's JSON type tree, None without a parser, `ValueError` on a panic. -#[pyfunction] -fn parse_warehouse_type(py: Python<'_>, spelling: &str, dialect: &str) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_analysis::warehouse_types::main::parse_warehouse_type::parse_warehouse_type( - spelling, dialect, - ) - .map(|tree| tree.as_ref().map(serde_json::Value::to_string)) - }) - .map_err(value_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - register_binding_catalog(module)?; - crate::bindings::_helpers::analysis::required_columns::register(module)?; - register_explicit_references(module)?; - crate::bindings::_helpers::analysis::binding_locations::register(module)?; - module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; - module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs deleted file mode 100644 index ed84f097b7..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Extract, plan and render SQL tests natively. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::types::CompilerDetach; - -#[pyfunction] -fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json) - }) - .map_err(value_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index 1131fb370c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,5 +1 @@ -pub(crate) mod getters; -pub(crate) mod model_analyses; -pub(crate) mod oracle; -pub(crate) mod outcome_objects; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 54c1e99891..bbcb8f25c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::analysis_session::{oracle, outcome_objects}; +use sqlbuild_python_compile::analysis_session::main::register_oracle::register_oracle; +use sqlbuild_python_compile::analysis_session::main::register_outcome_objects::register_outcome_objects; use sqlbuild_python_project::analysis_session::main::register_fixture_columns::register_fixture_columns; use sqlbuild_python_project::analysis_session::main::register_query_columns::register_query_columns; use sqlbuild_python_project::analysis_session::main::register_session::register_session; @@ -13,8 +14,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_session(module)?; register_fixture_columns(module)?; module.add_class::()?; - outcome_objects::register(module)?; + register_outcome_objects(module)?; register_query_columns(module)?; register_stage_functions(module)?; - oracle::register(module) + register_oracle(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index f7224beb48..09106652cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressRepo use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::result_output::documents::{ @@ -39,6 +38,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_planner::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs index ab6f2de2b5..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs @@ -1,4 +1 @@ -mod identity; -mod measurement; pub(crate) mod registration; -mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs index 42c46345ff..cde9306b3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs @@ -1,9 +1,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_compile::auditing::main::register_identity::register_identity; +use sqlbuild_python_compile::auditing::main::register_measurement::register_measurement; +use sqlbuild_python_compile::auditing::main::register_schema_audits::register_schema_audits; use sqlbuild_python_project::auditing::main::register_render::register_render; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::auditing::identity::register(module)?; - crate::bindings::_helpers::auditing::measurement::register(module)?; + register_identity(module)?; + register_measurement(module)?; register_render(module)?; - crate::bindings::_helpers::auditing::schema_audits::register(module) + register_schema_audits(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 92d599550d..f23f49ab87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,9 +19,6 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; -use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, selected_target_settings, -}; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; @@ -30,6 +27,8 @@ use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::diagnostics::models::BuildPhaseTimingTracker; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 05ee0cc822..c16a49fd10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -19,10 +19,6 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::check::check_results::check_result_text; -use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; -use crate::bindings::_helpers::compile_outputs::runtime_target::{ - write_python_check_runtime_target, write_runtime_target, -}; use crate::bindings::_helpers::cost_report::degradation::{ finalize_build_cost as finalize_cost, render_build_cost, }; @@ -44,6 +40,9 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; +use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; +use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; const SUCCESS_STATUS: &str = "success"; const CHECK_KIND: &str = "check"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 4239e54c9f..911e4772e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::policies::enforce_model_execution_limit_binding; use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 3eb750f15f..82138f625b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -26,8 +26,6 @@ use crate::bindings::_helpers::check::check_core::{ }; use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; -use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; -use crate::bindings::_helpers::compile_driver::targets::project_target_settings; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -49,6 +47,8 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 0596037169..68adf86f23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -25,6 +24,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 7e7b13b14b..f0985d6d82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -12,11 +12,6 @@ use sqlbuild_core::build_command::models::PythonNodeRow; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; -use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, project_target_settings, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -32,6 +27,10 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 462324401a..2ff106622a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::manifest::builder::build_manifest; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -26,6 +25,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::manifest::main::build_manifest::build_manifest; use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; @@ -232,7 +232,7 @@ pub(crate) fn write_compile_target( } else { none(py) }; - crate::bindings::_helpers::compile_command::target_writer::write_compile_target( + sqlbuild_python_compile::compile_command::main::artifacts::write_compile_target::write_compile_target( py, Some(keywords( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs index 16a825131d..2ec8654217 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -12,14 +12,14 @@ use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; use crate::bindings::_helpers::change_selection::definition_snapshot::definition_snapshot; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::run_compile_only_pipeline::run_compile_only_pipeline; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; /// The extension of the file beside the snapshot that holds notes for the run's output. pub(crate) const NOTES_EXTENSION: &str = "notes"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 2fee2b0bf3..b7441686ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -24,10 +24,10 @@ use crate::bindings::_helpers::change_selection::base_cache::{ }; use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; use crate::bindings::_helpers::change_selection::environment_nodes::with_base_dependencies; -use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; use sqlbuild_python_base::planner_identity::constants::COMPACT; use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; +use sqlbuild_python_compile::compile_command::main::invocation::interpreter_values::interpreter_values; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 8f084c260f..72fea2661e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -3,13 +3,7 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; -use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; -use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, project_target_settings, -}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ @@ -29,6 +23,11 @@ use sqlbuild_python_base::native_host::main::connection_failure::connection_fail use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; +use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; @@ -48,7 +47,6 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -56,6 +54,7 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index efc62cd85c..98b2b61bb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -1,9 +1,8 @@ //! Python check planning: run-state recording, dependency relations and read-side preflight. -use crate::bindings::_helpers::compile_driver::targets::TargetSettingsValues; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; -use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -11,6 +10,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; use sqlbuild_python_project::planner_phases::main::object::object; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f29dfb6eaa..67b92cc3d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -8,7 +8,6 @@ use crate::bindings::_helpers::check::check_command::run_check_command; use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; -use crate::bindings::_helpers::compile_command::reuse_attempt::run_compile; use crate::bindings::_helpers::contract_command::command::run_contract_command; use crate::bindings::_helpers::cost_command::command::run_cost_command; use crate::bindings::_helpers::dbt_command::command::run_dbt_command; @@ -55,6 +54,7 @@ use sqlbuild_core::format_command::main::format_path_selectors::format_path_sele use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_compile::compile_command::main::invocation::run_compile::run_compile; use sqlbuild_python_front::discovery::models::MissingSecretsScope; use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index b37d0aca7b..6921f7d3d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -32,8 +32,8 @@ use crate::bindings::_helpers::cli::termination::{ CommandTerminated, TERMINATED_MESSAGE, TerminationHandler, handle_sigterm, interrupting_signal, signal_exit_code, terminated, terminated_exit, }; -use crate::bindings::_helpers::compile_command::constants::COMPILE_COMMAND_NAME; -use crate::bindings::_helpers::compile_command::report::format_compile_error_json; +use sqlbuild_python_compile::compile_command::constants::COMPILE_COMMAND_NAME; +use sqlbuild_python_compile::compile_command::main::invocation::format_compile_error_json::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use sqlbuild_python_base::native_host::models::InterruptHandler; @@ -47,7 +47,7 @@ use crate::bindings::_helpers::recall::session::{ InvocationRecall, effective_selection, finish_invocation_recall, start_invocation_recall, }; use crate::bindings::_helpers::recall::sinks::{RecallLifecycleSink, publish_lifecycle_event}; -use crate::bindings::_helpers::rules::constants::DEFAULT_RULES_NOTICE_ATTR; +use sqlbuild_python_compile::rules::constants::DEFAULT_RULES_NOTICE_ATTR; use crate::bindings::_helpers::workspace_commands::commands::maintain_sqlbuild_skills; use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 4809d49ef6..461b7fb13a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::clone::clone_execution::{ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, timed_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 067fdb3917..68a3017237 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -4,12 +4,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_planner::planner_entries::main::plans::timed_clone_planning::timed_clone_planning; use sqlbuild_python_planner::planner_entries::types::PhaseTimings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 8590c9746b..305a7be971 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -6,7 +6,6 @@ use pyo3::types::PyBool; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -15,6 +14,7 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::models::SecretPolicy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index 61d9404905..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -1,7 +1 @@ -pub(crate) mod declaration_scope; -pub(crate) mod model_config; -pub(crate) mod model_inputs; pub(crate) mod registration; -pub(crate) mod sql_expansion_context; -mod storage_policies; -pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index 4bf39be9b7..fb466dd246 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -1,11 +1,13 @@ //! Register the compile-input attachment functions. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::compile_attachment::{ - declaration_scope, model_config, model_inputs, sql_expansion_context, target, -}; +use sqlbuild_python_compile::compile_attachment::main::build_compile_target_context_function::build_compile_target_context_function; +use sqlbuild_python_compile::compile_attachment::main::build_declaration_scope_function::build_declaration_scope_function; +use sqlbuild_python_compile::compile_attachment::main::build_model_inputs_function::build_model_inputs_function; +use sqlbuild_python_compile::compile_attachment::main::build_sql_expansion_context_function::build_sql_expansion_context_function; +use sqlbuild_python_compile::compile_attachment::main::native_model_config_session_function::native_model_config_session_function; +use sqlbuild_python_compile::compile_attachment::main::rebind_declaration_scope_function::rebind_declaration_scope_function; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config_function::build_effective_connection_config_function; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_function::build_effective_connection_function; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config_function::build_effective_runtime_config_function; @@ -16,10 +18,7 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_vars_functi use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles_function::resolve_cursor_input_roles_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - sql_expansion_context::build_sql_expansion_context, - module - )?)?; + module.add_function(build_sql_expansion_context_function(module)?)?; module.add_function(build_effective_connection_function(module)?)?; module.add_function(build_effective_settings_function(module)?)?; module.add_function(build_effective_vars_function(module)?)?; @@ -27,23 +26,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(build_effective_connection_config_function(module)?)?; module.add_function(build_effective_runtime_config_function(module)?)?; module.add_function(build_effective_settings_config_function(module)?)?; - module.add_function(wrap_pyfunction!( - model_config::native_model_config_session, - module - )?)?; - module.add_function(wrap_pyfunction!(model_inputs::build_model_inputs, module)?)?; - module.add_function(wrap_pyfunction!( - target::build_compile_target_context, - module - )?)?; - module.add_function(wrap_pyfunction!( - declaration_scope::build_declaration_scope, - module - )?)?; - module.add_function(wrap_pyfunction!( - declaration_scope::rebind_declaration_scope, - module - )?)?; + module.add_function(native_model_config_session_function(module)?)?; + module.add_function(build_model_inputs_function(module)?)?; + module.add_function(build_compile_target_context_function(module)?)?; + module.add_function(build_declaration_scope_function(module)?)?; + module.add_function(rebind_declaration_scope_function(module)?)?; module.add_function(resolve_cursor_input_roles_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs deleted file mode 100644 index 1fc3984b85..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! Compiled artifacts queued for one native write, keeping unchanged files when asked. - -use std::path::PathBuf; - -use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; -use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; - -use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, artifact_written, artifact_written_at, -}; -use crate::bindings::_helpers::compile_outputs::artifacts::{ - BackgroundArtifactWriter, write_compiled_artifacts, -}; - -const PHYSICAL_WRITE_PHASE: &str = "physical_write_ms"; - -/// The artifacts of one write group, written together by `flush`. -pub(crate) struct ArtifactBatch<'a> { - check_existing: bool, - recorder: ArtifactRecorder<'a>, - files: Vec<(PathBuf, Vec)>, - /// Earlier groups, still being written in the background. - writer: Option, -} - -/// `contents` with platform line endings. -fn platform_lines(contents: Vec) -> Vec { - if cfg!(windows) { - contents - .iter() - .flat_map(|byte| { - if *byte == b'\n' { - vec![b'\r', b'\n'] - } else { - vec![*byte] - } - }) - .collect() - } else { - contents - } -} - -/// `sql` with trailing whitespace replaced by one newline. -fn sql_artifact(sql: &str) -> Vec { - let trimmed: &str = - sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); - format!("{trimmed}\n").into_bytes() -} - -impl<'a> ArtifactBatch<'a> { - pub(crate) fn new(check_existing: bool, recorder: ArtifactRecorder<'a>) -> Self { - Self { - check_existing, - recorder, - files: Vec::new(), - writer: None, - } - } - - /// Queue one artifact with platform line endings, noting its bytes for compile reuse. - pub(crate) fn queue(&mut self, path: &Bound<'_, PyAny>, contents: Vec) -> PyResult<()> { - let contents: Vec = platform_lines(contents); - artifact_written(self.recorder, path, &contents)?; - self.files.push((path.extract()?, contents)); - Ok(()) - } - - /// Queue `sql` with trailing whitespace replaced by one newline. - pub(crate) fn queue_sql(&mut self, path: &Bound<'_, PyAny>, sql: &str) -> PyResult<()> { - self.queue(path, sql_artifact(sql)) - } - - /// `queue_sql` at a native path. - pub(crate) fn queue_sql_at( - &mut self, - py: Python<'_>, - path: PathBuf, - sql: &str, - ) -> PyResult<()> { - let contents: Vec = platform_lines(sql_artifact(sql)); - artifact_written_at(py, self.recorder, &path, &contents)?; - self.files.push((path, contents)); - Ok(()) - } - - /// Write every queued artifact natively, after any background group, and empty the queue. - pub(crate) fn flush(&mut self, py: Python<'_>) -> PyResult<()> { - self.finish_pending(py)?; - if self.files.is_empty() { - return Ok(()); - } - let files: Vec<(PathBuf, Vec)> = std::mem::take(&mut self.files); - record_compile_timing(PHYSICAL_WRITE_PHASE, || { - write_compiled_artifacts(py, files, self.check_existing).map(|_| ()) - }) - } - - /// Write the queued artifacts in the background, after every earlier background group. - pub(crate) fn flush_in_background(&mut self, py: Python<'_>) -> PyResult<()> { - if self.files.is_empty() { - return Ok(()); - } - if self.writer.is_none() { - self.writer = BackgroundArtifactWriter::start(); - } - let Some(writer) = &self.writer else { - return self.flush(py); - }; - writer.send(std::mem::take(&mut self.files), self.check_existing); - Ok(()) - } - - /// `result`, except that a background group's earlier write error wins and interrupts stop it. - pub(crate) fn settle(&mut self, py: Python<'_>, result: PyResult) -> PyResult { - let error: PyErr = match result { - Ok(value) => return Ok(value), - Err(error) => error, - }; - let Some(writer) = self.writer.take() else { - return Err(error); - }; - if !error.is_instance_of::(py) { - writer.stop(py); - return Err(error); - } - writer.finish(py)?; - Err(error) - } - - /// Wait for the background groups, raising the first write error. - fn finish_pending(&mut self, py: Python<'_>) -> PyResult<()> { - let Some(writer) = self.writer.take() else { - return Ok(()); - }; - let elapsed = writer.finish(py)?; - add_compile_timing( - PHYSICAL_WRITE_PHASE, - u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX), - ); - Ok(()) - } -} - -impl Drop for ArtifactBatch<'_> { - fn drop(&mut self) { - if let Some(writer) = self.writer.take() { - Python::attach(|py| writer.stop(py)); - } - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index 53a8cc1556..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -1,14 +1 @@ -pub(crate) mod artifact_batch; -pub(crate) mod constants; -pub(crate) mod getters; -pub(crate) mod models; -pub(crate) mod prepared_artifacts; pub(crate) mod registration; -pub(crate) mod report; -pub(crate) mod reuse_attempt; -pub(crate) mod reuse_recording; -pub(crate) mod runtime_identity; -pub(crate) mod startup_inputs; -pub(crate) mod surrogates; -pub(crate) mod target_writer; -pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs deleted file mode 100644 index cee41aa6d2..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! The per-invocation compile reuse state carried from the reuse check into a full compile. - -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{IntoPyObjectExt, pyclass, pymethods}; - -use crate::bindings::_helpers::compile_command::constants::REUSE_ATTEMPT_FIELDS; -use sqlbuild_python_base::boundary::models::KeywordArguments; - -/// Result of checking for a reusable compile, carried into a full compile on a miss. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct CompileReuseAttempt { - /// `hit`, `miss` or `bypass`. - pub(crate) outcome: Py, - pub(crate) project_dir: Py, - pub(crate) started: Py, - pub(crate) check_ms: Py, - pub(crate) exit_code: Py, - pub(crate) entry_path: Py, - /// The natively checked attempt, recorded natively after a miss. - pub(crate) native: Py, - pub(crate) selected_target: Py, - pub(crate) json_output: Py, -} - -#[pymethods] -impl CompileReuseAttempt { - #[new] - #[pyo3(signature = (**options))] - fn new(py: Python<'_>, options: Option>) -> PyResult { - let fields: KeywordArguments<'_> = - KeywordArguments::read(py, "CompileReuseAttempt", options, REUSE_ATTEMPT_FIELDS)?; - let field = |name: &str| -> PyResult> { Ok(fields.value(name)?.unbind()) }; - Ok(Self { - outcome: field("outcome")?, - project_dir: field("project_dir")?, - started: field("started")?, - check_ms: if fields.passed("check_ms")? { - field("check_ms")? - } else { - 0_i64.into_py_any(py)? - }, - exit_code: field("exit_code")?, - entry_path: field("entry_path")?, - native: field("native")?, - selected_target: field("selected_target")?, - json_output: fields.value_or_false("json_output")?.unbind(), - }) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index 6b1d20cd31..9088d51e50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -2,20 +2,23 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; +use sqlbuild_python_compile::compile_command::models::CompileReuseAttempt; -use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_attempt, reuse_recording, runtime_identity, startup_inputs, - target_writer, -}; +use sqlbuild_python_compile::compile_command::main::artifacts::register_prepared_artifacts::register_prepared_artifacts; +use sqlbuild_python_compile::compile_command::main::artifacts::register_target_writer::register_target_writer; +use sqlbuild_python_compile::compile_command::main::invocation::register_report::register_report; +use sqlbuild_python_compile::compile_command::main::invocation::register_reuse_attempt::register_reuse_attempt; +use sqlbuild_python_compile::compile_command::main::invocation::register_reuse_recording::register_reuse_recording; +use sqlbuild_python_compile::compile_command::main::invocation::register_runtime_identity::register_runtime_identity; +use sqlbuild_python_compile::compile_command::main::invocation::register_startup_inputs::register_startup_inputs; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - report::register(module)?; - reuse_attempt::register(module)?; - reuse_recording::register(module)?; - runtime_identity::register(module)?; - startup_inputs::register(module)?; - target_writer::register(module)?; - prepared_artifacts::register(module) + register_report(module)?; + register_reuse_attempt(module)?; + register_reuse_recording(module)?; + register_runtime_identity(module)?; + register_startup_inputs(module)?; + register_target_writer(module)?; + register_prepared_artifacts(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs deleted file mode 100644 index 126919ed30..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ /dev/null @@ -1,646 +0,0 @@ -//! What one full compile printed and left in each artifact, and storing it for compile reuse. - -use sqlbuild_cache::project_reuse::types::ImageSections; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Mutex, PoisonError}; - -use pyo3::exceptions::{PyException, PyOSError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; - -use sqlbuild_cache::project_reuse::models::RecordedArtifact; -use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; -use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; - -use crate::bindings::_helpers::compile_command::constants::{ - BYTES_PER_MEBIBYTE, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_HIT_TIMING, - REUSE_LOGGER_NAME, REUSE_MISS_TIMING, REUSE_STORE_DONE_PREFIX, REUSE_STORE_NOTICE_BYTES, - REUSE_STORE_NOTICE_PATHS, REUSE_STORE_SKIPPED_PREFIX, -}; -use crate::bindings::_helpers::compile_command::reuse_attempt::{reuse_rules, write_launch_record}; -use crate::bindings::_helpers::compile_outputs::reuse::{ - ArtifactRow, CompileRecordPy, NativeReuseAttempt, SettingsRow, artifact_digest, -}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const MISS_OUTCOME: &str = "miss"; -const HIT_OUTCOME: &str = "hit"; -const BYPASS_OUTCOME: &str = "bypass"; -const STORED: &str = "stored"; -const SKIPPED: &str = "skipped"; - -/// Artifacts one compile wrote or kept, by absolute path, following staged moves. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CompileArtifactWrites { - artifacts: Mutex>, -} - -impl CompileArtifactWrites { - fn locked(&self) -> std::sync::MutexGuard<'_, HashMap> { - self.artifacts - .lock() - .unwrap_or_else(PoisonError::into_inner) - } - - fn record(&self, path: String, artifact: RecordedArtifact) { - let _ = self.locked().insert(path, artifact); - } - - fn move_one(&self, source: &str, destination: String) { - let mut artifacts = self.locked(); - if let Some(artifact) = artifacts.remove(source) { - let _ = artifacts.insert(destination, artifact); - } - } - - fn move_tree(&self, source: &str, destination: &str) { - let prefix: String = format!("{source}{}", std::path::MAIN_SEPARATOR); - let mut artifacts = self.locked(); - let moved: Vec<(String, RecordedArtifact)> = artifacts - .iter() - .filter(|(path, _)| path.starts_with(&prefix)) - .map(|(path, artifact)| { - ( - format!("{destination}{}", &path[source.len()..]), - artifact.clone(), - ) - }) - .collect(); - artifacts.retain(|path, _| !path.starts_with(&prefix)); - artifacts.extend(moved); - } -} - -impl CompileArtifactWrites { - /// Every recorded artifact's absolute path, with its digest when this compile wrote it. - pub(crate) fn written_digests(&self) -> HashMap> { - self.locked() - .iter() - .map(|(path, artifact)| { - let digest: Option = match artifact { - RecordedArtifact::Written { digest } => Some(digest.clone()), - RecordedArtifact::Kept { .. } => None, - }; - (path.clone(), digest) - }) - .collect() - } - - /// The Python-facing artifact rows, built natively for recording or the public view. - fn artifact_rows(&self) -> HashMap { - self.locked() - .iter() - .map(|(path, artifact)| { - let row = match artifact { - RecordedArtifact::Written { digest } => (Some(digest.clone()), None, None), - RecordedArtifact::Kept { size, mtime_ns } => { - (None, Some(*size), Some(*mtime_ns)) - } - }; - (path.clone(), row) - }) - .collect() - } -} - -#[pymethods] -impl CompileArtifactWrites { - /// Every recorded artifact by absolute path, as `(digest, size, mtime_ns)`. - #[getter] - pub(crate) fn artifacts<'py>(&self, py: Python<'py>) -> PyResult> { - self.artifact_rows().into_pyobject(py) - } -} - -/// Where a compile notes the artifacts it writes; `None` outside a recorded compile. -pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; - -/// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. -fn absolute(path: &Bound<'_, PyAny>) -> PyResult { - absolute_path(path.py(), &path.extract::()?) -} - -/// `os.path.abspath(path)` of a native path. -fn absolute_path(py: Python<'_>, path: &Path) -> PyResult { - if cfg!(unix) - && let Some(found) = native_absolute(path) - { - return Ok(found); - } - imported(py, "os.path", "abspath")? - .call1((path,))? - .extract() -} - -fn native_absolute(path: &Path) -> Option { - let Ok(cwd) = std::env::current_dir() else { - return None; - }; - Some(posix_abspath(cwd.to_str()?, path.to_str()?)) -} - -/// A new, empty recorder of one compile's artifact writes. -pub(crate) fn new_artifact_recorder(py: Python<'_>) -> PyResult> { - Py::new( - py, - CompileArtifactWrites { - artifacts: Mutex::new(HashMap::new()), - }, - ) -} - -/// Note the exact bytes an artifact holds after a write or an unchanged-file check. -pub(crate) fn artifact_written( - recorder: ArtifactRecorder<'_>, - path: &Bound<'_, PyAny>, - contents: &[u8], -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - recorder.get().record( - absolute(path)?, - RecordedArtifact::Written { - digest: artifact_digest(contents), - }, - ); - Ok(()) -} - -/// `artifact_written` for a native path. -pub(crate) fn artifact_written_at( - py: Python<'_>, - recorder: ArtifactRecorder<'_>, - path: &Path, - contents: &[u8], -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - recorder.get().record( - absolute_path(py, path)?, - RecordedArtifact::Written { - digest: artifact_digest(contents), - }, - ); - Ok(()) -} - -/// Note an earlier artifact kept as-is after its size and mtime were validated. -pub(crate) fn artifact_kept( - py: Python<'_>, - recorder: ArtifactRecorder<'_>, - path: &Path, - (size, mtime_ns): (u64, i64), -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - recorder.get().record( - absolute_path(py, path)?, - RecordedArtifact::Kept { size, mtime_ns }, - ); - Ok(()) -} - -/// Note that one staged artifact was published at another path. -pub(crate) fn artifact_moved( - recorder: ArtifactRecorder<'_>, - source: &Bound<'_, PyAny>, - destination: &Bound<'_, PyAny>, -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - recorder - .get() - .move_one(&absolute(source)?, absolute(destination)?); - Ok(()) -} - -/// Note that a staged artifact directory was published at another directory. -pub(crate) fn artifact_tree_moved( - recorder: ArtifactRecorder<'_>, - source: &Bound<'_, PyAny>, - destination: &Bound<'_, PyAny>, -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - recorder - .get() - .move_tree(&absolute(source)?, &absolute(destination)?); - Ok(()) -} - -/// The stderr lines and stdout report one compile printed, kept as the Python text it printed. -type PrintedOutput = (Vec>, Option>); - -/// Compile output printed at once (notes) or after storing (the report), keeping the text. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct RecordedCompileOutput { - output: Mutex, -} - -impl RecordedCompileOutput { - pub(crate) fn new() -> Self { - Self { - output: Mutex::new((Vec::new(), None)), - } - } - - fn locked(&self) -> std::sync::MutexGuard<'_, PrintedOutput> { - self.output.lock().unwrap_or_else(PoisonError::into_inner) - } - - /// Output already printed or recorded elsewhere: its stderr lines and its report. - pub(crate) fn recorded(py: Python<'_>, stderr_lines: &[String], stdout: &str) -> Self { - let lines: Vec> = stderr_lines - .iter() - .map(|line| PyString::new(py, line).unbind()) - .collect(); - Self { - output: Mutex::new((lines, Some(PyString::new(py, stdout).unbind()))), - } - } - - /// The recorded stderr lines and report, as text. - pub(crate) fn texts(&self, py: Python<'_>) -> PyResult<(Vec, Option)> { - let output = self.locked(); - let lines: Vec = output - .0 - .iter() - .map(|line| Ok(line.bind(py).to_str()?.to_owned())) - .collect::>()?; - let stdout: Option = match &output.1 { - Some(text) => Some(text.bind(py).to_str()?.to_owned()), - None => None, - }; - Ok((lines, stdout)) - } -} - -/// `print(*values, file=sys.)`. -pub(crate) fn print_to<'py>( - py: Python<'py>, - stream: &str, - text: impl pyo3::IntoPyObject<'py>, -) -> PyResult<()> { - let keywords: Bound<'_, PyDict> = - keywords(py, &[("file", py.import("sys")?.getattr(stream)?)])?; - let _ = py - .import("builtins")? - .getattr("print")? - .call((text,), Some(&keywords))?; - Ok(()) -} - -#[pymethods] -impl RecordedCompileOutput { - /// Print one stderr result line, such as a note, and record it; a delta only records it. - pub(crate) fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { - if sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() { - print_to(line.py(), "stderr", &line)?; - } - self.locked().0.push(line.unbind()); - Ok(()) - } - - /// Record the compile report; it is printed once the compile has been stored. - pub(crate) fn record_stdout(&self, text: &Bound<'_, PyString>) -> PyResult<()> { - let recorded: Bound<'_, PyString> = text.add("\n")?.downcast_into()?; - self.locked().1 = Some(recorded.unbind()); - Ok(()) - } - - /// Print the recorded compile report, if any, exactly as it is stored. - pub(crate) fn print_stdout(&self, py: Python<'_>) -> PyResult<()> { - let Some(text) = self.locked().1.as_ref().map(|text| text.clone_ref(py)) else { - return Ok(()); - }; - let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; - let _ = stdout.call_method1("write", (text,))?; - let _ = stdout.call_method0("flush")?; - Ok(()) - } - - #[getter] - pub(crate) fn stderr_lines(&self, py: Python<'_>) -> Vec> { - self.locked() - .0 - .iter() - .map(|line| line.clone_ref(py)) - .collect() - } - - #[getter] - pub(crate) fn stdout(&self, py: Python<'_>) -> Option> { - self.locked().1.as_ref().map(|text| text.clone_ref(py)) - } -} - -/// The hit, miss and bypass counters plus the reuse check time, keyed as `compile_timings` is. -pub(crate) fn compile_reuse_timings<'py>( - attempt: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = attempt.py(); - let outcome: String = native_field(attempt, "outcome")?.str()?.to_string(); - let timings: Bound<'py, PyDict> = PyDict::new(py); - for (name, counted) in [ - (REUSE_HIT_TIMING, HIT_OUTCOME), - (REUSE_MISS_TIMING, MISS_OUTCOME), - (REUSE_BYPASS_TIMING, BYPASS_OUTCOME), - ] { - timings.set_item(name, i64::from(outcome == counted))?; - } - timings.set_item(REUSE_CHECK_TIMING, native_field(attempt, "check_ms")?)?; - Ok(timings) -} - -fn reuse_logger(py: Python<'_>) -> PyResult> { - imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) -} - -/// Loaded module files, then the macro store's extra module paths, as Python lists them. -fn module_paths(py: Python<'_>) -> PyResult> { - let paths: Bound<'_, PyList> = PyList::empty(py); - let modules: Bound<'_, PyAny> = py.import("builtins")?.getattr("tuple")?.call1((py - .import("sys")? - .getattr("modules")? - .call_method0("values")?,))?; - let getattr: Bound<'_, PyAny> = py.import("builtins")?.getattr("getattr")?; - for module in modules.try_iter()? { - let path: Bound<'_, PyAny> = getattr.call1((module?, "__file__", py.None()))?; - if path.is_instance_of::() { - paths.append(path)?; - } - } - let extra: Bound<'_, PyAny> = py - .import("builtins")? - .getattr("sorted")? - .call1((host_entry(py, UserCodeHost::MacroStoreModulePaths)?.call0()?,))?; - for path in extra.try_iter()? { - let path: Bound<'_, PyAny> = path?; - if path.is_instance_of::() { - paths.append(path)?; - } - } - Ok(paths) -} - -/// The notice a long reuse record prints first, with when it started. -fn start_notice(py: Python<'_>, native: &Bound<'_, PyAny>) -> PyResult> { - let pending: u64 = native.call_method0("pending_digest_bytes")?.extract()?; - let paths: usize = native.call_method0("snapshot_paths")?.extract()?; - if pending < REUSE_STORE_NOTICE_BYTES && paths < REUSE_STORE_NOTICE_PATHS { - return Ok(None); - } - let mebibytes: f64 = pending as f64 / BYTES_PER_MEBIBYTE; - let message: String = - format!("Recording compile for reuse ({paths} paths, {mebibytes:.0} MiB to hash)..."); - print_to(py, "stderr", &message.into_bound_py_any(py)?)?; - Ok(Some(imported(py, "time", "monotonic")?.call0()?.extract()?)) -} - -fn finish_notice(py: Python<'_>, started: Option, prefix: &str) -> PyResult<()> { - let Some(started) = started else { - return Ok(()); - }; - let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; - let seconds: f64 = now - started; - print_to( - py, - "stderr", - &format!("{prefix}{seconds:.1} s)").into_bound_py_any(py)?, - ) -} - -/// What `write_reusable_compile` records about one full compile. -struct CompileFacts<'py> { - native: Bound<'py, PyAny>, - output: Bound<'py, PyAny>, - exit_code: Bound<'py, PyAny>, - input_reads: Bound<'py, PyAny>, - artifact_writes: Bound<'py, PyAny>, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Bound<'py, PyAny>, - json_output: Bound<'py, PyAny>, -} - -/// Store a reusable compile natively; returns whether it was stored. -fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { - let py: Python<'py> = facts.native.py(); - let settings: Bound<'_, PyAny> = call_with( - &host_entry(py, FacadeHost::ProviderSettingsInputs)?, - &[( - "settings_classes", - native_field(&facts.input_reads, "settings_classes")?, - )], - )?; - let unsupported: Bound<'_, PyAny> = native_field(&settings, "unsupported_reason")?; - if !unsupported.is_none() { - let _ = reuse_logger(py)?.call_method1( - "debug", - ("Compile reuse is off for this project: %s", &unsupported), - )?; - } - let read_run_id: Bound<'_, PyAny> = native_field(&facts.input_reads, "read_run_id")?; - let stdout: Bound<'_, PyAny> = native_field(&facts.output, "stdout")?; - let eligible: bool = !stdout.is_none() - && facts.compile_cache_enabled - && !read_run_id.is_truthy()? - && unsupported.is_none(); - let notice_started: Option = if eligible { - start_notice(py, &facts.native)? - } else { - None - }; - let strings = |value: Bound<'py, PyAny>| -> PyResult> { - value.try_iter()?.map(|item| item?.extract()).collect() - }; - let settings_inputs: Option> = if unsupported.is_none() { - let mut rows: Vec = Vec::new(); - for item in native_field(&settings, "inputs")?.try_iter()? { - let item: Bound<'_, PyAny> = item?; - rows.push(SettingsRow { - case_sensitive: native_field(&item, "case_sensitive")?.extract()?, - names: strings(native_field(&item, "names")?)?, - prefixes: strings(native_field(&item, "prefixes")?)?, - env_files: strings(native_field(&item, "env_files")?)?, - secrets_dirs: strings(native_field(&item, "secrets_dirs")?)?, - }); - } - Some(rows) - } else { - None - }; - let dag_path: Option = if facts.dag_artifact_path.is_none() { - None - } else { - Some( - imported(py, "os.path", "abspath")? - .call1((&facts.dag_artifact_path,))? - .extract()?, - ) - }; - let payload = CompileRecordPy { - stdout: stdout.extract()?, - stderr_lines: strings(native_field(&facts.output, "stderr_lines")?)?, - exit_code: facts.exit_code.extract()?, - compile_cache_enabled: facts.compile_cache_enabled, - read_run_id: read_run_id.extract()?, - settings_inputs, - template_environment_names: strings(native_field( - &facts.input_reads, - "environment_names", - )?)?, - module_paths: module_paths(py)?.extract()?, - artifacts: facts - .artifact_writes - .extract::>()? - .artifact_rows(), - artifacts_written: facts.artifacts_written, - dag_path, - json_output: facts.json_output.extract()?, - rules: reuse_rules(py)?, - }; - if let Some(mut capture) = - sqlbuild_python_base::compile_context::main::delta::take_capture::take_capture() - { - capture.environment_names = - native_field(&facts.input_reads, "environment_names")?.extract()?; - let json_output: bool = facts.json_output.is_truthy()?; - match crate::bindings::_helpers::compile_delta::image::ProjectImage::from_capture( - capture, - json_output, - ) { - Ok(image) => { - let mut pending: Option = None; - if let Ok(sections) = image.encode() { - pending = Some(sections); - } - sqlbuild_python_base::compile_context::main::delta::set_pending_image::set_pending_image(pending); - } - Err(reason) => { - crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug( - py, - &format!("Compile reuse kept no project image: {reason}"), - )?; - } - } - } - let (outcome, digested): (&str, usize) = facts - .native - .extract::>()? - .record(py, payload)?; - let _ = reuse_logger(py)?.call_method1( - "debug", - ("Compile reuse record digested %d project files", digested), - )?; - finish_notice( - py, - notice_started, - if outcome == SKIPPED { - REUSE_STORE_SKIPPED_PREFIX - } else { - REUSE_STORE_DONE_PREFIX - }, - )?; - Ok(outcome == STORED) -} - -/// Store a reusable full compile, or drop the stored one when it cannot be reused. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn write_reusable_compile( - py: Python<'_>, - options: Option>, -) -> PyResult<()> { - let arguments: KeywordArguments<'_> = KeywordArguments::read( - py, - "write_reusable_compile", - options, - ( - &[ - "attempt", - "output", - "exit_code", - "input_reads", - "artifact_writes", - "compile_cache_enabled", - "artifacts_written", - "dag_artifact_path", - "json_output", - ], - &[], - ), - )?; - let attempt: Bound<'_, PyAny> = arguments.value("attempt")?; - let entry_path: Bound<'_, PyAny> = native_field(&attempt, "entry_path")?; - let native: Bound<'_, PyAny> = native_field(&attempt, "native")?; - if !native_field(&attempt, "outcome")?.eq(MISS_OUTCOME)? - || entry_path.is_none() - || native.is_none() - { - return Ok(()); - } - let facts: CompileFacts<'_> = CompileFacts { - native, - output: arguments.value("output")?, - exit_code: arguments.value("exit_code")?, - input_reads: arguments.value("input_reads")?, - artifact_writes: arguments.value("artifact_writes")?, - compile_cache_enabled: arguments.flag("compile_cache_enabled")?, - artifacts_written: arguments.flag("artifacts_written")?, - dag_artifact_path: arguments.value("dag_artifact_path")?, - json_output: arguments.value("json_output")?, - }; - let stored: bool = match record(&facts) { - Ok(stored) => stored, - Err(error) if error.is_instance_of::(py) => { - let _ = reuse_logger(py)?.call_method( - "debug", - ("Compile reuse did not store this compile",), - Some(&keywords( - py, - &[("exc_info", error.value(py).clone().into_any())], - )?), - )?; - let unlinked: PyResult> = entry_path.call_method( - "unlink", - (), - Some(&keywords( - py, - &[("missing_ok", true.into_bound_py_any(py)?)], - )?), - ); - if let Err(error) = unlinked - && !error.is_instance_of::(py) - { - return Err(error); - } - false - } - Err(error) => return Err(error), - }; - if stored { - write_launch_record(&attempt)?; - } - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(write_reusable_compile, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs deleted file mode 100644 index a0b7f70019..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ /dev/null @@ -1,734 +0,0 @@ -//! The project image a full or delta compile stores beside its reuse slot. - -use std::collections::{BTreeMap, BTreeSet}; - -use sqlbuild_analysis::assembly::analysis_session::models::{DynamicFamily, RelationTables}; -use sqlbuild_analysis::assembly::analysis_session::types::Shapes; -use sqlbuild_cache::project_reuse::types::ImageSections; -use sqlbuild_rules::models::RetainedRows; - -use sqlbuild_discovery::header_changes::types::HeaderKeys; -use sqlbuild_python_base::compile_context::models::BaseRules; -use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; -use sqlbuild_python_base::compile_context::models::CapturedModel; -use sqlbuild_python_base::compile_context::models::CapturedTest; -use sqlbuild_python_base::compile_context::models::Decoder; -use sqlbuild_python_base::compile_context::models::Encoder; -use sqlbuild_python_base::compile_context::models::ImageCapture; -use sqlbuild_python_base::compile_context::models::ModelCoverage; -use sqlbuild_python_base::compile_context::models::PendingRules; -use sqlbuild_python_base::compile_context::types::ModelShape; - -/// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v9"; -pub(crate) const GATE_SECTION: &str = "gate"; -/// The upstream kind of a model another model reads. -const MODEL_KIND: &str = "model"; -pub(crate) const FACTS_SECTION: &str = "facts"; -const RULES_SECTION: &str = "rules"; -pub(crate) const RULE_DIGESTS_SECTION: &str = "rule-digests"; -const RULE_ROWS_SECTION: &str = "rule-rows"; - -/// Everything a delta compile reads of the compile it patches. -#[derive(Debug, Default)] -pub(crate) struct ProjectImage { - pub(crate) format: String, - pub(crate) json_output: bool, - pub(crate) models: Vec, - pub(crate) usages: BTreeMap, - pub(crate) tests: Vec, - pub(crate) relations: ImageRelations, - pub(crate) fact_digests: Vec<(Vec, String)>, - pub(crate) model_subjects: Option)>>, - pub(crate) diagnostics: Vec, - pub(crate) text_models: Option>, - pub(crate) environment_names: Vec, - pub(crate) pinned_sources: Vec, - pub(crate) test_sources: BTreeMap>, - pub(crate) model_shapes: Vec, - pub(crate) coverage: Option>, - pub(crate) lint_model_keys: Option>, - pub(crate) lint_file_keys: Option)>>, - pub(crate) header_keys: BTreeMap, - pub(crate) audit_references: BTreeMap>, - pub(crate) rules: Option, -} - -/// An image's rules request: decoding from a stored image, or as a compile just retained it. -#[derive(Debug)] -pub(crate) enum ImageRules { - Decoding(PendingRules), - Retained(RetainedRows), -} - -impl ImageRules { - /// The rules request still decoding from a stored image, if this is one. - pub(crate) fn pending(self) -> Option { - match self { - Self::Decoding(rules) => Some(rules), - Self::Retained(_) => None, - } - } - - /// The rows a compile just retained, if this is them. - pub(crate) fn retained(self) -> Option { - match self { - Self::Retained(rules) => Some(rules), - Self::Decoding(_) => None, - } - } -} - -/// The value `bytes` hold as JSON, or `None` when they are malformed. -fn from_json(bytes: &[u8]) -> Option { - let Ok(value) = serde_json::from_slice(bytes) else { - return None; - }; - Some(value) -} - -fn decoded_rules(request_json: Vec, rows: Vec, digests: &[u8]) -> Option { - let retained: RetainedRows = RetainedRows { - request_json, - rows, - digests: decoded_strings(digests)?, - }; - BaseRules::decoded(&retained) -} - -/// The analysis session's final relation facts, as stored. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct ImageRelations { - pub(crate) types: Shapes, - pub(crate) nullability: Shapes, - pub(crate) complete: Shapes, - pub(crate) schemas: Shapes, - pub(crate) dependency_ordered: bool, - pub(crate) cyclic: bool, - pub(crate) legacy_models: Vec, - pub(crate) families: FamiliesByTable, -} - -/// Each table's dynamic column families, in the order the analysis request listed them. -pub(crate) type FamiliesByTable = Vec<(String, Vec)>; - -impl From for ImageRelations { - fn from(tables: RelationTables) -> Self { - Self { - types: tables.types, - nullability: tables.nullability, - complete: tables.complete, - schemas: tables.schemas, - dependency_ordered: tables.dependency_ordered, - cyclic: tables.cyclic, - legacy_models: tables.legacy_models, - families: tables.families, - } - } -} - -impl From for RelationTables { - fn from(relations: ImageRelations) -> Self { - Self { - types: relations.types, - nullability: relations.nullability, - complete: relations.complete, - schemas: relations.schemas, - dependency_ordered: relations.dependency_ordered, - cyclic: relations.cyclic, - legacy_models: relations.legacy_models, - families: relations.families, - } - } -} - -/// Rows of `table` that differ from the same relation's `types` row; `None` where equal. -fn differing(table: &Shapes, types: &Shapes) -> StoredShapes { - let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types - .iter() - .map(|(name, shape)| (name.as_str(), shape)) - .collect(); - table - .iter() - .map(|(name, shape)| { - let same: bool = types - .get(name.as_str()) - .is_some_and(|typed| *typed == shape); - (name.clone(), (!same).then(|| shape.clone())) - }) - .collect() -} - -/// `differing`'s rows restored against `types`. -fn restored(rows: StoredShapes, types: &Shapes) -> Option { - let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types - .iter() - .map(|(name, shape)| (name.as_str(), shape)) - .collect(); - rows.into_iter() - .map(|(name, shape)| { - let shape: Vec<(String, String)> = match shape { - Some(shape) => shape, - None => (*types.get(name.as_str())?).clone(), - }; - Some((name, shape)) - }) - .collect() -} - -/// A shape table as stored: `None` where a relation's row equals its `types` row. -type StoredShapes = Vec<(String, Option>)>; - -/// What a delta compile reads first to decide, cheaply, whether it can patch the image. -#[derive(Clone, Debug, Default)] -pub(crate) struct ImageGate { - pub(crate) format: String, - pub(crate) json_output: bool, - /// Whether the stored compile reported any error diagnostic. - pub(crate) has_errors: bool, - /// Whether custom rules recorded which models they select. - pub(crate) has_model_subjects: bool, - pub(crate) models: Vec, - pub(crate) tests: Vec, - pub(crate) fact_digests: Vec<(Vec, String)>, - pub(crate) header_keys: BTreeMap, -} - -/// The rest of the image's facts as stored; shapes equal to a relation's types are stored once. -struct StoredFacts { - usages: BTreeMap, - types: Shapes, - nullability: Shapes, - complete: StoredShapes, - schemas: StoredShapes, - dependency_ordered: bool, - cyclic: bool, - legacy_models: Vec, - families: FamiliesByTable, - model_subjects: Option)>>, - diagnostics: Vec, - text_models: Option>, - environment_names: Vec, - pinned_sources: Vec, - test_sources: BTreeMap>, - model_shapes: StoredShapes, - coverage: Option>, - lint_model_keys: Option>, - lint_file_keys: Option)>>, - audit_references: BTreeMap>, -} - -fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { - sections - .iter() - .find(|(section, _)| section == name) - .map(|(_, bytes)| bytes.as_slice()) -} - -/// The diagnostics and report lines an image keeps as one JSON array, being small and varied. -struct StoredMisc { - diagnostics: Vec, - text_models: Option>, - environment_names: Vec, - coverage: Option>, -} - -impl<'de> serde::Deserialize<'de> for StoredMisc { - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_tuple(4, StoredMiscVisitor) - } -} - -/// Reads [`StoredMisc`] from its four-element array. -struct StoredMiscVisitor; - -impl<'de> serde::de::Visitor<'de> for StoredMiscVisitor { - type Value = StoredMisc; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("a tuple of size 4") - } - - fn visit_seq>(self, mut seq: A) -> Result { - let missing = |index: usize| serde::de::Error::invalid_length(index, &self); - Ok(StoredMisc { - diagnostics: seq.next_element()?.ok_or_else(|| missing(0))?, - text_models: seq.next_element()?.ok_or_else(|| missing(1))?, - environment_names: seq.next_element()?.ok_or_else(|| missing(2))?, - coverage: seq.next_element()?.ok_or_else(|| missing(3))?, - }) - } -} - -fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { - encoder.count(shapes.len()); - for (name, shape) in shapes { - encoder.string(name); - encoder.pairs(shape); - } -} - -fn put_stored_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a StoredShapes) { - encoder.count(shapes.len()); - for (name, shape) in shapes { - encoder.string(name); - encoder.flag(shape.is_some()); - if let Some(shape) = shape { - encoder.pairs(shape); - } - } -} - -fn shapes(decoder: &mut Decoder<'_>) -> Option { - decoder.list(|decoder| Some((decoder.string()?, decoder.pairs()?))) -} - -fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { - decoder.list(|decoder| { - let name: String = decoder.string()?; - let shape: Option> = if decoder.flag()? { - Some(decoder.pairs()?) - } else { - None - }; - Some((name, shape)) - }) -} - -fn put_families<'a>(encoder: &mut Encoder<'a>, families: &'a FamiliesByTable) { - encoder.count(families.len()); - for (table, declared) in families { - encoder.string(table); - encoder.count(declared.len()); - for family in declared { - encoder.string(&family.name); - encoder.string(&family.pivot_column); - encoder.string(&family.value_column); - encoder.string(&family.aggregate); - encoder.string(&family.data_type); - encoder.optional_string(family.name_pattern.as_deref()); - } - } -} - -fn families(decoder: &mut Decoder<'_>) -> Option { - decoder.list(|decoder| { - let table: String = decoder.string()?; - let declared: Vec = decoder.list(|decoder| { - Some(DynamicFamily { - name: decoder.string()?, - pivot_column: decoder.string()?, - value_column: decoder.string()?, - aggregate: decoder.string()?, - data_type: decoder.string()?, - name_pattern: decoder.optional_string()?, - }) - })?; - Some((table, declared)) - }) -} - -impl ImageGate { - fn encode(&self) -> Vec { - let mut encoder: Encoder<'_> = Encoder::default(); - encoder.string(&self.format); - encoder.flag(self.json_output); - encoder.flag(self.has_errors); - encoder.flag(self.has_model_subjects); - encoder.count(self.models.len()); - for model in &self.models { - encoder.string(&model.name); - encoder.string(&model.relative_path); - encoder.pairs(&model.upstream); - } - encoder.count(self.tests.len()); - for test in &self.tests { - encoder.string(&test.key); - encoder.string(&test.relative_path); - encoder.strings(test.chain.iter().map(String::as_str)); - } - encoder.count(self.fact_digests.len()); - for (key, digest) in &self.fact_digests { - encoder.strings(key.iter().map(String::as_str)); - encoder.string(digest); - } - encoder.count(self.header_keys.len()); - for (path, keys) in &self.header_keys { - encoder.string(path); - encoder.pairs(keys); - } - encoder.finish() - } - - /// The gate stored in `sections`, or `None` when it is unreadable or of another format. - pub(crate) fn decode(sections: &ImageSections) -> Option { - let mut decoder: Decoder<'_> = Decoder::new(section(sections, GATE_SECTION)?)?; - let format: String = decoder.string()?; - if format != IMAGE_FORMAT { - return None; - } - let gate: Self = Self { - format, - json_output: decoder.flag()?, - has_errors: decoder.flag()?, - has_model_subjects: decoder.flag()?, - models: decoder.list(|decoder| { - Some(CapturedModel { - name: decoder.string()?, - relative_path: decoder.string()?, - upstream: decoder.pairs()?, - }) - })?, - tests: decoder.list(|decoder| { - Some(CapturedTest { - key: decoder.string()?, - relative_path: decoder.string()?, - chain: decoder.strings()?, - }) - })?, - fact_digests: decoder.list(|decoder| Some((decoder.strings()?, decoder.string()?)))?, - header_keys: decoder - .list(|decoder| Some((decoder.string()?, decoder.pairs()?)))? - .into_iter() - .collect(), - }; - decoder.finish()?; - Some(gate) - } -} - -impl StoredFacts { - fn encode(&self) -> Result, String> { - let misc: Vec = serde_json::to_vec(&( - &self.diagnostics, - &self.text_models, - &self.environment_names, - &self.coverage, - )) - .map_err(|error| error.to_string())?; - let mut encoder: Encoder<'_> = Encoder::default(); - encoder.count(self.usages.len()); - for (path, digest) in &self.usages { - encoder.string(path); - encoder.string(digest); - } - put_shapes(&mut encoder, &self.types); - put_shapes(&mut encoder, &self.nullability); - put_stored_shapes(&mut encoder, &self.complete); - put_stored_shapes(&mut encoder, &self.schemas); - put_stored_shapes(&mut encoder, &self.model_shapes); - encoder.flag(self.dependency_ordered); - encoder.flag(self.cyclic); - encoder.strings(self.legacy_models.iter().map(String::as_str)); - put_families(&mut encoder, &self.families); - encoder.flag(self.model_subjects.is_some()); - if let Some(subjects) = &self.model_subjects { - encoder.count(subjects.len()); - for (path, name, materialized) in subjects { - encoder.string(path); - encoder.string(name); - encoder.optional_string(materialized.as_deref()); - } - } - encoder.flag(self.lint_model_keys.is_some()); - if let Some(keys) = &self.lint_model_keys { - encoder.pairs(keys); - } - encoder.flag(self.lint_file_keys.is_some()); - if let Some(keys) = &self.lint_file_keys { - encoder.count(keys.len()); - for (path, key) in keys { - encoder.string(path); - encoder.optional_string(key.as_deref()); - } - } - encoder.strings(self.pinned_sources.iter().map(String::as_str)); - encoder.count(self.test_sources.len()); - for (path, names) in &self.test_sources { - encoder.string(path); - encoder.strings(names.iter().map(String::as_str)); - } - encoder.count(self.audit_references.len()); - for (path, names) in &self.audit_references { - encoder.string(path); - encoder.strings(names.iter().map(String::as_str)); - } - encoder.bytes(&misc); - Ok(encoder.finish()) - } - - fn decode(section: &[u8]) -> Option { - let mut decoder: Decoder<'_> = Decoder::new(section)?; - let usages: BTreeMap = decoder - .list(|decoder| Some((decoder.string()?, decoder.string()?)))? - .into_iter() - .collect(); - let types: Shapes = shapes(&mut decoder)?; - let nullability: Shapes = shapes(&mut decoder)?; - let complete: StoredShapes = stored_shapes(&mut decoder)?; - let schemas: StoredShapes = stored_shapes(&mut decoder)?; - let model_shapes: StoredShapes = stored_shapes(&mut decoder)?; - let dependency_ordered: bool = decoder.flag()?; - let cyclic: bool = decoder.flag()?; - let legacy_models: Vec = decoder.strings()?; - let families: FamiliesByTable = families(&mut decoder)?; - let model_subjects = if decoder.flag()? { - Some(decoder.list(|decoder| { - Some(( - decoder.string()?, - decoder.string()?, - decoder.optional_string()?, - )) - })?) - } else { - None - }; - let lint_model_keys = if decoder.flag()? { - Some(decoder.pairs()?) - } else { - None - }; - let lint_file_keys = if decoder.flag()? { - Some(decoder.list(|decoder| Some((decoder.string()?, decoder.optional_string()?)))?) - } else { - None - }; - let pinned_sources: Vec = decoder.strings()?; - let test_sources: BTreeMap> = decoder - .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? - .into_iter() - .collect(); - let audit_references: BTreeMap> = decoder - .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? - .into_iter() - .collect(); - let StoredMisc { - diagnostics, - text_models, - environment_names, - coverage, - }: StoredMisc = from_json(decoder.bytes()?)?; - decoder.finish()?; - Some(Self { - usages, - types, - nullability, - complete, - schemas, - dependency_ordered, - cyclic, - legacy_models, - families, - model_subjects, - diagnostics, - text_models, - environment_names, - pinned_sources, - test_sources, - model_shapes, - coverage, - lint_model_keys, - lint_file_keys, - audit_references, - }) - } -} - -fn encoded_strings(values: &[String]) -> Vec { - let mut encoder: Encoder<'_> = Encoder::default(); - encoder.strings(values.iter().map(String::as_str)); - encoder.finish() -} - -fn decoded_strings(section: &[u8]) -> Option> { - let mut decoder: Decoder<'_> = Decoder::new(section)?; - let values: Vec = decoder.strings()?; - decoder.finish()?; - Some(values) -} - -impl ProjectImage { - /// The image of a finished compile's capture, or why it cannot be a delta base. - pub(crate) fn from_capture(capture: ImageCapture, json_output: bool) -> Result { - if let Some(reason) = capture.unsupported { - return Err(reason); - } - if capture.stdlib_json { - return Err("the report fell back to the standard library serializer".to_owned()); - } - let Some(relations) = capture.relations else { - return Err("no model analysis session ran".to_owned()); - }; - if !json_output && capture.text_models.is_none() { - return Err("no text report was rendered".to_owned()); - } - let Some(model_shapes) = capture.model_shapes else { - return Err("no semantic shapes were published".to_owned()); - }; - let mut environment_names: Vec = capture.environment_names; - environment_names.sort(); - environment_names.dedup(); - Ok(Self { - format: IMAGE_FORMAT.to_owned(), - json_output, - models: capture.models, - usages: capture.usages, - tests: capture.test_chains, - relations: relations.into(), - fact_digests: capture.fact_digests.into_iter().collect(), - model_subjects: capture.model_subjects, - diagnostics: capture.diagnostics, - text_models: capture.text_models, - environment_names, - pinned_sources: capture.pinned_sources.into_iter().collect(), - test_sources: capture - .test_sources - .into_iter() - .map(|(path, names)| (path, names.into_iter().collect())) - .collect(), - model_shapes, - coverage: capture.coverage, - lint_model_keys: capture.lint_model_keys, - lint_file_keys: capture.lint_file_keys, - header_keys: capture.header_keys, - audit_references: capture.audit_references, - rules: capture - .rules - .map(|rules| ImageRules::Retained(rules.retained)), - }) - } - - /// The image as the sections of its file. - pub(crate) fn encode(self) -> Result { - let gate: ImageGate = ImageGate { - format: self.format, - json_output: self.json_output, - has_errors: self.diagnostics.iter().any(|diagnostic| diagnostic.error), - has_model_subjects: self.model_subjects.is_some(), - models: self.models, - tests: self.tests, - fact_digests: self.fact_digests, - header_keys: self.header_keys, - }; - let relations: ImageRelations = self.relations; - let types: &Shapes = &relations.types; - let facts: StoredFacts = StoredFacts { - usages: self.usages, - complete: differing(&relations.complete, types), - schemas: differing(&relations.schemas, types), - model_shapes: differing(&self.model_shapes, types), - types: relations.types, - nullability: relations.nullability, - dependency_ordered: relations.dependency_ordered, - cyclic: relations.cyclic, - legacy_models: relations.legacy_models, - families: relations.families, - model_subjects: self.model_subjects, - diagnostics: self.diagnostics, - text_models: self.text_models, - environment_names: self.environment_names, - coverage: self.coverage, - pinned_sources: self.pinned_sources, - test_sources: self.test_sources, - lint_model_keys: self.lint_model_keys, - lint_file_keys: self.lint_file_keys, - audit_references: self.audit_references, - }; - let rules: Option = self.rules.and_then(ImageRules::retained); - let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); - let rules = rules.map(|rules| { - ( - rules.request_json, - rules.rows, - encoded_strings(&rules.digests), - ) - }); - let mut sections: ImageSections = vec![ - (GATE_SECTION.to_owned(), gate), - (FACTS_SECTION.to_owned(), facts?), - ]; - if let Some((request, rows, digests)) = rules { - sections.push((RULES_SECTION.to_owned(), request)); - sections.push((RULE_ROWS_SECTION.to_owned(), rows)); - sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); - } - Ok(sections) - } - - /// The image behind `gate`, its rules request decoding on its own thread; `None` if unreadable. - pub(crate) fn decode(mut sections: ImageSections, gate: ImageGate) -> Option { - let mut taken = |name: &str| { - sections - .iter_mut() - .find(|(section, _)| section == name) - .map(|(_, bytes)| std::mem::take(bytes)) - }; - let stored_rules = ( - taken(RULES_SECTION), - taken(RULE_ROWS_SECTION), - taken(RULE_DIGESTS_SECTION), - ); - let rules: Option = match stored_rules { - (Some(request), Some(rows), Some(digests)) => { - Some(ImageRules::Decoding(PendingRules::Decoding( - std::thread::spawn(move || decoded_rules(request, rows, &digests)), - ))) - } - (None, None, None) => None, - _ => return None, - }; - let facts: StoredFacts = StoredFacts::decode(section(§ions, FACTS_SECTION)?)?; - let types: Shapes = facts.types; - let relations: ImageRelations = ImageRelations { - complete: restored(facts.complete, &types)?, - schemas: restored(facts.schemas, &types)?, - nullability: facts.nullability, - dependency_ordered: facts.dependency_ordered, - cyclic: facts.cyclic, - legacy_models: facts.legacy_models, - families: facts.families, - types: Shapes::new(), - }; - let model_shapes: Shapes = restored(facts.model_shapes, &types)?; - Some(Self { - format: gate.format, - json_output: gate.json_output, - models: gate.models, - usages: facts.usages, - tests: gate.tests, - relations: ImageRelations { types, ..relations }, - fact_digests: gate.fact_digests, - model_subjects: facts.model_subjects, - diagnostics: facts.diagnostics, - text_models: facts.text_models, - environment_names: facts.environment_names, - pinned_sources: facts.pinned_sources, - test_sources: facts.test_sources, - model_shapes, - coverage: facts.coverage, - lint_model_keys: facts.lint_model_keys, - lint_file_keys: facts.lint_file_keys, - header_keys: gate.header_keys, - audit_references: facts.audit_references, - rules, - }) - } - - /// Every model whose reads include a relation of `names`, outside `names`. - pub(crate) fn dependents(&self, names: &BTreeSet) -> BTreeSet { - self.models - .iter() - .filter(|model| !names.contains(&model.name)) - .filter(|model| reads_any_model(model, names)) - .map(|model| model.name.clone()) - .collect() - } -} - -/// Whether `model` reads any of the models `names`. -fn reads_any_model(model: &CapturedModel, names: &BTreeSet) -> bool { - model - .upstream - .iter() - .any(|(kind, name)| kind == MODEL_KIND && names.contains(name)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs deleted file mode 100644 index da4c6ceec2..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! The SQL lint keys a full compile leaves for its image, and a delta compile's use of them. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use pyo3::PyResult; - -use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; -use sqlbuild_python_base::compile_context::main::delta::capture::capture; -use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; -use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported; -use sqlbuild_python_base::compile_context::main::delta::injection::injection; - -use crate::bindings::_helpers::rules::lint_pipeline::LintFile; - -/// Each model file's lint key, recorded for the image or, in a delta, patched for reruns. -pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult> { - let Some(injection) = injection() else { - if capturing() { - let keys: Vec<(String, String)> = fresh.clone(); - capture(|capture| capture.lint_model_keys = Some(keys)); - } - return Ok(fresh); - }; - let Some(stored) = injection.lint_model_keys.as_ref() else { - return Err(delta_unsupported( - "the lint keys of the models were not recorded", - )); - }; - let mut fresh: HashMap = fresh.into_iter().collect(); - let whole: Vec<(String, String)> = stored - .iter() - .map(|(path, key)| match fresh.remove(path) { - Some(key) => (path.clone(), key), - None => (path.clone(), key.clone()), - }) - .collect(); - if !fresh.is_empty() { - return Err(delta_unsupported( - "a recompiled model has no recorded lint key", - )); - } - capture(|capture| capture.lint_model_keys = Some(whole.clone())); - Ok(whole) -} - -/// The changed files a delta compile lints, read as a full lint walk reads them. -pub(crate) fn changed_files(project_dir: &Path) -> PyResult> { - let Some(injection) = injection() else { - return Ok(Vec::new()); - }; - let mut files: Vec = Vec::new(); - for path in &injection.changed_paths { - let linted: bool = path - .split_once('/') - .is_some_and(|(root, _)| LINT_DIRECTORY_NAMES.contains(&root)); - if !linted { - continue; - } - let path: PathBuf = project_dir.join(path); - let contents: String = std::fs::read_to_string(&path) - .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; - files.push(LintFile { path, contents }); - } - Ok(files) -} - -/// Each non-model file's lint key in walk order, recorded or, in a delta, patched. -pub(crate) fn file_keys( - project_dir: &Path, - fresh: Vec<(PathBuf, String, Option)>, -) -> PyResult)>> { - let Some(injection) = injection() else { - if capturing() { - let keys: Vec<(String, Option)> = fresh - .iter() - .map(|(_, path, key)| (path.clone(), key.clone())) - .collect(); - capture(|capture| capture.lint_file_keys = Some(keys)); - } - return Ok(fresh); - }; - let Some(stored) = injection.lint_file_keys.as_ref() else { - return Err(delta_unsupported( - "the lint keys of the project files were not recorded", - )); - }; - let mut fresh: HashMap)> = - fresh.into_iter().map(|row| (row.1.clone(), row)).collect(); - let whole: Vec<(PathBuf, String, Option)> = stored - .iter() - .map(|(path, key)| { - fresh - .remove(path) - .unwrap_or_else(|| (project_dir.join(path), path.clone(), key.clone())) - }) - .collect(); - if !fresh.is_empty() { - return Err(delta_unsupported("a changed file has no recorded lint key")); - } - let keys: Vec<(String, Option)> = whole - .iter() - .map(|(_, path, key)| (path.clone(), key.clone())) - .collect(); - capture(|capture| capture.lint_file_keys = Some(keys)); - Ok(whole) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs deleted file mode 100644 index 5bc7c4197a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs +++ /dev/null @@ -1,238 +0,0 @@ -//! The hidden check that each delta compile matches the full compile it replaces. - -use std::collections::{BTreeMap, HashMap}; -use std::path::{Path, PathBuf}; -use std::sync::{Mutex, MutexGuard, PoisonError}; - -use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_cache::digest::main::digest_files::digest_files; -use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use sqlbuild_cache::project_reuse::constants::NATIVE_DIGEST_PREFIX; - -use crate::bindings::_helpers::compile_command::reuse_recording::{ - CompileArtifactWrites, RecordedCompileOutput, -}; - -/// The exit code of a compile whose delta differed from its full compile. -pub(crate) const MISMATCH_EXIT_CODE: i64 = 70; -const MISMATCH_FILE: &str = "delta-mismatch.json"; - -/// What the delta compile produced, held until the full compile finishes. -#[derive(Debug, Default)] -pub(crate) struct Expected { - pub(crate) stdout: Option, - pub(crate) exit_code: Option, - pub(crate) raised: Option, - /// Every file under the compiled root after the delta, with its native digest. - pub(crate) compiled: BTreeMap, - pub(crate) compiled_root: String, - pub(crate) artifacts_written: bool, - /// Every file under `target/compiled-failed/` after the delta, with its native digest. - pub(crate) failed: BTreeMap, - pub(crate) failed_root: String, -} - -/// What the full compile produced. -#[derive(Debug, Default)] -struct Actual { - stdout: Option, - exit_code: Option, - artifacts_written: bool, - written: HashMap>, -} - -static EXPECTED: Mutex> = Mutex::new(None); -static ACTUAL: Mutex> = Mutex::new(None); - -fn locked(slot: &Mutex) -> MutexGuard<'_, T> { - slot.lock().unwrap_or_else(PoisonError::into_inner) -} - -/// Every file under `directory`, depth first. -fn files_under(directory: &Path) -> Vec { - let Ok(entries) = std::fs::read_dir(directory) else { - return Vec::new(); - }; - let mut found: Vec = Vec::new(); - for entry in entries.flatten() { - let path: PathBuf = entry.path(); - match entry.file_type() { - Ok(kind) if kind.is_dir() => found.extend(files_under(&path)), - Ok(kind) if kind.is_file() => found.push(path), - _ => {} - } - } - found -} - -/// Every file under `root` with its native digest, keyed by absolute path text. -pub(crate) fn compiled_digests(root: &Path) -> BTreeMap { - let paths: Vec = files_under(root); - let digests = digest_files(&paths); - paths - .into_iter() - .zip(digests) - .filter_map(|(path, digest)| { - let Ok(digest) = digest else { - return None; - }; - Some(( - path.to_string_lossy().into_owned(), - format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest)), - )) - }) - .collect() -} - -/// Hold the delta's result for comparison once the full compile finishes. -pub(crate) fn hold_expected(expected: Expected) { - *locked(&EXPECTED) = Some(expected); - *locked(&ACTUAL) = None; -} - -/// Note the full compile's report, exit code and artifact writes, when a check is armed. -pub(crate) fn full_compile_finished( - output: &Bound<'_, PyAny>, - (exit_code, artifacts_written): (i64, bool), - artifact_writes: &Bound<'_, PyAny>, -) -> PyResult<()> { - if locked(&EXPECTED).is_none() { - return Ok(()); - } - let (_, stdout): (Vec, Option) = output - .cast::()? - .get() - .texts(output.py())?; - let written: HashMap> = artifact_writes - .cast::() - .map(|writes| writes.get().written_digests()) - .unwrap_or_default(); - *locked(&ACTUAL) = Some(Actual { - stdout, - exit_code: Some(exit_code), - artifacts_written, - written, - }); - Ok(()) -} - -/// The report with its `compile_timings` value blanked. -fn without_timings(stdout: &str) -> String { - let Some(start) = stdout.find("\n \"compile_timings\": {") else { - return stdout.to_owned(); - }; - let Some(length) = stdout[start..].find("\n },\n") else { - return stdout.to_owned(); - }; - format!("{}{}", &stdout[..start], &stdout[start + length + 5..]) -} - -/// Compare the held delta result with the full compile's; every difference found. -pub(crate) fn differences(raised: Option) -> Vec { - let Some(expected) = locked(&EXPECTED).take() else { - return Vec::new(); - }; - let actual: Actual = locked(&ACTUAL).take().unwrap_or_default(); - let mut found: Vec = Vec::new(); - if expected.raised != raised { - found.push(format!( - "raised error differs: delta {:?}, full {:?}", - expected.raised, raised - )); - } - if expected.raised.is_some() || raised.is_some() { - return found; - } - if expected.exit_code != actual.exit_code { - found.push(format!( - "exit code differs: delta {:?}, full {:?}", - expected.exit_code, actual.exit_code - )); - } - let delta_stdout: Option = expected.stdout.as_deref().map(without_timings); - let full_stdout: Option = actual.stdout.as_deref().map(without_timings); - if delta_stdout != full_stdout { - let (delta, full) = ( - delta_stdout.unwrap_or_default(), - full_stdout.unwrap_or_default(), - ); - let common: usize = delta - .bytes() - .zip(full.bytes()) - .take_while(|(left, right)| left == right) - .count(); - let start: usize = common.saturating_sub(200); - let excerpt = |text: &str| -> String { - text.get(start..(common + 300).min(text.len())) - .unwrap_or_default() - .to_owned() - }; - found.push(format!( - "report differs at byte {common}:\n--- delta\n{}\n--- full\n{}", - excerpt(&delta), - excerpt(&full) - )); - } - if expected.artifacts_written != actual.artifacts_written { - found.push(format!( - "artifact writing differs: delta {}, full {}", - expected.artifacts_written, actual.artifacts_written - )); - } - found.extend(tree_differences( - (&expected.failed_root, &expected.failed), - &actual.written, - )); - if actual.artifacts_written { - found.extend(tree_differences( - (&expected.compiled_root, &expected.compiled), - &actual.written, - )); - } - found.truncate(50); - found -} - -/// Where the files the full compile wrote under `root` differ from what the delta left there. -fn tree_differences( - (root, expected): (&str, &BTreeMap), - written: &HashMap>, -) -> Vec { - let mut found: Vec = Vec::new(); - let prefix: String = format!("{root}{}", std::path::MAIN_SEPARATOR); - let full_paths: BTreeMap<&str, &Option> = written - .iter() - .filter(|(path, _)| path.starts_with(prefix.as_str())) - .map(|(path, digest)| (path.as_str(), digest)) - .collect(); - for (path, digest) in &full_paths { - match (expected.get(*path), digest) { - (None, _) => found.push(format!("the delta left no artifact at {path}")), - (Some(left), Some(right)) if left != right => { - found.push(format!("artifact differs: {path}")); - } - _ => {} - } - } - for path in expected.keys() { - if !full_paths.contains_key(path.as_str()) { - found.push(format!( - "the delta left an artifact the full compile does not: {path}" - )); - } - } - found -} - -/// Write the differences beside the compiler cache for inspection. -pub(crate) fn write_mismatch(cache_dir: &Path, differences: &[String]) { - let _ = std::fs::create_dir_all(cache_dir); - let text: String = serde_json::to_string_pretty(differences).unwrap_or_default(); - let _ = std::fs::write(cache_dir.join(MISMATCH_FILE), text); -} - -/// Forget any held result. -pub(crate) fn disarm() { - *locked(&EXPECTED) = None; - *locked(&ACTUAL) = None; -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index e7112cca71..57c5e43654 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,28 +1,4 @@ -pub(crate) mod analysis; -pub(crate) mod analysis_cache; -pub(crate) mod assembly; -pub(crate) mod assembly_analysis; -pub(crate) mod assembly_checks; -pub(crate) mod assembly_facts; -pub(crate) mod assembly_models; -pub(crate) mod assembly_project; -pub(crate) mod assembly_sql_tests; -pub(crate) mod command; -pub(crate) mod deferral; -pub(crate) mod diff; -pub(crate) mod discovery; -pub(crate) mod inputs; -pub(crate) mod integration_loaders; -pub(crate) mod phases; -pub(crate) mod pipeline; -pub(crate) mod pipeline_entries; pub(crate) mod plan_pipeline; pub(crate) mod plan_result; -pub(crate) mod project; -pub(crate) mod python_plan_entries; pub(crate) mod registration; -pub(crate) mod relation_targets; -pub(crate) mod selection_phases; pub(crate) mod static_context; -pub(crate) mod target_defaults; -pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs deleted file mode 100644 index eb1a3b50b9..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ /dev/null @@ -1,211 +0,0 @@ -//! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PySet, PySetMethods}; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_driver::phases::monotonic; -use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::diagnostics::main::current_tracker::current_tracker; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced::replaced; -use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; - -/// The current build's phase timing tracker, when one is recording. -pub(crate) fn timing_tracker(py: Python<'_>) -> PyResult> { - current_tracker(py) -} - -/// Report `message` through `on_progress` when a callback is set. -pub(crate) fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { - if !on_progress.is_none() { - on_progress.call1((message,))?; - } - Ok(()) -} - -/// Compile a project without opening a warehouse planning connection. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn compile_project_phase<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "compile_project_phase", - options, - ( - &["discovered_inputs", "adapter"], - &["options", "on_progress"], - ), - )?; - compiled_phase( - ( - &arguments.value("discovered_inputs")?, - &arguments.value("adapter")?, - ), - &arguments.value("options")?, - &arguments.value("on_progress")?, - None, - ) -} - -/// `compile_project_phase` with resolved arguments. -pub(crate) fn compiled_phase<'py>( - (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - options: &Bound<'py, PyAny>, - on_progress: &Bound<'py, PyAny>, - on_inputs_ready: Option<&Bound<'py, PyAny>>, -) -> PyResult> { - let py: Python<'py> = discovered.py(); - let options: Bound<'py, PyAny> = if options.is_truthy()? { - options.clone() - } else { - new_record(py, RecordId::CompilePipelineOptions, (), None)? - }; - let configured: Bound<'py, PyAny> = native_field(&options, "connection_config")?; - let connection_config: Bound<'py, PyAny> = if configured.is_none() { - build_effective_connection_config( - discovered, - Some(native_field(&options, "selected_target")?), - Some(native_field(&options, "cli_vars")?), - false, - )? - } else { - configured - }; - progress(on_progress, "Compiling project...")?; - let started: f64 = monotonic(py)?; - let compiled: PyResult> = (|| { - let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; - let selection: Bound<'py, PyAny> = match on_inputs_ready { - Some(hook) => replaced(&selection, &[("on_inputs_ready", hook.clone())])?, - None => selection, - }; - build_compiled_project( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ( - "selected_target", - native_field(&options, "selected_target")?, - ), - ( - "no_sql_validation", - native_field(&options, "no_sql_validation")?, - ), - ("cli_vars", native_field(&options, "cli_vars")?), - ( - "external_sql_reference_resolver", - native_field(&options, "external_sql_reference_resolver")?, - ), - ("resolved_connection", connection_config.clone()), - ("analysis_selection", selection), - ( - "source_lexical_syntax", - native_field(&options, "source_lexical_syntax")?, - ), - ], - )?), - ) - })(); - let compile_seconds: f64 = monotonic(py)? - started; - let tracker: Bound<'py, PyAny> = timing_tracker(py)?; - if !tracker.is_none() { - tracker.setattr("compile_seconds", compile_seconds)?; - } - let project: Bound<'py, PyAny> = compiled?; - reject_compile_errors(&project, &options, on_progress)?; - progress( - on_progress, - &format!("Compiled project. ({compile_seconds:.2}s)"), - )?; - new_record_with( - py, - RecordId::CompiledProjectPhaseResult, - &[ - ("project", project), - ("connection_config", connection_config), - ( - "compile_seconds", - compile_seconds.into_pyobject(py)?.into_any(), - ), - ], - ) -} - -/// Raise one `CompileInputError` listing every compile error the command must stop on. -fn reject_compile_errors( - project: &Bound<'_, PyAny>, - options: &Bound<'_, PyAny>, - on_progress: &Bound<'_, PyAny>, -) -> PyResult<()> { - let py: Python<'_> = project.py(); - let plan_sql_tests: bool = native_field(options, "plan_sql_tests")?.is_truthy()?; - let sql_test: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "SQL_TEST")?; - let mut errors: Vec> = Vec::new(); - for diagnostic in native_field(project, "diagnostics")?.try_iter()? { - let diagnostic: Bound<'_, PyAny> = diagnostic?; - if native_field(&diagnostic, "is_error")?.is_truthy()? - && (plan_sql_tests || native_field(&diagnostic, "resource_type")?.ne(&sql_test)?) - { - errors.push(diagnostic); - } - } - let Some(first) = errors.first() else { - return Ok(()); - }; - progress(on_progress, "Project compilation failed.")?; - let mut lines: Vec = Vec::with_capacity(errors.len()); - let helps: Bound<'_, PySet> = PySet::empty(py)?; - for diagnostic in &errors { - let or_one = |name: &str| -> PyResult { - let value: Bound<'_, PyAny> = native_field(diagnostic, name)?; - Ok(if value.is_truthy()? { - value.str()?.to_string() - } else { - "1".to_owned() - }) - }; - lines.push(format!( - "[{}] {}:{}:{}: {}", - native_field(diagnostic, "code")?.str()?, - native_field(diagnostic, "path")?.str()?, - or_one("line")?, - or_one("column")?, - native_field(diagnostic, "message")?.str()?, - )); - helps.add(native_field(diagnostic, "help")?)?; - } - let help: Bound<'_, PyAny> = if helps.len() == 1 { - helps.pop().unwrap_or_else(|| py.None().into_bound(py)) - } else { - py.None().into_bound(py) - }; - let error: Bound<'_, PyAny> = error_class(py, ErrorId::CompileInput)?.call( - (lines.join("\n"),), - Some(&keywords( - py, - &[("code", native_field(first, "code")?), ("help", help)], - )?), - )?; - Err(PyErr::from_value(error)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compile_project_phase, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs deleted file mode 100644 index c2ca68e69e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Small pipeline entry points: analysis selection, project graphs and pipeline results. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyFloat}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::graph::models::{CompiledProjectGraph, NativeProjectGraph}; - -/// Discovered Python node functions whose selectors need the Python graph. -const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; -/// Plan entry lists that are runtime work. -const PLAN_WORK_ENTRIES: [&str; 6] = [ - "model_entries", - "seed_entries", - "source_load_entries", - "function_entries", - "audit_entries", - "test_entries", -]; - -/// Limit analysis when selectors cannot resolve through Python graph nodes. -#[pyfunction] -#[pyo3(signature = (**options))] -fn resolve_compile_analysis_selection<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_compile_analysis_selection", - options, - (&["options", "discovered_inputs"], &[]), - )?; - compile_analysis_selection( - &arguments.value("options")?, - &arguments.value("discovered_inputs")?, - ) -} - -/// `resolve_compile_analysis_selection` with its arguments. -pub(crate) fn compile_analysis_selection<'py>( - options: &Bound<'py, PyAny>, - discovered: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = options.py(); - let mut has_python_nodes: bool = false; - for name in PYTHON_NODE_FUNCTIONS { - if native_field(discovered, name)?.is_truthy()? { - has_python_nodes = true; - break; - } - } - let no_cache: Bound<'py, PyAny> = native_field(options, "no_cache")?; - if (native_field(options, "resolve_python_run_selectors")?.is_truthy()? && has_python_nodes) - || native_field(options, "sql_test_selection")? - .getattr("names")? - .is_truthy()? - { - return new_record_with( - py, - RecordId::CompileAnalysisSelection, - &[("no_cache", no_cache)], - ); - } - new_record_with( - py, - RecordId::CompileAnalysisSelection, - &[ - ("select", native_field(options, "select")?), - ("exclude", native_field(options, "exclude")?), - ( - "auto_load_sources", - native_field(options, "auto_load_sources")?, - ), - ("no_cache", no_cache), - ], - ) -} - -/// Build a static dependency graph for an already-compiled project. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_project_graph_from_compiled_project<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_project_graph_from_compiled_project", - options, - (&["project"], &[]), - )?; - project_graph(&arguments.value("project")?) -} - -/// The project's natively held graph; dict views are built only when a boundary reads them. -pub(crate) fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let native: Bound<'py, NativeProjectGraph> = - native_field(project, "lineage_graph")?.cast_into()?; - Ok(Bound::new( - project.py(), - CompiledProjectGraph::new(project.clone().unbind(), native.unbind()), - )? - .into_any()) -} - -/// Return whether a planned build has runtime work to execute. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn plan_has_executable_work<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "plan_has_executable_work", - options, - (&["plan"], &["python_plan_entries"]), - )?; - let plan: Bound<'py, PyAny> = arguments.value("plan")?; - for name in PLAN_WORK_ENTRIES { - if native_field(&plan, name)?.is_truthy()? { - return Ok(true); - } - } - if !arguments.passed("python_plan_entries")? { - return Ok(false); - } - arguments.value("python_plan_entries")?.is_truthy() -} - -/// Wrap a focused command projection in the established pipeline envelope. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_static_pipeline_result<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_static_pipeline_result", - options, - (&["context", "plan_output"], &[]), - )?; - let context: Bound<'py, PyAny> = arguments.value("context")?; - new_record_with( - py, - RecordId::CompilePipelineResult, - &[ - ("project", native_field(&context, "project")?), - ("plan_output", arguments.value("plan_output")?), - ( - "compile_seconds", - native_field(&context, "compile_seconds")?, - ), - ("planning_seconds", PyFloat::new(py, 0.0).into_any()), - ], - ) -} - -/// Compile a project for consumers that construct their own focused plans. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn run_compile_only_pipeline<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "run_compile_only_pipeline", - options, - ( - &["discovered_inputs", "adapter", "options"], - &["on_progress"], - ), - )?; - let compiled: Bound<'py, PyAny> = compile_project_phase( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("options", arguments.value("options")?), - ("on_progress", arguments.value("on_progress")?), - ], - )?), - )?; - new_record_with( - py, - RecordId::CompilePipelineResult, - &[ - ("project", native_field(&compiled, "project")?), - ( - "plan_output", - new_record(py, RecordId::PlanOutput, (), None)?, - ), - ( - "compile_seconds", - native_field(&compiled, "compile_seconds")?, - ), - ("planning_seconds", PyFloat::new(py, 0.0).into_any()), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - resolve_compile_analysis_selection, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_project_graph_from_compiled_project, - module - )?)?; - module.add_function(wrap_pyfunction!(plan_has_executable_work, module)?)?; - module.add_function(wrap_pyfunction!(build_static_pipeline_result, module)?)?; - module.add_function(wrap_pyfunction!(run_compile_only_pipeline, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 131bb1acfe..b7eebb23d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -8,23 +8,9 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; -use crate::bindings::_helpers::compile_driver::phases::monotonic; -use crate::bindings::_helpers::compile_driver::pipeline::{ - compiled_phase, progress, timing_tracker, -}; -use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; use crate::bindings::_helpers::compile_driver::plan_result::{ PlanningInputs, connection_with_hooks, pipeline_result, }; -use crate::bindings::_helpers::compile_driver::selection_phases::{ - apply_unit_test_selectors, resolve_configured_rule_selection, -}; -use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; -use crate::bindings::_helpers::rules::constants::DEFAULT_RULES_NOTICE_ATTR; -use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; -use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; -use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -35,6 +21,20 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::compiled_phase::compiled_phase; +use sqlbuild_python_compile::compile_driver::main::setup::progress::progress; +use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; +use sqlbuild_python_compile::compile_driver::main::setup::timing_tracker::timing_tracker; +use sqlbuild_python_compile::compile_driver::main::stages::apply_unit_test_selectors::apply_unit_test_selectors; +use sqlbuild_python_compile::compile_driver::main::stages::monotonic::monotonic; +use sqlbuild_python_compile::compile_driver::main::stages::resolve_configured_rule_selection::resolve_configured_rule_selection; +use sqlbuild_python_compile::frontier::main::write_plan_capture::write_plan_capture; +use sqlbuild_python_compile::rules::constants::DEFAULT_RULES_NOTICE_ATTR; +use sqlbuild_python_compile::rules::main::checking::format_rules_result::format_rules_result; +use sqlbuild_python_compile::rules::main::checking::rules_progress::rules_progress; +use sqlbuild_python_compile::rules::main::configuration::load_rules_config_with_cache::load_rules_config_with_cache; +use sqlbuild_python_compile::rules::main::rule::evaluate_rules::evaluate_rules; +use sqlbuild_python_compile::rules::models::EarlySqlLint; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 22c26a8fa0..1c45c533c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -2,13 +2,12 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compile_driver::deferral::build_deferred_locations; -use crate::bindings::_helpers::compile_driver::deferral::gather_deferred_relations; -use crate::bindings::_helpers::compile_driver::deferral::resolve_deferred_target_config; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; -use crate::bindings::_helpers::compile_driver::python_plan_entries::{ - build_python_run_plan_outputs, python_read_source_names_for, -}; +use sqlbuild_python_compile::compile_driver::main::stages::build_deferred_locations::build_deferred_locations; +use sqlbuild_python_compile::compile_driver::main::stages::gather_deferred_relations::gather_deferred_relations; +use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_target_config::resolve_deferred_target_config; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; +use sqlbuild_python_compile::compile_driver::main::setup::build_python_run_plan_outputs::build_python_run_plan_outputs; +use sqlbuild_python_compile::compile_driver::main::setup::python_read_source_names_for::python_read_source_names_for; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -24,12 +23,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::phases::monotonic; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::monotonic::monotonic; use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; const SINGLE_CONNECTION: i64 = 1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 9fa5fc9389..51121cd798 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -4,29 +4,44 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use sqlbuild_python_base::compile_context::main::diagnostic_collector::register_diagnostic_collector::register_diagnostic_collector; use sqlbuild_python_base::compile_context::main::input_reads::register_input_reads::register_input_reads; -use crate::bindings::_helpers::compile_driver::{analysis, analysis_cache, assembly, command, deferral, diff, discovery, inputs, pipeline, pipeline_entries, plan_pipeline, plan_result, project, python_plan_entries, relation_targets, selection_phases, static_context, targets}; +use crate::bindings::_helpers::compile_driver::{plan_pipeline, plan_result, static_context}; +use sqlbuild_python_compile::compile_driver::main::stages::register_analysis::register_analysis; +use sqlbuild_python_compile::compile_driver::main::stages::register_analysis_cache::register_analysis_cache; +use sqlbuild_python_compile::compile_driver::main::stages::register_assembly::register_assembly; +use sqlbuild_python_compile::compile_driver::main::stages::register_command::register_command; +use sqlbuild_python_compile::compile_driver::main::stages::register_deferral::register_deferral; +use sqlbuild_python_compile::compile_driver::main::stages::register_diff::register_diff; +use sqlbuild_python_compile::compile_driver::main::stages::register_discovery::register_discovery; +use sqlbuild_python_compile::compile_driver::main::setup::register_inputs::register_inputs; +use sqlbuild_python_compile::compile_driver::main::setup::register_pipeline::register_pipeline; +use sqlbuild_python_compile::compile_driver::main::setup::register_pipeline_entries::register_pipeline_entries; +use sqlbuild_python_compile::compile_driver::main::setup::register_project::register_project; +use sqlbuild_python_compile::compile_driver::main::setup::register_python_plan_entries::register_python_plan_entries; +use sqlbuild_python_compile::compile_driver::main::setup::register_relation_targets::register_relation_targets; +use sqlbuild_python_compile::compile_driver::main::stages::register_selection_phases::register_selection_phases; +use sqlbuild_python_compile::compile_driver::main::setup::register_targets::register_targets; use sqlbuild_python_base::compile_driver::main::register_stage_work::register_stage_work; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - analysis::register(module)?; - analysis_cache::register(module)?; - assembly::register(module)?; - command::register(module)?; - project::register(module)?; - inputs::register(module)?; + register_analysis(module)?; + register_analysis_cache(module)?; + register_assembly(module)?; + register_command(module)?; + register_project(module)?; + register_inputs(module)?; register_input_reads(module)?; - discovery::register(module)?; - pipeline::register(module)?; + register_discovery(module)?; + register_pipeline(module)?; plan_pipeline::register(module)?; - pipeline_entries::register(module)?; + register_pipeline_entries(module)?; plan_result::register(module)?; - deferral::register(module)?; + register_deferral(module)?; register_diagnostic_collector(module)?; - diff::register(module)?; - python_plan_entries::register(module)?; - relation_targets::register(module)?; - selection_phases::register(module)?; + register_diff(module)?; + register_python_plan_entries(module)?; + register_relation_targets(module)?; + register_selection_phases(module)?; static_context::register(module)?; register_stage_work(module)?; - targets::register(module) + register_targets(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs deleted file mode 100644 index 1ec02765cb..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Unit-test selector splitting, analysis and rule selection, and static command compilation. - -use std::sync::Arc; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyFrozenSet; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_driver::assembly::assemble_project; -use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -use sqlbuild_python_project::planner_scope::models::ScopeData; -use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; - -const DATACLASSES_MODULE: &str = "dataclasses"; - -/// Whether a selection names nothing to select or exclude. -fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { - Ok(!native_field(selection, "select")?.is_truthy()? - && !native_field(selection, "exclude")?.is_truthy()?) -} - -/// Options whose selectors name only resources, with unit tests selected apart. -pub(crate) fn unit_test_selectors_applied<'py>( - options: &Bound<'py, PyAny>, - discovered: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = discovered.py(); - let resolved: Bound<'py, PyAny> = if options.is_none() { - new_record(py, RecordId::CompilePipelineOptions, (), None)? - } else { - options.clone() - }; - if unselective(&resolved)? { - return Ok(resolved); - } - let split: Bound<'py, PyAny> = split_project_unit_test_selectors( - &native_field(&resolved, "select")?, - &native_field(&resolved, "exclude")?, - discovered, - native_field(&resolved, "accepts_unit_test_selectors")?.extract()?, - )?; - imported(py, DATACLASSES_MODULE, "replace")?.call( - (resolved,), - Some(&keywords( - py, - &[ - ("select", native_field(&split, "select")?), - ("exclude", native_field(&split, "exclude")?), - ( - "sql_test_selection", - native_field(&split, "sql_test_selection")?, - ), - ], - )?), - ) -} - -/// The planner scopes `selection` resolves over `project`. -fn planner_scopes<'py>( - project: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project.py(); - resolve_planner_scopes( - project, - &new_record_with( - py, - RecordId::PlannerSelection, - &[ - ("select", native_field(selection, "select")?), - ("exclude", native_field(selection, "exclude")?), - ], - )?, - &new_record_with( - py, - RecordId::PlannerPolicies, - &[( - "auto_load_sources", - native_field(selection, "auto_load_sources")?, - )], - )?, - ) -} - -/// The native scope whose models receive deep analysis. -fn analysis_scope(scopes: &Bound<'_, PyAny>) -> PyResult> { - scope_data(&native_field(scopes, "stale_warning_scope")?) -} - -/// Return options whose selectors name only resources, with unit tests selected apart. -#[pyfunction] -#[pyo3(signature = (*, options, discovered_inputs))] -pub(crate) fn apply_unit_test_selectors<'py>( - options: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - unit_test_selectors_applied(options, discovered_inputs) -} - -/// Align configured-rule subjects with models that received deep SQL analysis. -#[pyfunction] -#[pyo3(signature = (*, options, discovered_inputs, project))] -pub(crate) fn resolve_configured_rule_selection<'py>( - options: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = options.py(); - if !native_field(options, "sql_test_selection")? - .getattr("models_selected")? - .is_truthy()? - { - return Ok(PyFrozenSet::empty(py)?.into_any()); - } - let selection: Bound<'py, PyAny> = compile_analysis_selection(options, discovered_inputs)?; - if selection.is_none() || unselective(&selection)? { - return Ok(py.None().into_bound(py)); - } - let scope = analysis_scope(&planner_scopes(project, &selection)?)?; - let keys: Vec> = scope - .models - .values() - .map(|model| native_field(&model.bind(py), "key")) - .collect::>()?; - Ok(PyFrozenSet::new(py, keys)?.into_any()) -} - -/// Return the models selected for deep analysis, or None to analyse every model. -#[pyfunction] -#[pyo3(signature = (*, compile_inputs, inference_profile, selection))] -pub(crate) fn resolve_analysis_model_names<'py>( - compile_inputs: &Bound<'py, PyAny>, - inference_profile: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = compile_inputs.py(); - if selection.is_none() || unselective(selection)? { - return Ok(py.None().into_bound(py)); - } - let structural_project: Bound<'py, PyAny> = assemble_project( - py, - Some(keywords( - py, - &[ - ("inputs", compile_inputs.clone()), - ("inference_profile", inference_profile.clone()), - ( - "skip_column_inference", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), - ], - )?), - )?; - let scope = analysis_scope(&planner_scopes(&structural_project, selection)?)?; - Ok(PyFrozenSet::new(py, scope.models.keys())?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(apply_unit_test_selectors, module)?)?; - module.add_function(wrap_pyfunction!(resolve_configured_rule_selection, module)?)?; - module.add_function(wrap_pyfunction!(resolve_analysis_model_names, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 446d4abf23..6def3694f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -4,12 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::deferral::{ - build_deferred_locations, resolve_deferred_target_config, -}; -use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; -use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; +use sqlbuild_python_compile::compile_driver::main::stages::build_deferred_locations::build_deferred_locations; +use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_target_config::resolve_deferred_target_config; +use sqlbuild_python_compile::compile_driver::main::setup::compile_project_phase::compile_project_phase; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; +use sqlbuild_python_compile::compile_driver::main::stages::unit_test_selectors_applied::unit_test_selectors_applied; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs deleted file mode 100644 index b32e659958..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! Write, clean and publish compiled artifacts natively, raising the errors Python raised. - -use std::collections::HashSet; -use std::panic::resume_unwind; -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{Receiver, Sender, channel}; -use std::sync::{Arc, Condvar, Mutex, PoisonError}; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -use pyo3::exceptions::PyOSError; -use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::PyBytes; -use sqlbuild_cache::artifacts::errors::ArtifactError; -use sqlbuild_cache::artifacts::main::publish_staged::publish_staged; -use sqlbuild_cache::artifacts::main::remove_stale::remove_stale; -use sqlbuild_cache::artifacts::main::remove_stale_unless_unchanged::remove_stale_unless_unchanged; -use sqlbuild_cache::artifacts::main::staged_files::staged_files; -use sqlbuild_cache::artifacts::main::write_artifacts::write_artifacts; -use sqlbuild_cache::artifacts::models::{Publication, WrittenArtifacts}; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; - -type PublishedRow = (String, Vec<(String, String)>); - -/// One write group's `(path, contents)` artifacts and whether unchanged files are kept. -type ArtifactGroup = (Vec<(PathBuf, Vec)>, bool); - -const WRITER_THREAD: &str = "sqlbuild-artifact-writer"; -const WRITER_POLL: Duration = Duration::from_millis(50); - -/// Write each `(path, contents)` artifact; returns how many were written and left unchanged. -pub(crate) fn write_compiled_artifacts( - py: Python<'_>, - files: Vec<(PathBuf, Vec)>, - check_existing: bool, -) -> PyResult<(usize, usize)> { - compiler_guard(|| { - let outcome: Result = - py.detach(|| write_artifacts(&files, check_existing)); - let written: WrittenArtifacts = outcome.map_err(|error| artifact_error(py, error))?; - Ok((written.written, written.unchanged)) - }) -} - -/// Artifact groups written one after another on their own thread, stopping at the first error. -pub(crate) struct BackgroundArtifactWriter { - groups: Sender, - /// Set to skip every group not yet started; the group being written still completes. - stop: Arc, - /// Marked once the writer returns, so a wait can check for signals meanwhile. - finished: Arc, - handle: JoinHandle<(Result<(), ArtifactError>, Duration)>, -} - -/// Write each group in order until one fails or a stop is requested between groups. -fn write_groups( - groups: &Receiver, - stop: &AtomicBool, -) -> (Result<(), ArtifactError>, Duration) { - let mut elapsed: Duration = Duration::ZERO; - for (files, check_existing) in groups { - if stop.load(Ordering::SeqCst) { - break; - } - let started: Instant = Instant::now(); - let written: Result = - write_artifacts(&files, check_existing); - elapsed += started.elapsed(); - if let Err(error) = written { - return (Err(error), elapsed); - } - } - (Ok(()), elapsed) -} - -impl BackgroundArtifactWriter { - /// Start the writer thread, or `None` when no thread could start. - pub(crate) fn start() -> Option { - let (groups, received) = channel(); - let finished: Arc = Arc::new(WriterDone::default()); - let marker: DoneMarker = DoneMarker(Arc::clone(&finished)); - let stop: Arc = Arc::new(AtomicBool::new(false)); - let stopping: Arc = Arc::clone(&stop); - let spawned = std::thread::Builder::new() - .name(WRITER_THREAD.to_owned()) - .spawn(move || { - let _marker: DoneMarker = marker; - write_groups(&received, &stopping) - }); - match spawned { - Ok(handle) => Some(Self { - groups, - stop, - finished, - handle, - }), - Err(_) => None, - } - } - - /// Queue one group behind the groups already sent; a writer that stopped drops it. - pub(crate) fn send(&self, files: Vec<(PathBuf, Vec)>, check_existing: bool) { - let _ = self.groups.send((files, check_existing)); - } - - /// Wait without the GIL for every sent group; a signal stops it after the current group. - pub(crate) fn finish(self, py: Python<'_>) -> PyResult { - let Self { - groups, - stop, - finished, - handle, - } = self; - drop(groups); - if let Err(interrupt) = wait_for_writer(py, &finished) { - stop.store(true, Ordering::SeqCst); - let _ = py.detach(|| handle.join()); - return Err(interrupt); - } - compiler_guard(|| { - let joined = py.detach(|| handle.join()); - let (outcome, elapsed) = joined.unwrap_or_else(|payload| resume_unwind(payload)); - outcome.map_err(|error| artifact_error(py, error))?; - Ok(elapsed) - }) - } - - /// Stop after the group being written and wait for it without the GIL, ignoring the outcome. - pub(crate) fn stop(self, py: Python<'_>) { - let Self { - groups, - stop, - finished, - handle, - } = self; - stop.store(true, Ordering::SeqCst); - drop(groups); - drop(finished); - let _ = py.detach(|| handle.join()); - } -} - -/// Whether the writer thread has returned, with a condition variable to wait on it. -#[derive(Default)] -struct WriterDone { - finished: Mutex, - changed: Condvar, -} - -/// Marks the writer done when the writer thread returns or unwinds. -struct DoneMarker(Arc); - -impl Drop for DoneMarker { - fn drop(&mut self) { - *self - .0 - .finished - .lock() - .unwrap_or_else(PoisonError::into_inner) = true; - self.0.changed.notify_all(); - } -} - -impl WriterDone { - /// Whether the writer returned within `timeout`. - fn finished_within(&self, timeout: Duration) -> bool { - let finished = self.finished.lock().unwrap_or_else(PoisonError::into_inner); - let (finished, _) = self - .changed - .wait_timeout_while(finished, timeout, |finished| !*finished) - .unwrap_or_else(PoisonError::into_inner); - *finished - } -} - -/// Wait without the GIL until the writer returns, checking for signals between waits. -fn wait_for_writer(py: Python<'_>, finished: &WriterDone) -> PyResult<()> { - while !py.detach(|| finished.finished_within(WRITER_POLL)) { - py.check_signals()?; - } - Ok(()) -} - -/// Delete unmanaged compiled files and empty directories; returns how many files were removed. -pub(crate) fn remove_stale_artifacts( - py: Python<'_>, - compiled_dir: PathBuf, - managed: Vec, - stamp: Option<(PathBuf, i64)>, -) -> PyResult { - compiler_guard(|| { - let managed: HashSet = managed.into_iter().collect(); - py.detach(|| match &stamp { - Some((stamp_path, racy_window_ns)) => { - remove_stale_unless_unchanged(&compiled_dir, &managed, stamp_path, *racy_window_ns) - } - None => remove_stale(&compiled_dir, &managed), - }) - .map_err(|error| artifact_error(py, error)) - }) -} - -/// Publish staged artifacts: `("changed", [])`, `("tree", [])` or `("files", [(source, path)])`. -pub(crate) fn publish_staged_artifacts( - py: Python<'_>, - staged_dir: PathBuf, - compiled_dir: PathBuf, - expected: Vec, -) -> PyResult { - compiler_guard(|| { - let expected: HashSet = expected.into_iter().collect(); - let publication: Publication = py - .detach(|| publish_staged(&staged_dir, &compiled_dir, &expected)) - .map_err(|error| artifact_error(py, error))?; - Ok(match publication { - Publication::StagedChanged => ("changed".to_owned(), Vec::new()), - Publication::MovedTree => ("tree".to_owned(), Vec::new()), - Publication::Files(files) => ( - "files".to_owned(), - files - .into_iter() - .map(|(source, path)| (lossy(&source), lossy(&path))) - .collect(), - ), - }) - }) -} - -/// Every staged artifact's path relative to `staged_dir`. -pub(crate) fn staged_artifact_files(py: Python<'_>, staged_dir: PathBuf) -> PyResult> { - compiler_guard(|| { - let files: Vec = py.detach(|| staged_files(&staged_dir)); - Ok(files.iter().map(|path| lossy(path)).collect()) - }) -} - -fn lossy(path: &std::path::Path) -> String { - path.to_string_lossy().into_owned() -} - -/// The exception Python's writer raises for the same failure. -fn artifact_error(py: Python<'_>, error: ArtifactError) -> PyErr { - match error { - ArtifactError::Io { path, error } => match error.raw_os_error() { - Some(code) => PyOSError::new_err((code, strerror(py, code), lossy(&path))), - None => PyOSError::new_err(error.to_string()), - }, - ArtifactError::Move { - source, - path, - error, - } => match error.raw_os_error() { - Some(code) => PyOSError::new_err(( - code, - strerror(py, code), - lossy(&source), - py.None(), - lossy(&path), - )), - None => PyOSError::new_err(error.to_string()), - }, - ArtifactError::ExistingNotUtf8 { path } => existing_decode_error(py, &path), - } -} - -fn strerror(py: Python<'_>, code: i32) -> String { - py.import("os") - .and_then(|os| os.call_method1("strerror", (code,))) - .and_then(|text| text.extract::()) - .unwrap_or_default() -} - -/// Decode the existing file in Python so the error is exactly the one `bytes.decode` raises. -fn existing_decode_error(py: Python<'_>, path: &PathBuf) -> PyErr { - let bytes: Vec = match std::fs::read(path) { - Ok(bytes) => bytes, - Err(error) => return PyOSError::new_err(error.to_string()), - }; - match PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",)) { - Err(error) => error, - Ok(_) => PyOSError::new_err(format!("compiled artifact '{}' changed", path.display())), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs index 6ebbc96c6e..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs @@ -1,4 +1 @@ -pub(crate) mod artifacts; pub(crate) mod registration; -pub(crate) mod reuse; -pub(crate) mod runtime_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs index 5fe2825d3a..4370d839c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_outputs::{reuse, runtime_target}; use sqlbuild_python_base::compile_outputs::main::register_json_report::register_json_report; +use sqlbuild_python_compile::compile_outputs::main::register_reuse::register_reuse; +use sqlbuild_python_compile::compile_outputs::main::register_runtime_target::register_runtime_target; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_json_report(module)?; - reuse::register(module)?; - runtime_target::register(module) + register_reuse(module)?; + register_runtime_target(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs deleted file mode 100644 index 79a17820b5..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ /dev/null @@ -1,360 +0,0 @@ -//! Whole-project compile reuse on the shared native store, driven by the Python host's facts. - -use sqlbuild_python_base::positional_record; -use std::collections::{BTreeMap, HashMap}; -use std::ffi::{OsStr, OsString}; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{FromPyObject, IntoPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; -use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use sqlbuild_cache::project_reuse::constants::{NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX}; -use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; -use sqlbuild_cache::project_reuse::main::pending_digest_bytes::pending_digest_bytes; -use sqlbuild_cache::project_reuse::main::record_reuse::record_reuse; -use sqlbuild_cache::project_reuse::main::replay_stdout::replay_stdout; -use sqlbuild_cache::project_reuse::models::{ - CompileRecord, HostIdentity, RecordOutcome, RecordResult, RecordedArtifact, ReuseAttempt, - ReuseCheck, ReuseOutcome, ReuseRules, SettingsInputs, StoredOutput, -}; -use sqlbuild_cache::project_snapshot::main::path_text::path_text; -use sqlbuild_cache::project_snapshot::models::SnapshotRules; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; - -positional_record! { - extract; - /// Settings case sensitivity, names, prefixes, env files and secrets directories. - pub(crate) struct SettingsRow { - pub(crate) case_sensitive: bool, - pub(crate) names: Vec, - pub(crate) prefixes: Vec, - pub(crate) env_files: Vec, - pub(crate) secrets_dirs: Vec, - } -} -type ReplayRow = (String, Vec, i64); -type LaunchIdentityRow = (String, BTreeMap, Vec); -/// A written artifact's digest, or a kept artifact's size and mtime. -pub(crate) type ArtifactRow = (Option, Option, Option); - -/// The host constants the walk, environment and store checks use. -#[derive(FromPyObject, IntoPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ReuseRulesPy { - pub(crate) excluded: Vec, - pub(crate) excluded_root: Vec, - pub(crate) presence_suffixes: Vec, - pub(crate) output_files: Vec<(u64, u64)>, - pub(crate) tracked_prefixes: Vec, - pub(crate) untracked_names: Vec, - pub(crate) disabling_variables: Vec<(String, String)>, - pub(crate) config_filenames: Vec, - pub(crate) racy_window_ns: i64, - pub(crate) max_stored_entries: usize, - pub(crate) max_entry_bytes: usize, - pub(crate) missing_environment_value: String, - pub(crate) missing_file_digest: String, - pub(crate) missing_path_mtime_ns: i64, - pub(crate) project_root_path_mtime_ns: i64, - pub(crate) compiled_root: PathBuf, - pub(crate) retired_directories: Vec, -} - -impl From for ReuseRules { - fn from(rules: ReuseRulesPy) -> Self { - Self { - snapshot: SnapshotRules { - excluded: rules.excluded, - excluded_root: rules.excluded_root, - presence_suffixes: rules.presence_suffixes, - output_files: rules.output_files, - }, - tracked_prefixes: rules.tracked_prefixes, - untracked_names: rules.untracked_names, - disabling_variables: rules.disabling_variables, - config_filenames: rules.config_filenames, - racy_window_ns: rules.racy_window_ns, - max_stored_entries: rules.max_stored_entries, - max_entry_bytes: rules.max_entry_bytes, - missing_environment_value: rules.missing_environment_value, - missing_file_digest: rules.missing_file_digest, - missing_path_mtime_ns: rules.missing_path_mtime_ns, - project_root_path_mtime_ns: rules.project_root_path_mtime_ns, - compiled_root: rules.compiled_root, - retired_directories: rules.retired_directories, - } - } -} - -/// One reuse check: where the project and its store slot are, and the host's identity facts. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ReuseCheckPy { - pub(crate) project_dir: PathBuf, - pub(crate) store_directory: PathBuf, - pub(crate) selected_target: Option, - pub(crate) invocation: Vec, - pub(crate) runtime: BTreeMap, - pub(crate) search_path: Vec, - pub(crate) bypass_requested: bool, - pub(crate) json_output: bool, - pub(crate) rules: ReuseRulesPy, -} - -/// What a full compile leaves for recording; `artifacts` maps a path to `(digest, size, mtime)`. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct CompileRecordPy { - pub(crate) stdout: Option, - pub(crate) stderr_lines: Vec, - pub(crate) exit_code: i64, - pub(crate) compile_cache_enabled: bool, - pub(crate) read_run_id: bool, - pub(crate) settings_inputs: Option>, - pub(crate) template_environment_names: Vec, - pub(crate) module_paths: Vec, - pub(crate) artifacts: HashMap, - pub(crate) artifacts_written: bool, - pub(crate) dag_path: Option, - pub(crate) json_output: bool, - pub(crate) rules: ReuseRulesPy, -} - -/// One invocation's native reuse state, from the check to the record. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeReuseAttempt { - attempt: Mutex, -} - -fn poisoned(_: T) -> pyo3::PyErr { - PyValueError::new_err("compile reuse attempt is poisoned") -} - -impl NativeReuseAttempt { - /// Run `body` on the native attempt. - pub(crate) fn with_attempt(&self, body: impl FnOnce(&ReuseAttempt) -> T) -> PyResult { - let attempt = self.attempt.lock().map_err(poisoned)?; - Ok(body(&attempt)) - } -} - -#[pymethods] -impl NativeReuseAttempt { - /// `"hit"`, `"miss"` or `"bypass"`. - fn outcome(&self) -> PyResult<&'static str> { - Ok(match self.attempt.lock().map_err(poisoned)?.outcome { - ReuseOutcome::Hit => "hit", - ReuseOutcome::Miss => "miss", - ReuseOutcome::Bypass => "bypass", - }) - } - - /// The store file holding this invocation's slot. - fn store_path(&self) -> PyResult { - let attempt = self.attempt.lock().map_err(poisoned)?; - Ok(attempt.store_path.to_string_lossy().into_owned()) - } - - /// Whether a stored compile matched but could not be replayed. - fn replay_failed(&self) -> PyResult { - Ok(self.attempt.lock().map_err(poisoned)?.replay_failed) - } - - /// How many project paths the check stamped. - fn snapshot_paths(&self) -> PyResult { - Ok(self.attempt.lock().map_err(poisoned)?.snapshot.len()) - } - - /// Bytes recording would hash for files whose stamp moved. - fn pending_digest_bytes(&self) -> PyResult { - let attempt = self.attempt.lock().map_err(poisoned)?; - Ok(pending_digest_bytes(&attempt)) - } - - /// On a hit, the stored report with these timings plus its stderr lines and exit code. - fn replay(&self, timings: Vec<(String, i64)>) -> PyResult> { - let attempt = self.attempt.lock().map_err(poisoned)?; - let Some(output) = attempt.replay.as_ref() else { - return Ok(None); - }; - let output: &StoredOutput = output; - Ok(replay_stdout(output, &timings) - .map(|stdout| (stdout, output.stderr_lines.clone(), output.exit_code))) - } - - /// The invocation digest, runtime and search path the check compared, for a launch record. - fn launch_identity(&self) -> PyResult { - let attempt = self.attempt.lock().map_err(poisoned)?; - let identity: &HostIdentity = &attempt.identity; - Ok(( - identity.invocation_digest.clone(), - identity.runtime.clone(), - identity.search_path.clone(), - )) - } - - /// How many project files the check read for content digests. - fn digested_files(&self) -> PyResult { - Ok(self.attempt.lock().map_err(poisoned)?.digested_files) - } - - /// Store this compile or clear the slot; a failure clears the slot and raises `OSError`. - pub(crate) fn record( - &self, - py: Python<'_>, - record: CompileRecordPy, - ) -> PyResult<(&'static str, usize)> { - compiler_guard(|| { - let rules: ReuseRules = record.rules.into(); - let mut compile: CompileRecord = CompileRecord { - stdout: record.stdout, - stderr_lines: record.stderr_lines, - exit_code: record.exit_code, - compile_cache_enabled: record.compile_cache_enabled, - read_run_id: record.read_run_id, - settings: record - .settings_inputs - .map(|rows| rows.into_iter().map(settings_inputs).collect()) - .ok_or_else(String::new), - template_environment_names: record.template_environment_names, - module_paths: texts(&record.module_paths), - artifacts: record - .artifacts - .into_iter() - .map(|(path, artifact)| (path, recorded_artifact(artifact))) - .collect(), - artifacts_written: record.artifacts_written, - dag_path: record.dag_path, - json_output: record.json_output, - image: sqlbuild_python_base::compile_context::main::delta::take_pending_image::take_pending_image(), - vouched_target_stamps: None, - }; - if let Some(merge) = - sqlbuild_python_base::compile_context::main::delta::take_record_merge::take_record_merge() - { - for stamp in &merge.kept { - compile - .artifacts - .entry(stamp.relative_path.clone()) - .or_insert(RecordedArtifact::Kept { - size: stamp.size, - mtime_ns: stamp.mtime_ns, - }); - } - compile.vouched_target_stamps = Some(merge.kept); - compile - .template_environment_names - .extend(merge.environment_names); - compile.template_environment_names.sort(); - compile.template_environment_names.dedup(); - compile.module_paths.extend(merge.module_paths); - } - let attempt = self.attempt.lock().map_err(poisoned)?; - let result: RecordResult = py - .detach(|| record_reuse(&attempt, &compile, &rules)) - .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.message))?; - let outcome: &'static str = match result.outcome { - RecordOutcome::NotAttempted => "not_attempted", - RecordOutcome::Stored => "stored", - RecordOutcome::Removed => "removed", - RecordOutcome::Skipped => "skipped", - }; - Ok((outcome, result.digested_files)) - }) - } -} - -fn settings_inputs(row: SettingsRow) -> SettingsInputs { - let SettingsRow { - case_sensitive, - names, - prefixes, - env_files, - secrets_dirs, - } = row; - SettingsInputs { - case_sensitive, - names, - prefixes, - env_files, - secrets_dirs, - } -} - -fn recorded_artifact(artifact: ArtifactRow) -> RecordedArtifact { - match artifact { - (Some(digest), _, _) => RecordedArtifact::Written { digest }, - (None, size, mtime_ns) => RecordedArtifact::Kept { - size: size.unwrap_or_default(), - mtime_ns: mtime_ns.unwrap_or_default(), - }, - } -} - -/// Check the stored compile for this invocation, replaying it on a hit. -#[pyfunction] -pub(crate) fn check_compile_reuse( - py: Python<'_>, - check: ReuseCheckPy, -) -> PyResult { - compiler_guard(|| { - let rules: ReuseRules = check.rules.into(); - let request: ReuseCheck<'_> = ReuseCheck { - project_dir: check.project_dir, - store_path: store_slot(&check.store_directory, check.selected_target.as_deref()), - identity: HostIdentity { - invocation_digest: native_digest(&check.invocation), - runtime: check - .runtime - .iter() - .map(|(name, value)| (name.clone(), path_text(value))) - .collect(), - search_path: texts(&check.search_path), - }, - bypass_requested: check.bypass_requested, - json_output: check.json_output, - rules: &rules, - }; - let attempt: ReuseAttempt = py.detach(|| check_reuse(request)); - Ok(NativeReuseAttempt { - attempt: Mutex::new(attempt), - }) - }) -} - -/// The native content digest of artifact bytes, as reuse verifies written artifacts. -pub(crate) fn artifact_digest(contents: &[u8]) -> String { - native_digest(contents) -} - -fn native_digest(contents: &[u8]) -> String { - format!( - "{NATIVE_DIGEST_PREFIX}{}", - hex_digest(&bytes_digest(contents)) - ) -} - -/// The one store file kept for the selected target. -fn store_slot(directory: &Path, selected_target: Option<&OsStr>) -> PathBuf { - let label: String = selected_target.map_or_else(String::new, |target| { - format!("target:{}", path_text(target)) - }); - directory.join(format!( - "{}.{REUSE_STORE_SUFFIX}", - hex_digest(&bytes_digest(label.as_bytes())) - )) -} - -/// Host strings that may carry undecodable bytes, as exact text. -fn texts(values: &[OsString]) -> Vec { - values.iter().map(|value| path_text(value)).collect() -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(check_compile_reuse, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs deleted file mode 100644 index 250d1058f8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! SQL test artifact paths, and the executed SQL and Python check results written to `target/run`. - -use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, -}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; -use pyo3::wrap_pyfunction; -use sqlbuild_analysis::target_paths::main::function_run_path::function_run_path; -use sqlbuild_analysis::target_paths::main::model_run_path::model_run_path; -use sqlbuild_analysis::target_paths::main::runtime_sql_text::runtime_sql_text; -use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; -use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; -use sqlbuild_core::text::main::python_isspace::python_isspace; -use sqlbuild_host::models::{ - BuildExecutionResult, LifeCycleEvent, LifeCycleEventKind, SqlTestExecutionResult, -}; - -use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::types::ReadResult; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const RUN_DIR: &str = "run"; -const TESTS_DIR: &str = "tests"; -const CHECKS_DIR: &str = "checks"; -const PYTHON_CHECKS_FILE: &str = "python_checks.json"; -const PYTHON_CHECK_KIND: &str = "python_check"; -const JSON_INDENT: i64 = 2; - -/// One test's identity as the test runtime target matches entries to results. -#[derive(Hash, PartialEq, Eq)] -struct TestKey { - name: String, - source_path: Option, - block_index: Option, - case_name: Option, -} - -fn posix(path: &Bound<'_, PyAny>) -> PyResult> { - if path.is_none() { - return Ok(None); - } - path.call_method0("as_posix")?.extract().map(Some) -} - -fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> { - imported(py, "pathlib", "Path")?.call1((text,)) -} - -fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - Ok(TestKey { - name: native_field(value, name_field)?.extract()?, - source_path: posix(&native_field(value, "source_path")?)?, - block_index: native_field(value, "block_index")?.extract()?, - case_name: native_field(value, "case_name")?.extract()?, - }) -} - -fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { - let TestKey { - name, - source_path, - block_index, - case_name, - } = test_key(test, "name")?; - Ok(SqlTestPathFacts { - name, - source_path, - block_index, - case_name, - model_names, - }) -} - -/// The target path of one planned SQL test, relative to its artifact directory. -pub(crate) fn planned_sql_test_path<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { - let mut model_names: Vec = Vec::new(); - for step in native_field(entry, "chain")?.try_iter()? { - model_names.push(native_field(&step?, "model_name")?.extract()?); - } - path_object( - entry.py(), - &sql_test_output_path(&path_facts(entry, model_names)?), - ) -} - -/// The target path of one compiled SQL test from its chain's model names. -pub(crate) fn compiled_sql_test_path( - test: &Bound<'_, PyAny>, - model_names: Vec, -) -> PyResult { - Ok(PathBuf::from(sql_test_output_path(&path_facts( - test, - model_names, - )?))) -} - -fn write_text(path: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { - let py: Python<'_> = path.py(); - let mkdir: Bound<'_, PyDict> = PyDict::new(py); - mkdir.set_item("parents", true)?; - mkdir.set_item("exist_ok", true)?; - native_field(path, "parent")?.call_method("mkdir", (), Some(&mkdir))?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("encoding", "utf-8")?; - path.call_method("write_text", (text,), Some(&options))?; - Ok(()) -} - -fn entry_paths( - entries: &Bound<'_, PyAny>, - path_of: impl Fn(&Bound<'_, PyAny>) -> PyResult, -) -> PyResult> { - let mut paths: HashMap = HashMap::new(); - for entry in entries.try_iter()? { - let entry: Bound<'_, PyAny> = entry?; - paths.insert(native_field(&entry, "name")?.extract()?, path_of(&entry)?); - } - Ok(paths) -} - -fn relative_posix(entry: &Bound<'_, PyAny>) -> PyResult { - native_field(entry, "relative_path")? - .call_method0("as_posix")? - .extract() -} - -/// Write executed model and function lifecycle SQL under `target/run`. -pub(crate) fn write_runtime_target( - target_dir: &Bound<'_, PyAny>, - plan: &Bound<'_, PyAny>, - result: &BuildExecutionResult>, -) -> PyResult<()> { - let run_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?; - let models: HashMap = - entry_paths(&native_field(plan, "model_entries")?, |entry| { - relative_posix(entry).map(|path| model_run_path(&path)) - })?; - let functions: HashMap = - entry_paths(&native_field(plan, "function_entries")?, |entry| { - Ok(function_run_path( - &relative_posix(entry)?, - &native_field(entry, "language")? - .getattr("value")? - .extract::()?, - )) - })?; - let models = result.model_results.iter().map(|model| { - ( - models.get(&model.model_name), - model.lifecycle_events.as_slice(), - ) - }); - let functions = result.function_results.iter().map(|function| { - ( - functions.get(&function.function_name), - function.lifecycle_events.as_slice(), - ) - }); - for (path, events) in models.chain(functions) { - let statements: Vec = events - .iter() - .filter(|event: &&LifeCycleEvent| event.kind == LifeCycleEventKind::Sql) - .map(|event| event.content.clone()) - .collect(); - if let Some(path) = path.filter(|_| !statements.is_empty()) { - write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; - } - } - Ok(()) -} - -/// Write the executed SQL unit-test comparison statements under `target/run/tests`. -pub(crate) fn write_test_runtime_target( - (target_dir, adapter): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - plan: &Bound<'_, PyAny>, - results: &[SqlTestExecutionResult>], -) -> PyResult<()> { - let mut executed: HashSet = HashSet::new(); - for result in results { - executed.insert(TestKey { - name: result.test_name.clone(), - source_path: result.source_path.as_ref().map(|path| { - path.to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }), - block_index: result.block_index, - case_name: result.case_name.clone(), - }); - } - let tests_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?.div(TESTS_DIR)?; - for entry in native_field(plan, "test_entries")?.try_iter()? { - let entry: Bound<'_, PyAny> = entry?; - if !executed.contains(&test_key(&entry, "name")?) { - continue; - } - let sql: String = single_comparison_sql(&entry, adapter)?; - write_text( - &tests_dir.div(planned_sql_test_path(&entry)?)?, - &format!("{}\n", sql.trim_end_matches(python_isspace)), - )?; - } - Ok(()) -} - -fn check_status(result: &Bound<'_, PyAny>) -> PyResult<&'static str> { - if native_field(result, "passed")?.is_truthy()? { - return Ok("pass"); - } - if native_field(result, "warned")?.is_truthy()? { - return Ok("warn"); - } - Ok("fail") -} - -/// Write Python check results under `target/run/checks` when any ran. -pub(crate) fn write_python_check_runtime_target( - target_dir: &Bound<'_, PyAny>, - results: &Bound<'_, PyAny>, -) -> PyResult<()> { - let py: Python<'_> = target_dir.py(); - let checks: Bound<'_, PyList> = PyList::empty(py); - for result in results.try_iter()? { - let result: Bound<'_, PyAny> = result?; - let name: Bound<'_, PyAny> = native_field(&result, "node_name")?; - let check: Bound<'_, PyDict> = PyDict::new(py); - check.set_item("kind", PYTHON_CHECK_KIND)?; - check.set_item("name", &name)?; - check.set_item("display_name", &name)?; - check.set_item("check_id", format!("{PYTHON_CHECK_KIND}:{}", name.str()?))?; - check.set_item("status", check_status(&result)?)?; - check.set_item("passed", native_field(&result, "passed")?)?; - check.set_item( - "severity", - native_field(&result, "severity")?.getattr("value")?, - )?; - check.set_item("message", native_field(&result, "message")?)?; - check.set_item("error_message", native_field(&result, "error_message")?)?; - check.set_item("metadata", native_field(&result, "metadata")?)?; - checks.append(check)?; - } - if checks.is_empty() { - return Ok(()); - } - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("checks", checks)?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("indent", JSON_INDENT)?; - let text: String = imported(py, "json", "dumps")? - .call((payload,), Some(&options))? - .extract()?; - write_text( - &target_dir - .div(RUN_DIR)? - .div(CHECKS_DIR)? - .div(PYTHON_CHECKS_FILE)?, - &format!("{text}\n"), - ) -} - -/// Write executed model and function lifecycle SQL under `target/run`. -#[pyfunction(name = "write_runtime_target")] -#[pyo3(signature = (*, target_dir, plan_output, result))] -fn write_runtime_target_binding( - target_dir: &Bound<'_, PyAny>, - plan_output: &Bound<'_, PyAny>, - result: &Bound<'_, PyAny>, -) -> PyResult<()> { - write_runtime_target( - target_dir, - plan_output, - &BuildExecutionResult::read(result)?, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(write_runtime_target_binding, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 557235e6ac..010058a310 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -20,9 +20,9 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index 9352b77d16..ee4ffdf568 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -4,8 +4,6 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::format_command::files::write_file_atomically; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -15,6 +13,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::refactor::main::edit_contract_model::edit_contract_model; use sqlbuild_python_front::refactor::main::edit_contract_source::edit_contract_source; use sqlbuild_python_front::refactor::main::remaining_contract_findings::remaining_contract_findings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 6b3a3776b8..c16138f37c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -11,7 +11,6 @@ use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::cost_report::functions::read_cost_run; use crate::bindings::_helpers::cost_report::functions::resolve_cost_run; use crate::bindings::_helpers::cost_report::functions::write_cost_run; @@ -23,6 +22,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs deleted file mode 100644 index cee82aa1ee..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! `build_dag_json`: the static DAG artifact of a compiled project graph. - -use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; -use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; -use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; -use sqlbuild_core::json::errors::JsonEmitError; - -use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_project::graph::models::CompiledProjectGraph; - -fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { - match error { - DagArtifactError::LoaderDestination(message) => error_class(py, ErrorId::SpecConfig) - .and_then(|class| class.call1((message,))) - .map_or_else(|error| error, PyErr::from_value), - DagArtifactError::MissingParameterType(name) => PyKeyError::new_err(name), - DagArtifactError::Unencodable(message) => PyTypeError::new_err(message), - DagArtifactError::Emit(JsonEmitError::NestingTooDeep) => { - PyRecursionError::new_err("maximum recursion depth exceeded") - } - DagArtifactError::Emit(error) => PyValueError::new_err(format!("{error:?}")), - } -} - -/// The DAG JSON of `graph` (a `ProjectGraph`) and the optional discovered Python node graph. -#[pyfunction] -#[pyo3(signature = (*, graph, project_name, python_graph=None))] -pub(crate) fn build_dag_json( - graph: &Bound<'_, PyAny>, - project_name: &str, - python_graph: Option<&Bound<'_, PyAny>>, -) -> PyResult { - compiler_guard(|| { - let py = graph.py(); - let graph: &Bound<'_, CompiledProjectGraph> = graph.cast()?; - let project: DagProjectInput = project_input(graph.get().project(py))?; - let python: Option = python_graph - .filter(|python_graph| !python_graph.is_none()) - .map(python_graph_input) - .transpose()?; - build( - graph.get().native(py).get().graph(), - &project, - python.as_ref(), - project_name, - ) - .map_err(|error| python_error(error, py)) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_dag_json, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index c86bdc51d8..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,5 +1 @@ -pub(crate) mod artifact; -pub(crate) mod projection; pub(crate) mod registration; -pub(crate) mod rows; -pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index ff1a3af1a3..f1855374db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -2,12 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dag_artifact::{artifact, projection, selection}; use crate::bindings::_helpers::dagster_invocation; +use sqlbuild_python_compile::dag_artifact::main::register_artifact::register_artifact; +use sqlbuild_python_compile::dag_artifact::main::register_projection::register_projection; +use sqlbuild_python_compile::dag_artifact::main::register_selection::register_selection; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - artifact::register(module)?; - projection::register(module)?; - selection::register(module)?; + register_artifact(module)?; + register_projection(module)?; + register_selection(module)?; dagster_invocation::registration::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index fec7e7abd1..f0528cc572 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -9,8 +9,6 @@ use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; use std::time::Instant; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; use crate::bindings::_helpers::dbt_interop::cli::resolve_dbt_config; use crate::bindings::_helpers::dbt_interop::execution::build_failed_sqlbuild_model_names; use crate::bindings::_helpers::dbt_interop::execution::build_merged_dbt_execution_argv; @@ -34,6 +32,8 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; const PLAN_OPTIONS: (&[&str], &[&str]) = ( &["project_dir", "args"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index e226cf5c8c..c2c5d9445c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -1,7 +1,5 @@ //! Discover, compile and resolve the combined dbt pipeline through its existing boundaries. -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::dbt_interop::cli::resolve_dbt_executable; use crate::bindings::_helpers::dbt_interop::cli::route_dbt_interop_args; use crate::bindings::_helpers::dbt_interop::graph::build_dbt_combined_graph; @@ -17,6 +15,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::DbtCompileReferenceResolver; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 69c050065a..53cf8e9db5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::command_runtime::connection::{ anchor_local_database_path, with_dbt_profile, }; use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; -use crate::bindings::_helpers::compile_driver::deferral::resolve_model_deferral_inputs; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -25,6 +24,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::resolve_model_deferral_inputs::resolve_model_deferral_inputs; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_planner::planner_entries::main::plans::build_display_only_sqlbuild_plan::build_display_only_sqlbuild_plan; use sqlbuild_python_planner::planner_entries::main::plans::build_execution_plan::build_execution_plan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index ab6dd33019..534e087b07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::format_execution_header; use crate::bindings::_helpers::build_progress::result_text::build_result_text; -use crate::bindings::_helpers::compile_outputs::runtime_target::write_runtime_target; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -36,6 +35,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; const WORK_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 9abb15a730..5f753700c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -33,7 +33,6 @@ use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehou use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; -use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -46,6 +45,7 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_attachment::main::effective_target_namespace::effective_target_namespace; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::main::project::discover_configured_providers::discover_configured_providers; use sqlbuild_python_front::discovery::models::SecretPolicy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 11cd978ce0..61813590ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -19,8 +19,6 @@ use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; -use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::diff_report::facts::diff_facts; use crate::bindings::_helpers::diff_report::output::publish_diff_output; @@ -39,6 +37,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_compile::compile_driver::main::stages::run_diff_pipeline::run_diff_pipeline; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index fda90eed6d..d20d5cc532 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -16,7 +16,6 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, object, progress_line, }; @@ -33,6 +32,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_compile::compile_attachment::main::effective_target_namespace::effective_target_namespace; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 185030d8e9..b7661a6bed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -38,15 +38,14 @@ use sqlbuild_state::execution_history::constants::{ DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, SCHEMA_VERSION, }; -use crate::bindings::_helpers::compile_command::constants::{ - REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, -}; -use crate::bindings::_helpers::compile_delta::driver::{ - DELTA_DISABLE_ENV_VAR, DELTA_VERIFY_ENV_VAR, -}; -use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; -use crate::bindings::_helpers::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; +use sqlbuild_python_compile::compile_command::constants::REUSE_DISABLE_ENV_VAR; +use sqlbuild_python_compile::compile_command::constants::TRACKED_ENVIRONMENT_PREFIXES; +use sqlbuild_python_compile::compile_command::constants::UNTRACKED_ENVIRONMENT_NAMES; +use sqlbuild_python_compile::compile_delta::constants::DELTA_DISABLE_ENV_VAR; +use sqlbuild_python_compile::compile_delta::constants::DELTA_VERIFY_ENV_VAR; +use sqlbuild_python_compile::frontier::constants::CAPTURE_DIR_ENV_VAR; +use sqlbuild_python_compile::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; use sqlbuild_python_front::attachments::constants::{ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs index b12b70c7a3..75e7b2d989 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/cache.rs @@ -9,13 +9,13 @@ use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; -use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; -use crate::bindings::_helpers::frontier::constants::{ - CACHE_DIRECTORY_NAME, COMPILER_CACHE_DIRECTORY_NAME, NATIVE_CACHE_NAMESPACE_SUFFIX, -}; use sqlbuild_python_base::cache::main::current_code_identity::current_code_identity; use sqlbuild_python_base::cache::main::open_store::open_store; use sqlbuild_python_base::cache::main::save_store::save_store; +use sqlbuild_python_compile::compile_command::constants::TARGET_DIRECTORY_NAME; +use sqlbuild_python_compile::frontier::constants::CACHE_DIRECTORY_NAME; +use sqlbuild_python_compile::frontier::constants::COMPILER_CACHE_DIRECTORY_NAME; +use sqlbuild_python_compile::frontier::constants::NATIVE_CACHE_NAMESPACE_SUFFIX; /// The native store kind holding clean format results. const FORMAT_RESULT_STORE_KIND: &str = "format-results"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index 977b0f7f8e..8309bf8376 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -16,7 +16,6 @@ use sqlbuild_core::format_command::main::render_lint_text::render_lint_text; use sqlbuild_core::format_command::models::LintReport; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::format_command::config::{FormatConfig, format_config, lint_config}; use crate::bindings::_helpers::format_command::files::FormatChange; use crate::bindings::_helpers::format_command::report::lint_report; @@ -32,6 +31,7 @@ use sqlbuild_python_base::build_progress::main::transient_status::status_reporte use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs index 8436f30327..976b7da8d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs @@ -9,9 +9,9 @@ use pyo3::types::PyString; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::rules::main::configuration::load_rules_config::load_rules_config; /// Format settings resolved for one run. pub(crate) struct FormatConfig { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 20ba1bd9b6..c41f21eb09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -26,12 +26,6 @@ use crate::bindings::_helpers::format_command::stages::{ QuoteFix, each_file, format_header_text, lint_header_text, on_format_workers, parse_faults, quote_fixed_text, }; -use crate::bindings::_helpers::rules::dsl_headers::rules_config; -use crate::bindings::_helpers::rules::fixture_formatting::format_redundant_fixture_nulls; -use crate::bindings::_helpers::rules::lint_pipeline::{ - LintViolationRow, ViolationClasses, project_relative_path, -}; -use crate::bindings::_helpers::rules::lint_text::{relative_parts, unsafe_fault}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::compile_driver::constants::FORMAT_CACHE_HITS; @@ -45,6 +39,13 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_compile::rules::main::checking::rules_config::rules_config; +use sqlbuild_python_compile::rules::main::lint::project_relative_path::project_relative_path; +use sqlbuild_python_compile::rules::main::lint::relative_parts::relative_parts; +use sqlbuild_python_compile::rules::main::lint::unsafe_fault::unsafe_fault; +use sqlbuild_python_compile::rules::main::sql::format_redundant_fixture_nulls::format_redundant_fixture_nulls; +use sqlbuild_python_compile::rules::models::LintViolationRow; +use sqlbuild_python_compile::rules::models::ViolationClasses; use sqlbuild_python_front::discovery::main::records::externally_described_models::externally_described_models; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -206,7 +207,7 @@ impl<'py> Run<'_, '_, 'py> { let request: &FormatRequest<'_, 'py> = self.request; let none = || py.None().into_bound(py); let result: Bound<'py, PyAny> = - crate::bindings::_helpers::rules::fix_session::plan_rule_fixes( + sqlbuild_python_compile::rules::main::fix::plan_rule_fixes::plan_rule_fixes( py, Some(keywords( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 5b4028aeb3..2d55857711 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -13,8 +13,6 @@ use sqlbuild_core::format_command::main::partition_lint_selectors::partition_lin use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::main::within::within; @@ -25,6 +23,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::main::project::load_local_config_file::load_local_config_file; use sqlbuild_python_front::discovery::main::project::load_project_config_file::load_project_config_file; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/stages.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/stages.rs index d7fa409d79..82d09741df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/stages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/stages.rs @@ -14,8 +14,8 @@ use sqlbuild_rules::dsl_headers::main::format_headers::format_headers; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderViolation}; -use crate::bindings::_helpers::rules::dsl_headers::header_violations; -use crate::bindings::_helpers::rules::native_header_parser::NativeHeaderParser; +use sqlbuild_python_compile::rules::main::checking::header_violations::header_violations; +use sqlbuild_python_compile::rules::models::NativeHeaderParser; /// The header kind of a SQL macro file. const MACRO_HEADER_KIND: &str = "MACRO"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index c58a5b69ae..b94e3b071f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -15,14 +15,14 @@ use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_planner::freshness::main::observe_freshness::observe_freshness; use sqlbuild_python_planner::freshness::main::read_latest::read_latest; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs deleted file mode 100644 index 8f958be39b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! The project-local compiler store root. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; -use crate::bindings::_helpers::frontier::constants::{ - CACHE_DIRECTORY_NAME, COMPILER_CACHE_DIRECTORY_NAME, NATIVE_CACHE_NAMESPACE_SUFFIX, -}; - -/// `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. -pub(crate) fn cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - let store: String = format!("{COMPILER_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}"); - project_dir - .div(TARGET_DIRECTORY_NAME)? - .div(CACHE_DIRECTORY_NAME)? - .div(store) -} - -/// Return the project's compiler store root. -#[pyfunction] -fn compiler_cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - cache_directory(project_dir) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compiler_cache_directory, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs deleted file mode 100644 index 2aea69d91f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Compiler store directory names and stage capture numbering. - -pub(crate) const CACHE_DIRECTORY_NAME: &str = "cache"; -pub(crate) const COMPILER_CACHE_DIRECTORY_NAME: &str = "compiler"; -/// The namespace suffix of the current native compiler store. -pub(crate) const NATIVE_CACHE_NAMESPACE_SUFFIX: &str = "-native-v1"; -/// The namespace suffixes of compiler store roots older releases wrote. -pub(crate) const RETIRED_CACHE_NAMESPACE_SUFFIXES: [&str; 3] = - ["", "-native-v1", "-native-preview-v1"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs index 031f90557f..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -1,5 +1 @@ -pub(crate) mod cache_directory; -pub(crate) mod compile_frontier; -pub(crate) mod constants; pub(crate) mod registration; -pub(crate) mod stage_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs index de98b71172..a400fb41fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::frontier::{cache_directory, compile_frontier, stage_capture}; +use sqlbuild_python_compile::frontier::main::register_cache_directory::register_cache_directory; +use sqlbuild_python_compile::frontier::main::register_compile_frontier::register_compile_frontier; +use sqlbuild_python_compile::frontier::main::register_stage_capture::register_stage_capture; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - cache_directory::register(module)?; - compile_frontier::register(module)?; - stage_capture::register(module) + register_cache_directory(module)?; + register_compile_frontier(module)?; + register_stage_capture(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/mod.rs index c871160e3f..f42d665192 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/mod.rs @@ -1,28 +1,2 @@ //! The native model loop and the loops that attach the project's other resources to it. -pub(crate) mod attachment; -pub(crate) mod audit_loop; -pub(crate) mod expanded_offsets; -pub(crate) mod function_loop; -pub(crate) mod loop_classes; -pub(crate) mod model_loop; -pub(crate) mod model_loop_calls; -pub(crate) mod model_loop_config; -pub(crate) mod model_loop_contracts; -pub(crate) mod model_loop_external; -pub(crate) mod model_loop_header; -pub(crate) mod model_loop_hooks; -pub(crate) mod model_loop_host; -pub(crate) mod model_loop_inputs; -pub(crate) mod model_loop_macros; -pub(crate) mod model_loop_named; -pub(crate) mod model_loop_opt_out; -pub(crate) mod model_loop_prepass; -pub(crate) mod model_loop_python_hooks; -pub(crate) mod model_loop_schema; -pub(crate) mod model_loop_visibility; pub(crate) mod registration; -pub(crate) mod resource_attachment; -pub(crate) mod resource_diagnostics; -pub(crate) mod scenario_loop; -pub(crate) mod source_loop; -pub(crate) mod sql_test_loop; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs index 65d017392a..af592c22a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs @@ -1,11 +1,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::input_loops::resource_attachment::RESOURCE_ATTACHMENT; +use sqlbuild_python_compile::input_loops::constants::RESOURCE_ATTACHMENT; +use sqlbuild_python_compile::input_loops::main::register_attachment::register_attachment; +use sqlbuild_python_compile::input_loops::main::register_expanded_offsets::register_expanded_offsets; +use sqlbuild_python_compile::input_loops::main::register_model_loop::register_model_loop; use sqlbuild_python_project::compiled_project::main::install::install; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { install(RESOURCE_ATTACHMENT); - crate::bindings::_helpers::input_loops::model_loop::register(module)?; - crate::bindings::_helpers::input_loops::attachment::register(module)?; - crate::bindings::_helpers::input_loops::expanded_offsets::register(module) + register_model_loop(module)?; + register_attachment(module)?; + register_expanded_offsets(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index ac79a21cd9..dc2ce178c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -30,7 +30,6 @@ use crate::bindings::_helpers::cli::machine_output::{ report_machine_output_written, take_machine_output_written, }; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, @@ -45,6 +44,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_project::planner_phases::main::object::object; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index a484b06665..a6248de9eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -2,9 +2,9 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; +use sqlbuild_python_compile::compile_driver::main::setup::build_compiled_project::build_compiled_project; use std::time::Instant; use pyo3::IntoPyObject; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs deleted file mode 100644 index 0346325648..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ /dev/null @@ -1,620 +0,0 @@ -//! `ProjectColumnLineage`: the project column lineage graph with lazily materialised views. - -use sqlbuild_python_base::positional_record; -use std::collections::HashMap; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyType}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; -use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; - -use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; -use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; - -/// Where a compact model's facts are: given, or projected from its compiled model on first read. -pub(crate) enum CompactColumns { - Facts(Py), - /// A compiled model whose `fast_lineage_columns` are the facts, and their upstream count. - Model { - model: Py, - edges: usize, - }, -} - -/// One compact model's facts and root-star flag. -pub(crate) struct CompactModel { - pub(crate) columns: CompactColumns, - pub(crate) has_star: bool, -} - -impl CompactModel { - /// The model's lineage facts as Python objects. - fn facts<'py>(&self, py: Python<'py>) -> PyResult> { - match &self.columns { - CompactColumns::Facts(columns) => Ok(columns.bind(py).clone()), - CompactColumns::Model { model, .. } => { - native_field(model.bind(py), "fast_lineage_columns") - } - } - } -} - -/// The Python objects one edge record materialises from. -enum EdgeParts { - /// An edge given to the eager constructor, returned as is. - Given(Py), - /// Fact values: source resource type, transform kind and confidence objects. - Facts { - resource_type: Py, - transform: Py, - confidence: Py, - }, -} - -positional_record! { - extract; - /// One compact lineage row: output name index, transform and confidence codes, sources. - struct CompactColumnRow<'py> { - name_index: Bound<'py, PyAny>, - transform_code: i64, - confidence_code: i64, - sources: Bound<'py, PyAny>, - } -} - -/// Indexed edge records, materialised edges shared between equal records. -struct EdgeRecords { - keys: Vec, - parts: Vec, - index: EdgeIndex, - materialised: Vec>>, -} - -#[pyclass(module = "sqlbuild._native", name = "ProjectColumnLineage")] -pub(crate) struct ProjectColumnLineage { - models: Py, - compact: HashMap, - model_order: Vec, - edges: Option>, - records: Option, - edge_counts: HashMap, -} - -#[pymethods] -impl ProjectColumnLineage { - #[new] - #[pyo3(signature = (*, models, edges))] - fn new(models: &Bound<'_, PyAny>, edges: &Bound<'_, PyAny>) -> PyResult { - let mut keys: Vec = Vec::new(); - let mut parts: Vec = Vec::new(); - for edge in edges.try_iter()? { - let edge: Bound<'_, PyAny> = edge?; - keys.push(EdgeKey { - source_name: native_field(&edge, "source")? - .getattr("resource_name")? - .extract()?, - target_model: native_field(&edge, "target")? - .getattr("resource_name")? - .extract()?, - ..empty_key() - }); - parts.push(EdgeParts::Given(edge.unbind())); - } - Ok(Self { - model_order: models - .try_iter()? - .map(|name| name?.extract()) - .collect::>()?, - models: models.clone().unbind(), - compact: HashMap::new(), - edges: Some(edges.clone().unbind()), - records: Some(EdgeRecords::new(keys, parts)), - edge_counts: HashMap::new(), - }) - } - - /// Build indexed project lineage without expanding compact analysis facts. - #[classmethod] - #[pyo3(signature = (*, models, compact_models, model_order))] - fn from_fast_facts( - cls: &Bound<'_, PyType>, - models: &Bound<'_, PyAny>, - compact_models: &Bound<'_, PyDict>, - model_order: Vec, - ) -> PyResult { - let classes: LineageClasses<'_> = LineageClasses::new(cls.py())?; - let mut compact: HashMap = HashMap::new(); - for (name, entry) in compact_models.iter() { - let (columns, has_star): (Bound<'_, PyAny>, bool) = entry.extract()?; - let _ = compact.insert( - name.extract()?, - CompactModel { - columns: CompactColumns::Facts(columns.unbind()), - has_star, - }, - ); - } - Self::compacted(&classes, models, compact, model_order) - } - - /// Return model lineage, expanding compact model facts on first access. - #[getter] - pub(crate) fn models(mut slf: PyRefMut<'_, Self>) -> PyResult> { - let py: Python<'_> = slf.py(); - if !slf.compact.is_empty() { - let classes: LineageClasses<'_> = LineageClasses::new(py)?; - let models: Bound<'_, PyAny> = slf.models.bind(py).clone(); - for name in &slf.model_order { - let Some(compact) = slf.compact.get(name) else { - continue; - }; - if models.contains(name)? { - continue; - } - let lineage: Bound<'_, PyAny> = - model_from_facts(&classes, name, &compact.facts(py)?, compact.has_star)?; - models.set_item(name, lineage)?; - } - slf.compact.clear(); - } - Ok(slf.models.clone_ref(py)) - } - - /// Return all graph edges, expanding compact edge records on first access. - #[getter] - pub(crate) fn edges(mut slf: PyRefMut<'_, Self>) -> PyResult> { - let py: Python<'_> = slf.py(); - if let Some(edges) = &slf.edges { - return Ok(edges.clone_ref(py)); - } - let classes: LineageClasses<'_> = LineageClasses::new(py)?; - slf.ensure_records(&classes)?; - let count: usize = slf.records.as_ref().map_or(0, |records| records.keys.len()); - let edges: Vec> = slf.materialised(&classes, 0..count)?; - let edges: Py = PyTuple::new(py, edges)?.into_any().unbind(); - slf.edges = Some(edges.clone_ref(py)); - Ok(edges) - } - - /// Return whether model lineage is available without expanding its columns. - #[pyo3(name = "has_model")] - fn contains_model(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { - if self.models.bind(py).contains(model_name)? { - return Ok(true); - } - Ok(model_name - .extract::() - .is_ok_and(|name| self.compact.contains_key(&name))) - } - - /// Return whether model lineage retains unresolved root-star uncertainty. - fn model_has_star(&self, py: Python<'_>, model_name: &Bound<'_, PyAny>) -> PyResult { - if let Ok(name) = model_name.extract::() - && let Some(compact) = self.compact.get(&name) - { - return Ok(compact.has_star); - } - let model: Bound<'_, PyAny> = self.models.bind(py).call_method1("get", (model_name,))?; - if model.is_none() { - return Ok(false); - } - native_field(&model, "has_star")?.extract() - } - - /// Return a target model's direct edge count without expanding edge objects. - fn edge_count_targeting(&self, model_name: &str) -> usize { - match &self.records { - Some(records) => records - .index - .by_target_model - .get(model_name) - .map_or(0, Vec::len), - None => self.edge_counts.get(model_name).copied().unwrap_or(0), - } - } - - /// Return edges whose target model is `model_name`. - fn edges_targeting<'py>( - mut slf: PyRefMut<'py, Self>, - model_name: &str, - ) -> PyResult> { - let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; - slf.ensure_records(&classes)?; - let positions: Vec = slf.positions(|index| index.by_target_model.get(model_name)); - let edges: Vec> = slf.materialised(&classes, positions)?; - PyTuple::new(slf.py(), edges) - } - - /// Return edges whose source resource is `resource_name`. - fn edges_sourced_from<'py>( - mut slf: PyRefMut<'py, Self>, - resource_name: &str, - ) -> PyResult> { - let classes: LineageClasses<'_> = LineageClasses::new(slf.py())?; - slf.ensure_records(&classes)?; - let positions: Vec = - slf.positions(|index| index.by_source_resource.get(resource_name)); - let edges: Vec> = slf.materialised(&classes, positions)?; - PyTuple::new(slf.py(), edges) - } - - /// Return direct downstream consumers of `resource_name.column_name`. - #[pyo3(signature = (*, resource_name, column_name))] - pub(crate) fn column_consumers<'py>( - slf: &Bound<'py, Self>, - resource_name: &str, - column_name: &Bound<'py, PyAny>, - ) -> PyResult> { - let sourced: Bound<'py, PyAny> = - slf.call_method1("edges_sourced_from", (resource_name,))?; - let mut consumers: Vec> = Vec::new(); - for edge in sourced.try_iter()? { - let edge: Bound<'py, PyAny> = edge?; - if native_field(&edge, "source")? - .getattr("column_name")? - .eq(column_name)? - { - consumers.push(edge); - } - } - PyTuple::new(slf.py(), consumers) - } - - fn __repr__(slf: &Bound<'_, Self>) -> PyResult { - let models: Bound<'_, PyString> = native_field(slf, "models")?.repr()?; - let edges: Bound<'_, PyString> = native_field(slf, "edges")?.repr()?; - Ok(format!( - "ProjectColumnLineage(models={models}, edges={edges})" - )) - } - - fn __eq__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = slf.py(); - if !other.is_instance_of::() { - return Ok(py.NotImplemented()); - } - let equal: bool = native_field(slf, "models")?.eq(native_field(other, "models")?)? - && native_field(slf, "edges")?.eq(native_field(other, "edges")?)?; - Ok(pyo3::types::PyBool::new(py, equal) - .to_owned() - .into_any() - .unbind()) - } -} - -impl ProjectColumnLineage { - /// Indexed project lineage of built `models` and `compact` facts, in `model_order`. - pub(crate) fn compacted( - classes: &LineageClasses<'_>, - models: &Bound<'_, PyAny>, - compact: HashMap, - model_order: Vec, - ) -> PyResult { - let py: Python<'_> = models.py(); - let mut edge_counts: HashMap = HashMap::new(); - for name in &model_order { - if let Some(model) = compact.get(name) { - let count: usize = match &model.columns { - CompactColumns::Facts(columns) => fact_edge_count(classes, columns.bind(py))?, - CompactColumns::Model { edges, .. } => *edges, - }; - let _ = edge_counts.insert(name.clone(), count); - } - } - for item in mapping_items(models)?.try_iter()? { - let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; - let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); - } - Ok(Self { - models: models.clone().unbind(), - compact, - model_order, - edges: None, - records: None, - edge_counts, - }) - } - - /// Build fast edge records from the compact facts, or the expanded models, in model order. - fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { - if self.records.is_some() { - return Ok(()); - } - let py: Python<'_> = classes.py(); - let mut builder = RecordBuilder::default(); - let models: Bound<'_, PyAny> = self.models.bind(py).clone(); - for name in &self.model_order { - if let Some(compact) = self.compact.get(name) { - builder.push_facts(classes, name, &compact.facts(py)?)?; - continue; - } - let model: Bound<'_, PyAny> = mapping_get(&models, name)?; - if model.is_none() { - continue; - } - builder.push_columns(name, &native_field(&model, "columns")?)?; - } - self.records = Some(EdgeRecords::new(builder.keys, builder.parts)); - Ok(()) - } - - /// Every edge's fields in model order, from the compact records, without building Python edges. - pub(crate) fn edge_keys(&mut self, py: Python<'_>) -> PyResult> { - self.ensure_records(&LineageClasses::new(py)?)?; - Ok(self - .records - .as_ref() - .map(|records| records.keys.clone()) - .unwrap_or_default()) - } - - fn positions(&self, select: impl Fn(&EdgeIndex) -> Option<&Vec>) -> Vec { - self.records - .as_ref() - .and_then(|records| select(&records.index)) - .cloned() - .unwrap_or_default() - } - - fn materialised( - &mut self, - classes: &LineageClasses<'_>, - positions: impl IntoIterator, - ) -> PyResult>> { - let Some(records) = self.records.as_mut() else { - return Ok(Vec::new()); - }; - positions - .into_iter() - .map(|position| records.edge(classes, position)) - .collect() - } -} - -impl EdgeRecords { - fn new(keys: Vec, parts: Vec) -> Self { - let index: EdgeIndex = edge_index(&keys); - let materialised: Vec>> = keys.iter().map(|_| None).collect(); - Self { - keys, - parts, - index, - materialised, - } - } - - /// The edge at `position`, built once per distinct fact edge. - fn edge(&mut self, classes: &LineageClasses<'_>, position: usize) -> PyResult> { - let py: Python<'_> = classes.py(); - let (resource_type, transform, confidence) = match &self.parts[position] { - EdgeParts::Given(edge) => return Ok(edge.clone_ref(py)), - EdgeParts::Facts { - resource_type, - transform, - confidence, - } => (resource_type, transform, confidence), - }; - let first: usize = self.index.first_equal[position]; - if let Some(edge) = &self.materialised[first] { - return Ok(edge.clone_ref(py)); - } - let key: &EdgeKey = &self.keys[position]; - let edge: Py = classes - .edge( - classes.qualified(resource_type.bind(py), &key.source_name, &key.source_column)?, - &classes.qualified( - &classes.model_resource, - &key.target_model, - &key.target_column, - )?, - transform.bind(py), - confidence.bind(py), - )? - .unbind(); - self.materialised[first] = Some(edge.clone_ref(py)); - Ok(edge) - } -} - -fn empty_key() -> EdgeKey { - EdgeKey { - source_type: String::new(), - source_name: String::new(), - source_column: String::new(), - target_model: String::new(), - target_column: String::new(), - transform: String::new(), - confidence: String::new(), - } -} - -fn text(value: &Bound<'_, PyAny>) -> PyResult { - value.str()?.extract() -} - -/// Edge records accumulated in model order. -#[derive(Default)] -struct RecordBuilder { - keys: Vec, - parts: Vec, -} - -impl RecordBuilder { - /// Records of compact facts, or of plain column facts, targeting `model_name`. - fn push_facts( - &mut self, - classes: &LineageClasses<'_>, - model_name: &str, - columns: &Bound<'_, PyAny>, - ) -> PyResult<()> { - if !columns.is_instance(&classes.compact_facts)? { - return self.push_columns(model_name, columns); - } - let py: Python<'_> = columns.py(); - let pool: Bound<'_, PyAny> = native_field(columns, "string_pool")?; - let remapped: bool = native_field(columns, "resource_names")?.is_truthy()? - || native_field(columns, "resource_name_indexes")?.is_truthy()?; - let mut members = CodeMembers::default(); - for row in native_field(columns, "rows")?.try_iter()? { - let CompactColumnRow { - name_index, - transform_code, - confidence_code, - sources, - }: CompactColumnRow<'_> = row?.extract()?; - let output_column: String = pool.get_item(name_index)?.extract()?; - for source in sources.try_iter()? { - let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = - source?.extract()?; - let resource_type: Bound<'_, PyAny> = pool.get_item(kind)?; - let resource_name: String = if remapped { - columns.call_method1("resource_name", (name,))?.extract()? - } else { - pool.get_item(name)?.extract()? - }; - let transform: Py = members.transform(columns, transform_code)?; - let confidence: Py = members.confidence(columns, confidence_code)?; - self.keys.push(EdgeKey { - source_type: text(&resource_type)?, - source_name: resource_name, - source_column: pool.get_item(column)?.extract()?, - target_model: model_name.to_owned(), - target_column: output_column.clone(), - transform: text(transform.bind(py))?, - confidence: text(confidence.bind(py))?, - }); - self.parts.push(EdgeParts::Facts { - resource_type: resource_type.unbind(), - transform, - confidence, - }); - } - } - Ok(()) - } - - /// Records of columns with `upstream_columns`, `transform_kind` and `confidence`. - fn push_columns(&mut self, model_name: &str, columns: &Bound<'_, PyAny>) -> PyResult<()> { - for column in columns.try_iter()? { - let column: Bound<'_, PyAny> = column?; - let output_column: String = native_field(&column, "output_column")?.extract()?; - let transform: Bound<'_, PyAny> = native_field(&column, "transform_kind")?; - let confidence: Bound<'_, PyAny> = native_field(&column, "confidence")?; - for source in native_field(&column, "upstream_columns")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - let resource_type: Bound<'_, PyAny> = native_field(&source, "resource_type")?; - self.keys.push(EdgeKey { - source_type: text(&resource_type)?, - source_name: native_field(&source, "resource_name")?.extract()?, - source_column: native_field(&source, "column_name")?.extract()?, - target_model: model_name.to_owned(), - target_column: output_column.clone(), - transform: text(&transform)?, - confidence: text(&confidence)?, - }); - self.parts.push(EdgeParts::Facts { - resource_type: resource_type.unbind(), - transform: transform.clone().unbind(), - confidence: confidence.clone().unbind(), - }); - } - } - Ok(()) - } -} - -/// Looks up the enum member of one native code. -type CodeLookup = fn(Python<'_>, usize) -> PyResult>; - -/// The transform kind and confidence members of native codes, looked up once per code. -#[derive(Default)] -struct CodeMembers { - members: HashMap<(&'static str, i64), Py>, -} - -impl CodeMembers { - fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts.py(), ("transform_kind", transform_kind_member), code) - } - - fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts.py(), ("confidence", confidence_member), code) - } - - fn member( - &mut self, - py: Python<'_>, - (kind, lookup): (&'static str, CodeLookup), - code: i64, - ) -> PyResult> { - if let Some(member) = self.members.get(&(kind, code)) { - return Ok(member.clone_ref(py)); - } - let member: Py = lookup(py, usize::try_from(code)?)?; - let _ = self.members.insert((kind, code), member.clone_ref(py)); - Ok(member) - } -} - -/// The edge count of compact facts or plain column facts. -fn fact_edge_count(classes: &LineageClasses<'_>, columns: &Bound<'_, PyAny>) -> PyResult { - if !columns.is_instance(&classes.compact_facts)? { - return upstream_count(columns); - } - let mut count: usize = 0; - for row in native_field(columns, "rows")?.try_iter()? { - count += row?.get_item(3)?.len()?; - } - Ok(count) -} - -fn upstream_count(columns: &Bound<'_, PyAny>) -> PyResult { - let mut count: usize = 0; - for column in columns.try_iter()? { - count += native_field(&column?, "upstream_columns")?.len()?; - } - Ok(count) -} - -/// `ModelColumnLineage` of compact or plain column facts. -fn model_from_facts<'py>( - classes: &LineageClasses<'py>, - model_name: &str, - columns: &Bound<'py, PyAny>, - has_star: bool, -) -> PyResult> { - let py: Python<'py> = classes.py(); - let mut lineage_columns: Vec> = Vec::new(); - for column in columns.try_iter()? { - let column: Bound<'py, PyAny> = column?; - let mut sources: Vec> = Vec::new(); - for source in native_field(&column, "upstream_columns")?.try_iter()? { - let source: Bound<'py, PyAny> = source?; - sources.push(classes.lineage_source.call1(( - native_field(&source, "resource_type")?, - native_field(&source, "resource_name")?, - native_field(&source, "column_name")?, - ))?); - } - lineage_columns.push(classes.column_lineage.call1(( - native_field(&column, "output_column")?, - native_field(&column, "transform_kind")?, - py.None(), - PyTuple::new(py, sources)?, - &classes.unknown_nullability, - native_field(&column, "confidence")?, - ))?); - } - classes.model(model_name, lineage_columns, has_star) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 3ba074d40d..99e6f35fc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -31,13 +31,6 @@ use sqlbuild_analysis::lineage_command::models::{ }; use sqlbuild_core::cli_text::models::CliStyle; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::lineage::lineage_cache::{ - lineage_fingerprint, read_lineage_cache, write_lineage_cache, -}; -use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; -use crate::bindings::_helpers::lineage::project_lineage::build_direct_semantic_uses; use sqlbuild_python_base::build_progress::main::transient_status::maybe_status::maybe_status; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; @@ -48,6 +41,15 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_compile::lineage::main::build_direct_semantic_uses::build_direct_semantic_uses; +use sqlbuild_python_compile::lineage::main::graph_index::graph_index; +use sqlbuild_python_compile::lineage::main::index_views::index_views; +use sqlbuild_python_compile::lineage::main::lineage_fingerprint::lineage_fingerprint; +use sqlbuild_python_compile::lineage::main::lineage_keys::lineage_keys; +use sqlbuild_python_compile::lineage::main::read_lineage_cache::read_lineage_cache; +use sqlbuild_python_compile::lineage::main::write_lineage_cache::write_lineage_cache; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::planner_selection::main::match_project_selector::match_project_selector; use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs deleted file mode 100644 index b6f0e3de45..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs +++ /dev/null @@ -1,222 +0,0 @@ -//! The relation lineage index read from a project graph, and its Python selector views. - -use std::collections::HashMap; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; - -/// A Python `CompiledObjectKey` as a lineage key. -pub(crate) fn lineage_key(key: &Bound<'_, PyAny>) -> PyResult { - Ok(LineageKey { - resource_type: native_field(key, "resource_type")?.str()?.extract()?, - name: native_field(key, "name")?.extract()?, - }) -} - -/// Lineage keys of every Python key in `keys`. -pub(crate) fn lineage_keys(keys: &Bound<'_, PyAny>) -> PyResult> { - keys.try_iter()?.map(|key| lineage_key(&key?)).collect() -} - -/// Python `CompiledObjectKey`s for lineage keys, one object per key. -pub(crate) struct PythonKeys<'py> { - class: Bound<'py, PyAny>, - made: HashMap>, -} - -impl<'py> PythonKeys<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { - Ok(Self { - class: record_class(py, RecordId::CompiledObjectKey)?, - made: HashMap::new(), - }) - } - - pub(crate) fn key(&mut self, key: &LineageKey) -> PyResult> { - if let Some(made) = self.made.get(key) { - return Ok(made.clone()); - } - let made: Bound<'py, PyAny> = self - .class - .call1((key.resource_type.as_str(), key.name.as_str()))?; - self.made.insert(key.clone(), made.clone()); - Ok(made) - } - - fn tuple(&mut self, keys: &[LineageKey]) -> PyResult> { - let items: Vec> = keys - .iter() - .map(|key| self.key(key)) - .collect::>()?; - PyTuple::new(self.class.py(), items) - } -} - -fn ordered_edges(edges: &Bound<'_, PyAny>) -> PyResult)>> { - let mut ordered: Vec<(LineageKey, Vec)> = Vec::new(); - for item in mapping_items(edges)?.try_iter()? { - let (key, targets): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - ordered.push((lineage_key(&key)?, lineage_keys(&targets)?)); - } - Ok(ordered) -} - -fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - Ok(Some(value.str()?.extract()?)) -} - -/// A source entry's `database.schema.table`, as far as it is set. -fn source_relation_name(entry: &Bound<'_, PyAny>) -> PyResult> { - let part = |name: &str| -> PyResult> { - match native_field(entry, name) { - Ok(value) if !value.is_none() => Ok(Some(value.extract()?)), - _ => Ok(None), - } - }; - let Some(table) = part("table")? else { - return Ok(None); - }; - Ok(Some(match (part("database")?, part("schema")?) { - (Some(database), Some(schema)) => format!("{database}.{schema}.{table}"), - (_, Some(schema)) => format!("{schema}.{table}"), - _ => table, - })) -} - -/// Display facts of every model, seed and source, models first as Python looks them up. -fn project_nodes(project: &Bound<'_, PyAny>) -> PyResult> { - let mut nodes: HashMap = HashMap::new(); - let mut add = |node: LineageNode| { - nodes.entry(node.key.clone()).or_insert(node); - }; - for model in native_field(project, "models")?.try_iter()? { - let model: Bound<'_, PyAny> = model?; - add(LineageNode { - key: lineage_key(&native_field(&model, "key")?)?, - relative_path: optional_text(native_field(&model, "relative_path")?)?, - qualified_name: optional_text( - native_field(&model, "destination")?.getattr("qualified_name")?, - )?, - }); - } - for seed in native_field(project, "seeds")?.try_iter()? { - let seed: Bound<'_, PyAny> = seed?; - add(LineageNode { - key: lineage_key(&native_field(&seed, "key")?)?, - relative_path: optional_text( - native_field(&seed, "seed_file")?.getattr("relative_path")?, - )?, - qualified_name: optional_text( - native_field(&seed, "destination")?.getattr("qualified_name")?, - )?, - }); - } - for source in native_field(project, "sources")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - add(LineageNode { - key: lineage_key(&native_field(&source, "key")?)?, - relative_path: optional_text( - native_field(&source, "source_file")?.getattr("relative_path")?, - )?, - qualified_name: source_relation_name(&native_field(&source, "source_entry")?)?, - }); - } - Ok(nodes) -} - -/// The lineage index of a compiled `ProjectGraph`; every named key has a node. -pub(crate) fn graph_index(graph: &Bound<'_, PyAny>) -> PyResult { - let views: Bound<'_, PyAny> = native_field(graph, "lineage_views")?; - let mut all_keys: Vec<(String, LineageKey)> = Vec::new(); - for item in native_field(&views, "all_keys")? - .call_method0("items")? - .try_iter()? - { - let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; - all_keys.push((name, lineage_key(&key)?)); - } - let mut tags: Vec<(String, Vec)> = Vec::new(); - for item in native_field(&views, "tag_index")? - .call_method0("items")? - .try_iter()? - { - let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; - tags.push((tag, lineage_keys(&keys)?)); - } - let mut paths: Vec<(LineageKey, String)> = Vec::new(); - for item in native_field(&views, "path_index")? - .call_method0("items")? - .try_iter()? - { - let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; - paths.push((lineage_key(&key)?, folder)); - } - let mut nodes: HashMap = - project_nodes(&native_field(graph, "project")?)?; - for (_, key) in &all_keys { - nodes.entry(key.clone()).or_insert_with(|| LineageNode { - key: key.clone(), - relative_path: None, - qualified_name: None, - }); - } - Ok(LineageIndex { - nodes, - upstream: ordered_edges(&native_field(&views, "upstream_deps")?)?, - downstream: ordered_edges(&native_field(&views, "downstream_deps")?)?, - tags, - paths, - all_keys, - }) -} - -/// `LineageGraphViews` of an index, for the planner's selector functions. -pub(crate) fn index_views<'py>( - py: Python<'py>, - index: &LineageIndex, -) -> PyResult> { - let mut keys: PythonKeys<'py> = PythonKeys::new(py)?; - let edges = |pairs: &[(LineageKey, Vec)], - keys: &mut PythonKeys<'py>| - -> PyResult> { - let dict: Bound<'py, PyDict> = PyDict::new(py); - for (key, targets) in pairs { - dict.set_item(keys.key(key)?, keys.tuple(targets)?)?; - } - Ok(dict) - }; - let upstream: Bound<'py, PyDict> = edges(&index.upstream, &mut keys)?; - let downstream: Bound<'py, PyDict> = edges(&index.downstream, &mut keys)?; - let tags: Bound<'py, PyDict> = PyDict::new(py); - for (tag, tagged) in &index.tags { - let items: Vec> = tagged - .iter() - .map(|key| keys.key(key)) - .collect::>()?; - tags.set_item(tag, PyFrozenSet::new(py, items)?)?; - } - let paths: Bound<'py, PyDict> = PyDict::new(py); - for (key, folder) in &index.paths { - paths.set_item(keys.key(key)?, folder)?; - } - let names: Bound<'py, PyDict> = PyDict::new(py); - for (name, key) in &index.all_keys { - names.set_item(name, keys.key(key)?)?; - } - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("upstream_deps", upstream)?; - options.set_item("downstream_deps", downstream)?; - options.set_item("tag_index", tags)?; - options.set_item("path_index", paths)?; - options.set_item("all_keys", names)?; - new_record(py, RecordId::LineageGraphViews, (), Some(&options)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs deleted file mode 100644 index 834b3e5aff..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! The Python lineage models and enums native lineage builds. - -use sqlbuild_host::entries::models::ExecutorHost; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; -use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - -const LOGGING_MODULE: &str = "logging"; -const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; - -/// The lineage model classes and enums, imported once per call. -pub(crate) struct LineageClasses<'py> { - pub(crate) qualified_column: Bound<'py, PyAny>, - pub(crate) lineage_source: Bound<'py, PyAny>, - pub(crate) lineage_edge: Bound<'py, PyAny>, - pub(crate) column_lineage: Bound<'py, PyAny>, - pub(crate) model_lineage: Bound<'py, PyAny>, - pub(crate) semantic_use: Bound<'py, PyAny>, - pub(crate) column_fact: Bound<'py, PyAny>, - pub(crate) source_fact: Bound<'py, PyAny>, - pub(crate) compact_facts: Bound<'py, PyAny>, - pub(crate) transform_kind: Bound<'py, PyAny>, - pub(crate) confidence: Bound<'py, PyAny>, - pub(crate) nullability: Bound<'py, PyAny>, - pub(crate) analysis_dialect: Bound<'py, PyAny>, - pub(crate) resource_type: Bound<'py, PyAny>, - pub(crate) model_resource: Bound<'py, PyAny>, - pub(crate) unknown_nullability: Bound<'py, PyAny>, -} - -impl<'py> LineageClasses<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { - let resource_type: Bound<'py, PyAny> = record_class(py, RecordId::CompiledResourceType)?; - let nullability: Bound<'py, PyAny> = record_class(py, RecordId::InferredNullability)?; - Ok(Self { - qualified_column: record_class(py, RecordId::QualifiedLineageColumn)?, - lineage_source: record_class(py, RecordId::ColumnLineageSource)?, - lineage_edge: record_class(py, RecordId::ColumnLineageEdge)?, - column_lineage: record_class(py, RecordId::ColumnLineage)?, - model_lineage: record_class(py, RecordId::ModelColumnLineage)?, - semantic_use: record_class(py, RecordId::DirectSemanticColumnUse)?, - column_fact: record_class(py, RecordId::CompiledLineageColumnFact)?, - source_fact: record_class(py, RecordId::CompiledLineageSourceFact)?, - compact_facts: record_class(py, RecordId::CompactLineageFacts)?, - transform_kind: record_class(py, RecordId::ColumnTransformKind)?, - confidence: record_class(py, RecordId::ColumnLineageConfidence)?, - analysis_dialect: record_class(py, RecordId::PolyglotAnalysisDialect)?, - model_resource: native_field(&resource_type, "MODEL")?, - unknown_nullability: native_field(&nullability, "UNKNOWN")?, - nullability, - resource_type, - }) - } - - pub(crate) fn py(&self) -> Python<'py> { - self.qualified_column.py() - } - - /// `QualifiedLineageColumn(resource_type, resource_name, column_name)`. - pub(crate) fn qualified( - &self, - resource_type: &Bound<'py, PyAny>, - resource_name: &str, - column_name: &str, - ) -> PyResult> { - self.qualified_column.call1(( - self.resource_type.call1((resource_type,))?, - resource_name, - column_name, - )) - } - - /// `ColumnLineageEdge(source, target, transform_kind, confidence)`. - pub(crate) fn edge( - &self, - source: Bound<'py, PyAny>, - target: &Bound<'py, PyAny>, - transform_kind: &Bound<'py, PyAny>, - confidence: &Bound<'py, PyAny>, - ) -> PyResult> { - self.lineage_edge - .call1((source, target, transform_kind, confidence)) - } - - /// `ModelColumnLineage(model_name, columns, has_star)`. - pub(crate) fn model( - &self, - model_name: &str, - columns: Vec>, - has_star: bool, - ) -> PyResult> { - self.model_lineage - .call1((model_name, PyTuple::new(self.py(), columns)?, has_star)) - } - - /// `ColumnLineage` of one native column; `nullability` is the enum member to record. - pub(crate) fn column( - &self, - column: &LineageColumn, - nullability: &Bound<'py, PyAny>, - ) -> PyResult> { - let mut sources: Vec> = Vec::with_capacity(column.upstream_columns.len()); - for source in &column.upstream_columns { - let resource_type: Bound<'py, PyAny> = - self.resource_type.call1((source.resource_type.as_str(),))?; - sources.push(self.lineage_source.call1(( - resource_type, - &source.resource_name, - &source.column_name, - ))?); - } - self.column_lineage.call1(( - &column.output_column, - self.transform_kind - .call1((column.transform_kind.as_str(),))?, - self.py().None(), - PyTuple::new(self.py(), sources)?, - nullability, - self.confidence.call1((column.confidence.as_str(),))?, - )) - } - - /// `CompiledLineageColumnFact` of one native star-expansion column. - pub(crate) fn column_fact(&self, column: &LineageColumn) -> PyResult> { - let sources: Vec> = column - .upstream_columns - .iter() - .map(|source| self.source_fact(source)) - .collect::>()?; - let fields: Bound<'py, PyDict> = PyDict::new(self.py()); - fields.set_item("output_column", &column.output_column)?; - fields.set_item("upstream_columns", PyTuple::new(self.py(), sources)?)?; - fields.set_item( - "transform_kind", - self.transform_kind - .call1((column.transform_kind.as_str(),))?, - )?; - fields.set_item( - "confidence", - self.confidence.call1((column.confidence.as_str(),))?, - )?; - self.column_fact.call((), Some(&fields)) - } - - fn source_fact(&self, source: &LineageSource) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(self.py()); - fields.set_item("resource_type", source.resource_type.as_str())?; - fields.set_item("resource_name", &source.resource_name)?; - fields.set_item("column_name", &source.column_name)?; - self.source_fact.call((), Some(&fields)) - } -} - -/// A structured debug event on the `sqlbuild.lineage` logger for one model. -pub(crate) fn log_lineage_event( - py: Python<'_>, - message: &str, - model_name: &str, - error: Option<&str>, -) -> PyResult<()> { - let logger: Bound<'_, PyAny> = py - .import(LOGGING_MODULE)? - .getattr("getLogger")? - .call1((LINEAGE_LOGGER,))?; - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("logger", logger)?; - fields.set_item("message", message)?; - fields.set_item("sqlbuild_model", model_name)?; - fields.set_item("sqlbuild_error", error)?; - let _ = host_entry(py, ExecutorHost::LogDebugEvent)?.call((), Some(&fields))?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index 26032298d2..a52d61eeb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,9 +1,2 @@ -pub(crate) mod column_lineage_graph; -pub(crate) mod lineage_cache; pub(crate) mod lineage_command; -pub(crate) mod lineage_index; -pub(crate) mod lineage_objects; -pub(crate) mod project_lineage; -pub(crate) mod project_schema; pub(crate) mod registration; -pub(crate) mod relation_fingerprint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index f624ff31f5..0bb1504c18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -2,13 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lineage::{ - column_lineage_graph, lineage_command, project_lineage, relation_fingerprint, -}; +use crate::bindings::_helpers::lineage::lineage_command; +use sqlbuild_python_compile::lineage::main::register_column_lineage_graph::register_column_lineage_graph; +use sqlbuild_python_compile::lineage::main::register_project_lineage::register_project_lineage; +use sqlbuild_python_compile::lineage::main::register_relation_fingerprint::register_relation_fingerprint; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - column_lineage_graph::register(module)?; - project_lineage::register(module)?; - relation_fingerprint::register(module)?; + register_column_lineage_graph(module)?; + register_project_lineage(module)?; + register_relation_fingerprint(module)?; lineage_command::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 6b5caa16b2..3067773bb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -21,10 +21,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, selected_target_settings, -}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -44,6 +40,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 8df8b56350..93aed88937 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -17,12 +17,6 @@ use sqlbuild_core::load_command::main::select_load_entries::select_load_entries use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compile_attachment::target::{ - connection_is_default_target, project_location, -}; -use crate::bindings::_helpers::compile_driver::targets::{ - expanded_loader_namespace, managed_write_schema, preserved_loader_schemas, -}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -32,6 +26,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_attachment::main::connection_is_default_target::connection_is_default_target; +use sqlbuild_python_compile::compile_attachment::main::project_location::project_location; +use sqlbuild_python_compile::compile_driver::main::setup::expanded_loader_namespace::expanded_loader_namespace; +use sqlbuild_python_compile::compile_driver::main::setup::managed_write_schema::managed_write_schema; +use sqlbuild_python_compile::compile_driver::main::setup::preserved_loader_schemas::preserved_loader_schemas; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::contracts::main::loader_destination_parts::loader_destination_parts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index aa218631f7..867deeb14e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -19,7 +19,6 @@ use sqlbuild_orchestration::dev_events::models::{DevPhase, DevStepState}; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; -use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::snapshot::{ @@ -36,6 +35,7 @@ use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::compiled_phase::compiled_phase; use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::project_snapshot::types::NodeKeys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs index 9e380d68d4..11943aa484 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyResult}; use serde_json::{Value, json}; -use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; -use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::lineage::main::build_project_column_lineage::build_project_column_lineage; +use sqlbuild_python_compile::lineage::models::ProjectColumnLineage; /// FAST column lineage of every model as `[source key, column, target key, column, transform]` rows; `None` without SQL analysis. pub(crate) fn column_lineage_document(project: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs index c50b252ca8..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/mod.rs @@ -1,3 +1 @@ -pub(crate) mod builder; pub(crate) mod registration; -pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs index f2c51d9e32..4768d09970 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/registration.rs @@ -2,8 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::manifest::builder; +use sqlbuild_python_compile::manifest::main::register_builder::register_builder; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - builder::register(module) + register_builder(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e02a4eae34..01bd265121 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod analysis; pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod audit_command; @@ -14,7 +13,6 @@ pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; -pub(crate) mod compile_delta; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; @@ -43,7 +41,6 @@ pub(crate) mod format_command; pub(crate) mod freshness; pub(crate) mod frontier; pub(crate) mod graph; -pub(crate) mod hand_built; pub(crate) mod init_command; pub(crate) mod input_loops; pub(crate) mod janitor; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index b5b8167d49..1ba70501f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspe use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; @@ -32,6 +31,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index 2e676e42da..4b1e6b0505 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -9,15 +9,6 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; use sqlbuild_rules::rule_catalogue::models::RuleListing; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::rules::catalogue::build_rule_catalogue; -use crate::bindings::_helpers::rules::guidance::{format_rule, install_rules_skills}; -use crate::bindings::_helpers::rules::results::{ - format_rules_result, format_skipped_type_proof_rules, -}; -use crate::bindings::_helpers::rules::rule_run::run_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; @@ -25,6 +16,15 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_compile::rules::main::checking::format_rules_result::format_rules_result; +use sqlbuild_python_compile::rules::main::checking::format_skipped_type_proof_rules::format_skipped_type_proof_rules; +use sqlbuild_python_compile::rules::main::configuration::build_rule_catalogue::build_rule_catalogue; +use sqlbuild_python_compile::rules::main::configuration::format_rule::format_rule; +use sqlbuild_python_compile::rules::main::configuration::install_rules_skills::install_rules_skills; +use sqlbuild_python_compile::rules::main::configuration::load_rules_config::load_rules_config; +use sqlbuild_python_compile::rules::main::rule::run_rules::run_rules; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 8873e278f1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1,8 +1 @@ -pub(crate) mod declared_relations; -pub(crate) mod getters; -pub(crate) mod models; -pub(crate) mod native_resources; pub(crate) mod registration; -pub(crate) mod resources; -pub(crate) mod syntax_checks; -pub(crate) mod syntax_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs deleted file mode 100644 index a369286d57..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! The resource and model facts the native project assembly hands to the compile driver. - -use pyo3::prelude::{Py, PyAny}; -use pyo3::pyclass; - -/// Per-resource facts in input order, and whether each model's SQL passed syntax validation. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct NativeProjectResources { - pub(crate) model_deps: Py, - pub(crate) model_syntax_valid: Py, - pub(crate) source_namespaces: Py, - pub(crate) seed_namespaces: Py, - pub(crate) function_deps: Py, - pub(crate) audit_scope_deps: Py, - /// The `NativeResourceFailure` Python's assembly raises at, or `None`. - pub(crate) failure: Py, -} - -/// The resource Python's assembly raises at, and the exception it raises there. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct NativeResourceFailure { - pub(crate) slot: Py, - pub(crate) index: Py, - pub(crate) error: Py, -} - -/// One model's native facts; its dependencies as `(resource type, name)` rows. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct NativeModelFacts { - pub(crate) deps: Py, - pub(crate) dynamic_contract: Py, - pub(crate) syntax_valid: Py, -} - -/// Every model's native facts and the resource facts. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct NativeProjectFacts { - pub(crate) models: Py, - pub(crate) resources: Py, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 160f660c9b..9c0db4dc14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -2,20 +2,22 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::project_assembly::models::{ - NativeModelFacts, NativeProjectFacts, NativeProjectResources, NativeResourceFailure, -}; -use crate::bindings::_helpers::project_assembly::{ - native_resources, resources, syntax_checks, syntax_validation, -}; +use sqlbuild_python_compile::project_assembly::main::register_native_resources::register_native_resources; +use sqlbuild_python_compile::project_assembly::main::register_resources::register_resources; +use sqlbuild_python_compile::project_assembly::main::register_syntax_checks::register_syntax_checks; +use sqlbuild_python_compile::project_assembly::main::register_syntax_validation::register_syntax_validation; +use sqlbuild_python_compile::project_assembly::models::NativeModelFacts; +use sqlbuild_python_compile::project_assembly::models::NativeProjectFacts; +use sqlbuild_python_compile::project_assembly::models::NativeProjectResources; +use sqlbuild_python_compile::project_assembly::models::NativeResourceFailure; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; - native_resources::register(module)?; - syntax_validation::register(module)?; - syntax_checks::register(module)?; - resources::register(module) + register_native_resources(module)?; + register_syntax_validation(module)?; + register_syntax_checks(module)?; + register_resources(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs deleted file mode 100644 index e0e1ef997e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Resource assembly facts for one project, computed natively. - -use std::collections::HashMap; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; -use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; -use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; -use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use sqlbuild_core::panics::main::native_failure::native_failure; - -use crate::bindings::_helpers::input_loops::model_loop_prepass::pooled; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::types::CompilerDetach; - -const SYNTAX_CHECK_CONTEXT: &str = "native SQL syntax check"; - -/// Python's syntax error message for one SQL string, None where it parses. -#[pyfunction] -#[pyo3(signature = (sql, placeholders, dialect, parse_one=false))] -fn sql_syntax_error( - sql: &Bound<'_, PyString>, - placeholders: Option>, - dialect: Option, - parse_one: bool, -) -> PyResult> { - let py: Python<'_> = sql.py(); - let check = SyntaxCheck { - sql: sql.to_str()?.to_owned(), - placeholders: placeholders.unwrap_or_default().into_iter().collect(), - }; - let dialect: String = dialect - .filter(|dialect| !dialect.is_empty()) - .unwrap_or_else(|| "generic".to_owned()); - let mode = if parse_one { - SyntaxMode::Parse - } else { - SyntaxMode::Validate - }; - syntax_error_message(py, &check, &dialect, mode) -} - -/// Python's syntax error message for one check, raising what Python's validation raises. -pub(crate) fn syntax_error_message( - py: Python<'_>, - check: &SyntaxCheck, - dialect: &str, - mode: SyntaxMode, -) -> PyResult> { - py.compiler_detach(|| Ok(check_syntax_error(check, dialect, mode))) - .map_err(compiler_error)? - .map_err(syntax_value_error) -} - -/// Each check's `syntax_error_message`, in order, parsed together on the compile pool. -pub(crate) fn syntax_error_messages( - py: Python<'_>, - checks: &[SyntaxCheck], - dialect: &str, - mode: SyntaxMode, -) -> Vec>> { - py.detach(|| { - pooled(checks, |check| { - catch_compiler_panic(|| Ok(check_syntax_error(check, dialect, mode))) - }) - }) - .into_iter() - .map(|outcome| outcome.map_err(compiler_error)?.map_err(syntax_value_error)) - .collect() -} - -/// The `ValueError` Python's syntax validation raises before parsing. -fn syntax_value_error(failure: SyntaxFailure) -> PyErr { - match failure { - SyntaxFailure::Normalization(message) => PyValueError::new_err(message), - SyntaxFailure::UnknownDialect(name) => { - PyValueError::new_err(format!("Unknown dialect: {name}")) - } - } -} - -/// Whether every check parses, raising what Python's validation raises. -pub(crate) fn syntax_valid( - py: Python<'_>, - dialect: &str, - checks: &[SyntaxCheck], -) -> PyResult { - py.compiler_detach(|| check_sql_syntax(dialect, checks)) - .map_err(|reason| compiler_error(native_failure(SYNTAX_CHECK_CONTEXT, &reason)))? - .map_err(syntax_value_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs deleted file mode 100644 index e05f8753e0..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! The model and hook SQL assembly validates, and the SQL standalone pivot proofs read. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; -use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; - -use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; -use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; - -const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; - -/// Native SQL and dynamic family specs indexed by their model's position. -type PivotModelRows = Vec<(usize, (String, Vec))>; - -fn config_value<'py>(model_input: &Bound<'py, PyAny>, key: &str) -> PyResult> { - native_field(&native_field(model_input, "config")?, "values")?.call_method1("get", (key,)) -} - -fn model_name(model_input: &Bound<'_, PyAny>) -> PyResult { - native_field( - &native_field(&native_field(model_input, "model_file")?, "file_path")?, - "stem", - )? - .extract() -} - -/// The model's `@@@name` placeholder defaults as strings, or `None`. -#[pyfunction] -pub(crate) fn model_placeholders<'py>( - model_input: &Bound<'py, PyAny>, -) -> PyResult>> { - let raw: Bound<'py, PyAny> = config_value(model_input, "placeholders")?; - let Ok(raw) = raw.downcast::() else { - return Ok(None); - }; - let placeholders: Bound<'py, PyDict> = PyDict::new(model_input.py()); - for (key, value) in raw.iter() { - placeholders.set_item(key.str()?, value.str()?)?; - } - Ok(Some(placeholders)) -} - -/// The model query with cursor intrinsics replaced by typed literals for static analysis. -fn analysis_sql<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model_input.py(); - Ok(PyString::new( - py, - &analysis_cursor_sql( - py, - &native_field(model_input, "query_sql")?.extract::()?, - &config_value(model_input, "cursor_type")?, - )?, - ) - .into_any()) -} - -fn syntax_checks<'py>( - model_input: &Bound<'py, PyAny>, - analysis_model_names: &Bound<'py, PyAny>, - analysis_succeeded: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model_input.py(); - let name: String = model_name(model_input)?; - if !native_field(model_input, "sql_validation_enabled")?.is_truthy()? - || (!analysis_model_names.is_none() && !analysis_model_names.contains(&name)?) - { - return Ok(PyTuple::empty(py)); - } - let placeholders: Bound<'py, PyAny> = match model_placeholders(model_input)? { - Some(placeholders) => placeholders.into_any(), - None => py.None().into_bound(py), - }; - let mut statements: Vec> = Vec::new(); - for hook_name in HOOK_NAMES { - for statement in hook_sql_statements(config_value(model_input, hook_name)?)? { - statements.push(PyString::new(py, &statement).into_any()); - } - } - if !analysis_succeeded.contains(&name)? { - statements.push(analysis_sql(model_input)?); - } - let checks: Vec> = statements - .into_iter() - .map(|statement| PyTuple::new(py, [statement, placeholders.clone()])) - .collect::>()?; - PyTuple::new(py, checks) -} - -/// Each model's `(sql, placeholders)` that assembly passes to syntax validation. -#[pyfunction] -#[pyo3(signature = (*, model_inputs, analysis_model_names, analysis_succeeded))] -pub(crate) fn model_syntax_checks<'py>( - model_inputs: &Bound<'py, PyAny>, - analysis_model_names: &Bound<'py, PyAny>, - analysis_succeeded: &Bound<'py, PyAny>, -) -> PyResult> { - let checks: Vec> = model_inputs - .try_iter()? - .map(|model_input| syntax_checks(&model_input?, analysis_model_names, analysis_succeeded)) - .collect::>()?; - PyTuple::new(model_inputs.py(), checks) -} - -/// `(sql, families)` by model index for each pivot model no analysis covered. -#[pyfunction] -#[pyo3(signature = (*, model_inputs, analysed))] -pub(crate) fn standalone_pivot_models<'py>( - model_inputs: &Bound<'py, PyAny>, - analysed: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model_inputs.py(); - let models: Bound<'py, PyDict> = PyDict::new(py); - for (index, (sql, _)) in standalone_pivot_rows(model_inputs, analysed)? { - let model = model_inputs.get_item(index)?; - let entry = native_field(&model, "schema_entry")?; - models.set_item(index, (sql, native_field(&entry, "dynamic_columns")?))?; - } - Ok(models) -} - -/// Native-only pivot requests; the public wrapper alone projects contract records. -pub(crate) fn standalone_pivot_rows( - model_inputs: &Bound<'_, PyAny>, - analysed: &Bound<'_, PyAny>, -) -> PyResult { - let mut models: PivotModelRows = Vec::new(); - for (index, model_input) in model_inputs.try_iter()?.enumerate() { - let model_input = model_input?; - let Some(schema) = attached_schema(&model_input)? else { - continue; - }; - let families = schema.dynamic_columns; - if families.is_empty() || analysed.contains(model_name(&model_input)?)? { - continue; - } - let query_sql: String = analysis_sql(&model_input)?.extract()?; - let sql: String = match model_placeholders(&model_input)? { - Some(placeholders) if !placeholders.is_empty() => { - let pairs: Vec<(String, String)> = placeholders - .iter() - .map(|(key, value)| Ok((key.extract()?, value.extract()?))) - .collect::>()?; - placeholder_defaults(&query_sql, &pairs) - } - _ => query_sql, - }; - models.push((index, (sql, families))); - } - Ok(models) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_placeholders, module)?)?; - module.add_function(wrap_pyfunction!(model_syntax_checks, module)?)?; - module.add_function(wrap_pyfunction!(standalone_pivot_models, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 866850a577..b35df57da0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -6,8 +6,6 @@ use pyo3::types::{PyBool, PyDict}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -15,6 +13,8 @@ use crate::bindings::_helpers::query_command::project::{ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::dag_artifact::main::build_dag_json::build_dag_json; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 95592855ca..68605d03d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -2,11 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::main::project::discover_connection_configuration::discover_connection_configuration; use sqlbuild_python_front::discovery::models::SecretPolicy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 73a8bcc57d..412cfbcf3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -69,8 +69,8 @@ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; use crate::bindings::_helpers::{ - analysis, analysis_session, build_progress, compile_command, compute_logs, dbt_events, - dbt_interop, event_exporting, execution_history, project_assembly, result_output, rules, + build_progress, compute_logs, dbt_events, dbt_interop, event_exporting, execution_history, + result_output, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; @@ -356,10 +356,10 @@ pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &[ /// The getter tables of every module that defines native pyclasses. pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ - analysis::getters::GETTERS, - analysis_session::getters::GETTERS, + sqlbuild_python_compile::analysis::constants::GETTERS, + sqlbuild_python_compile::analysis_session::constants::GETTERS, build_progress::getters::GETTERS, - compile_command::getters::GETTERS, + sqlbuild_python_compile::compile_command::constants::GETTERS, sqlbuild_python_base::compile_context::constants::GETTERS, sqlbuild_python_project::compiled_views::constants::GETTERS, compute_logs::getters::GETTERS, @@ -375,9 +375,9 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_planner::planner_migrations::constants::GETTERS, sqlbuild_python_planner::planner_phases::constants::GETTERS, sqlbuild_python_base::presentation_text::constants::GETTERS, - project_assembly::getters::GETTERS, + sqlbuild_python_compile::project_assembly::constants::GETTERS, result_output::getters::GETTERS, - rules::getters::GETTERS, + sqlbuild_python_compile::rules::constants::GETTERS, sqlbuild_python_front::scopes::constants::GETTERS, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index df33c553c6..9c750cacf9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -7,8 +7,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyString, PyTuple}; -use crate::bindings::_helpers::compile_command::target_writer::{Prepared, planning_diagnostics}; -use crate::bindings::_helpers::compile_driver::analysis::analyze_compile_project; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -17,6 +15,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_command::main::artifacts::planning_diagnostics::planning_diagnostics; +use sqlbuild_python_compile::compile_command::models::Prepared; +use sqlbuild_python_compile::compile_driver::main::stages::analyze_compile_project::analyze_compile_project; const DEFAULT_ERROR_CODE: &str = "E001"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index f0c038c686..547d358d70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -36,7 +36,6 @@ use sqlbuild_refactor::refactoring::models::{ }; use sqlbuild_refactor::refactoring::types::Originals; -use crate::bindings::_helpers::lineage::lineage_index::lineage_key; use crate::bindings::_helpers::refactor::offline_compile::{OfflineCompile, compile_for_refactor}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; @@ -46,6 +45,7 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_compile::lineage::main::lineage_key::lineage_key; use sqlbuild_python_front::refactor::main::declaration_placement::declaration_placement; use sqlbuild_python_front::refactor::main::model_facts::model_facts; use sqlbuild_python_front::refactor::main::refactor_facts::refactor_facts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 4ba0c73a2b..4419e0c112 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -17,7 +17,6 @@ use sqlbuild_cli::diff_report::models::{ use sqlbuild_cli::diff_report::types::DiffNamedValue; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::diff_report::command::target_max_full_rows; use crate::bindings::_helpers::diff_report::facts::diff_facts; @@ -32,6 +31,7 @@ use sqlbuild_python_base::build_command::main::boolean::boolean; use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_compile::compile_driver::main::stages::run_diff_pipeline::run_diff_pipeline; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; const KEYED_CHECK: &str = "keyed comparison"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs index e87d6ba7e5..57445e41f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs @@ -7,11 +7,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString}; use sqlbuild_analysis::review_command::models::NodeComparison; -use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; -use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::lineage::main::build_project_column_lineage::build_project_column_lineage; +use sqlbuild_python_compile::lineage::models::ProjectColumnLineage; use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index 0f2f9634b8..1fc761e725 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -22,8 +22,6 @@ use sqlbuild_config::project::models::ReviewConfig; use crate::bindings::_helpers::change_selection::base_snapshot::{base_comparison, base_variant}; use crate::bindings::_helpers::change_selection::definition_snapshot::project_wide_keys; use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::execution_history::conversion::optional_text; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -32,6 +30,8 @@ use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs deleted file mode 100644 index efb853e5fa..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ /dev/null @@ -1,461 +0,0 @@ -//! Speculative SQL checks started once compile inputs are ready, and the lint settings they use. - -use std::path::PathBuf; -use std::time::Instant; - -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use std::thread::JoinHandle; - -use pyo3::exceptions::{PyException, PyRuntimeError}; -use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRefMut, - PyResult, Python, -}; -use pyo3::types::PyTuple; -use pyo3::{pyclass, pymethods}; -use sqlbuild_rules::lint_text::models::LintSqlOptions; -use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; -use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; - -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; -use crate::bindings::_helpers::rules::lint_expansion::{ - LintExpansionSources, lint_expansion_inputs, -}; -use crate::bindings::_helpers::rules::lint_inputs::model_lint_expansion; -use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, -}; -use crate::bindings::_helpers::rules::rule_cache::stored_entries; -use crate::bindings::_helpers::rules::rules_config::rules_config_of; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interruptibly; -use sqlbuild_python_base::native_threads::models::Joined; -use sqlbuild_python_base::records::main::fields::field::field as record_field; -use sqlbuild_python_front::render::models::SqlExpansionInputs; -use sqlbuild_python_project::compiled_views::main::compiled_models::model_key_row::model_key_row; -use sqlbuild_python_project::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; -use sqlbuild_python_project::compiled_views::main::compiled_models::relation_keys::relation_keys; -use sqlbuild_python_project::compiled_views::main::compiled_models::source_key_input::source_key_input; -use sqlbuild_python_project::compiled_views::models::RelationKeyRow; -use sqlbuild_python_project::compiled_views::types::RelationKeys; - -/// The lint thresholds used when the project leaves them unset. -const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; -const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; -const MAX_LITERAL_LENGTH: &str = "max_literal_length"; -const MAX_RANKING_ORDER_BY: &str = "max_ranking_order_by"; -const LINT_THREAD: &str = "sqlbuild-lint"; -const SQL_RULES_CACHE_FILE: &str = "sql.json"; - -/// The SQL rule result cache file of `project_dir`. -pub(crate) fn sql_rule_cache_path(project_dir: &Path) -> PathBuf { - rules_bulk_cache_path(project_dir, SQL_RULES_CACHE_FILE) -} - -/// The native SQL lint settings that SQL rules run with. -pub(crate) fn sql_lint_settings( - dialect: &str, - codes: &[String], - threshold: impl Fn(&str) -> PyResult>, - relation_keys: RelationKeys, -) -> PyResult { - let defaults: LintSettings = LintSettings::default(); - Ok(LintSettings { - header_rules_enabled: false, - sql: LintSqlOptions { - dialect: dialect.to_owned(), - enabled_rules: Some(codes.to_vec()), - max_literal_length: threshold(MAX_LITERAL_LENGTH)? - .unwrap_or(DEFAULT_MAX_LITERAL_LENGTH), - max_ranking_order_by: threshold(MAX_RANKING_ORDER_BY)? - .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), - ..defaults.sql - }, - relation_keys, - ..defaults - }) -} - -/// What early SQL lint runs with: the project, its selected SQL rule codes and lint settings. -struct EarlyLintPlan { - project_dir: PathBuf, - codes: Vec, - settings: LintSettings, -} - -/// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. -fn early_lint_plan( - inputs: &Bound<'_, PyAny>, - dialect: &str, - no_cache: bool, -) -> PyResult> { - let py: Python<'_> = inputs.py(); - let project_dir: Bound<'_, PyAny> = - record_field(&record_field(inputs, "discovered_inputs")?, "project_dir")?; - if project_dir.is_none() { - return Ok(None); - } - let project_dir: PathBuf = project_dir.extract()?; - for diagnostic in record_field(inputs, "diagnostics")?.try_iter()? { - if record_field(&diagnostic?, "is_error")?.is_truthy()? { - return Ok(None); - } - } - let Ok(mut config) = rules_config_of(&project_dir)? else { - return Ok(None); - }; - config.cache.enabled &= !no_cache; - if config.cache.enabled && !stored_entries(&sql_rule_cache_path(&project_dir))?.is_empty() { - return Ok(None); - } - let codes: Vec = - compiler_guard(|| Ok(selected_sql_codes(config.select.as_deref(), &config.ignore)))?; - if codes.is_empty() { - return Ok(None); - } - let mut rows: Vec = Vec::new(); - for model in record_field(inputs, "model_inputs")?.try_iter()? { - rows.push(model_key_row(&model?)?); - } - let seed_inputs: Vec> = record_field(inputs, "seed_inputs")? - .try_iter()? - .collect::>()?; - let one_row_seeds: Vec = one_row_seed_names(&seed_inputs)?; - let mut seeds: Vec> = Vec::new(); - for seed in &seed_inputs { - seeds.push(record_field(seed, "schema_entry")?); - } - let mut sources: Vec> = Vec::new(); - for source in record_field(inputs, "source_inputs")?.try_iter()? { - sources.push(source_key_input(&source?)?); - } - let keys: RelationKeys = relation_keys( - py, - rows, - ( - PyTuple::new(py, seeds)?.as_any(), - PyTuple::new(py, sources)?.as_any(), - ), - &one_row_seeds, - )?; - let settings: LintSettings = sql_lint_settings( - dialect, - &codes, - |key| Ok(config.thresholds.get(key).map(|value| i64::from(*value))), - keys, - )?; - Ok(Some(EarlyLintPlan { - project_dir, - codes, - settings, - })) -} - -/// Early SQL lint's outcome and the expansion inputs it linted with. -pub(crate) struct EarlyLintResult { - pub(crate) outcome: LintOutcome, - pub(crate) context: SqlExpansionInputs, -} - -/// One early SQL lint running on a native thread, for one project and SQL rule codes. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct PreparedSqlLint { - codes: Vec, - project_dir: PathBuf, - thread: Mutex>>>, - result: OnceLock>, - stop: Arc, -} - -impl PreparedSqlLint { - /// Whether this lint ran for `project_dir` with exactly `codes`. - pub(crate) fn matches(&self, project_dir: &Path, codes: &[String]) -> bool { - self.project_dir == project_dir && same_codes(&self.codes, codes) - } - - /// Whether this lint ran with exactly `codes`. - pub(crate) fn has_codes(&self, codes: &[String]) -> bool { - same_codes(&self.codes, codes) - } - - /// Whether the lint's result is already known. - fn finished(&self) -> bool { - self.result.get().is_some() - } - - /// The lint's result, waiting without the GIL until it finishes or a signal handler raises. - pub(crate) fn result(&self, py: Python<'_>) -> &Result { - self.result.get_or_init(|| { - let thread = self - .thread - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); - let Some(thread) = thread else { - return Err(PyRuntimeError::new_err("early SQL lint was already joined")); - }; - match join_interruptibly(py, thread, || self.stop.store(true, Ordering::Release)) { - Joined::Finished(joined) => joined.unwrap_or_else(|_| { - Err(PyRuntimeError::new_err("early SQL lint thread panicked")) - }), - Joined::Interrupted(signal) => Err(signal), - } - }) - } -} - -fn same_codes(left: &[String], right: &[String]) -> bool { - left.iter().collect::>() == right.iter().collect::>() -} - -/// Start early SQL lint on a native thread when the project selects SQL rules without a cache. -fn start_early_lint( - (inputs, dialect): (&Bound<'_, PyAny>, &str), - reuse: ExpansionReuse, - no_cache: bool, - stop: &Arc, -) -> PyResult> { - let Some(plan) = early_lint_plan(inputs, dialect, no_cache)? else { - return Ok(None); - }; - let mut compiled: HashMap = HashMap::new(); - for model in record_field(inputs, "model_inputs")?.try_iter()? { - if let Some((path, expansion)) = model_lint_expansion(&model?)? { - compiled.insert(path, expansion); - } - } - let work: EarlyLintWork = EarlyLintWork { - project_dir: plan.project_dir.clone(), - settings: plan.settings, - reuse, - compiled, - stop: Arc::clone(stop), - }; - let thread: JoinHandle> = std::thread::Builder::new() - .name(LINT_THREAD.to_owned()) - .spawn(move || Python::attach(|py| early_lint(py, &work))) - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - Ok(Some(PreparedSqlLint { - codes: plan.codes, - project_dir: plan.project_dir, - thread: Mutex::new(Some(thread)), - result: OnceLock::new(), - stop: Arc::clone(stop), - })) -} - -/// What one early SQL lint thread lints: the project, its settings and the compile's expansions. -struct EarlyLintWork { - project_dir: PathBuf, - settings: LintSettings, - reuse: ExpansionReuse, - compiled: HashMap, - stop: Arc, -} - -/// Expand and lint the project's SQL early, stopping between files once `stop` is set. -fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult { - let project_dir: &Path = &work.project_dir; - let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { - project_dir: project_dir.into_pyobject(py)?.into_any(), - value_renderer: Some(work.reuse.value_renderer(py)), - vars: work.reuse.expansion_vars(py)?, - discovered_inputs: Some(work.reuse.discovered_inputs(py)?), - declaration_scope: None, - static_declaration_scope: present(work.reuse.declaration_scope(py)?), - })?; - let files: Vec = lint_sources(project_dir, None, None)?; - let outcome: LintOutcome = lint( - py, - &LintRequest { - project_dir, - settings: &work.settings, - dynamic_output_paths: &HashSet::new(), - compiled_expansions: &work.compiled, - context: Some(&context), - skip_unexpandable: false, - stop: Some(&work.stop), - }, - files, - )?; - Ok(EarlyLintResult { outcome, context }) -} - -/// The compiler's inputs and adapter, which lint expansion reuses for the same discovery. -pub(crate) struct ExpansionReuse { - inputs: Py, - value_renderer: Py, -} - -impl ExpansionReuse { - fn clone_ref(&self, py: Python<'_>) -> Self { - Self { - inputs: self.inputs.clone_ref(py), - value_renderer: self.value_renderer.clone_ref(py), - } - } - - /// The discovery the compile ran on. - pub(crate) fn discovered_inputs<'py>(&self, py: Python<'py>) -> PyResult> { - record_field(self.inputs.bind(py), "discovered_inputs") - } - - /// The compile's declaration scope. - pub(crate) fn declaration_scope<'py>(&self, py: Python<'py>) -> PyResult> { - record_field(self.inputs.bind(py), "declaration_scope") - } - - /// The compile's target and vars, read from its macro context. - pub(crate) fn expansion_vars<'py>(&self, py: Python<'py>) -> PyResult> { - let context: Bound<'py, PyAny> = record_field(self.inputs.bind(py), "macro_context")?; - Ok(ExpansionVars::Compile { - target_name: record_field(&context, "target_name")?, - vars: record_field(&context, "vars")?, - }) - } - - /// The adapter that renders declared values. - pub(crate) fn value_renderer<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { - self.value_renderer.bind(py).clone() - } -} - -/// Own speculative SQL checks for one compilation invocation. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct EarlySqlLint { - #[pyo3(get)] - pub(crate) enabled: bool, - /// Whether `--no-cache` turned the Rules cache off, so a warm SQL rule cache is ignored. - no_cache: bool, - preparation: Option>, - expansion_reuse: Option, - /// The dialect and adapter the `on_inputs_ready` call passes to `start`. - ready_arguments: Option<(String, Py)>, - stop_requested: Arc, -} - -impl EarlySqlLint { - /// A manager that starts speculative checks only when `enabled`. - pub(crate) fn new(enabled: bool, no_cache: bool) -> Self { - Self { - enabled, - no_cache, - preparation: None, - expansion_reuse: None, - ready_arguments: None, - stop_requested: Arc::new(AtomicBool::new(false)), - } - } - - /// The running early SQL lint, if one started. - pub(crate) fn preparation(&self) -> Option<&Py> { - self.preparation.as_ref() - } - - /// The compile's discovery and declaration scope, if it had one. - pub(crate) fn expansion_reuse(&self) -> Option<&ExpansionReuse> { - self.expansion_reuse.as_ref() - } - - /// This manager as the `on_inputs_ready` hook, which calls `start` with `dialect` and `value_renderer`. - pub(crate) fn inputs_ready_hook<'py>( - slf: &Bound<'py, Self>, - dialect: &str, - value_renderer: &Bound<'py, PyAny>, - ) -> Bound<'py, PyAny> { - slf.borrow_mut().ready_arguments = - Some((dialect.to_owned(), value_renderer.clone().unbind())); - slf.clone().into_any() - } - - /// Abandon speculative work, return the milliseconds spent joining it, and raise Ctrl+C caught while waiting. - pub(crate) fn stop(&self, py: Python<'_>) -> PyResult { - let started: Instant = Instant::now(); - self.stop_requested.store(true, Ordering::Release); - if let Some(preparation) = &self.preparation { - let preparation: &PreparedSqlLint = preparation.get(); - let waiting: bool = !preparation.finished(); - if let Err(error) = preparation.result(py) - && waiting - && !error.is_instance_of::(py) - { - return Err(error.clone_ref(py)); - } - } - Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) - } -} - -#[pymethods] -impl EarlySqlLint { - #[new] - #[pyo3(signature = (*, enabled, no_cache=false))] - fn py_new(enabled: bool, no_cache: bool) -> Self { - Self::new(enabled, no_cache) - } - - fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { - slf - } - - #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) { - let _ = self.stop(py); - } - - /// Keep the compile's expansion inputs, and start early SQL lint when enabled and SQL rules are selected. - #[pyo3(signature = (*, inputs, dialect, value_renderer))] - fn start( - mut slf: PyRefMut<'_, Self>, - inputs: &Bound<'_, PyAny>, - dialect: &str, - value_renderer: &Bound<'_, PyAny>, - ) -> PyResult<()> { - let py: Python<'_> = inputs.py(); - let reuse: ExpansionReuse = ExpansionReuse { - inputs: inputs.clone().unbind(), - value_renderer: value_renderer.clone().unbind(), - }; - if slf.enabled { - let stop: Arc = Arc::clone(&slf.stop_requested); - slf.preparation = - start_early_lint((inputs, dialect), reuse.clone_ref(py), slf.no_cache, &stop)? - .map(|preparation| Py::new(py, preparation)) - .transpose()?; - } - slf.expansion_reuse = (!reuse.declaration_scope(py)?.is_none()).then_some(reuse); - Ok(()) - } - - /// Start with the arguments `inputs_ready_hook` stored, as the compile's `on_inputs_ready` hook. - #[pyo3(signature = (*, inputs))] - fn __call__(slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = inputs.py(); - let Some((dialect, value_renderer)) = slf - .ready_arguments - .as_ref() - .map(|(dialect, renderer)| (dialect.clone(), renderer.clone_ref(py))) - else { - return Err(PyRuntimeError::new_err( - "EarlySqlLint was called before inputs_ready_hook armed it", - )); - }; - Self::start(slf, inputs, &dialect, value_renderer.bind(py)) - } - - /// Abandon unconsumed speculative work and return the milliseconds spent joining it. - #[pyo3(name = "stop")] - fn py_stop(&self, py: Python<'_>) -> PyResult { - self.stop(py) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs deleted file mode 100644 index c4bf41df81..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Register the versioned Python boundary for the native Rules engine. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::types::CompilerDetach; - -#[pyfunction] -fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::engine::main::custom_host::run_custom_host_json(spec_json) - }) - .map_err(value_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - crate::bindings::_helpers::rules::rows::register(module) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 12d300d1df..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,51 +1 @@ -pub(crate) mod builtin_catalogue; -pub(crate) mod catalogue; -pub(crate) mod config_json; -pub(crate) mod constants; -pub(crate) mod custom_evaluation; -pub(crate) mod custom_host_columns; -pub(crate) mod custom_host_observation; -pub(crate) mod custom_host_payload; -pub(crate) mod custom_hosts; -pub(crate) mod custom_rule_cache; -pub(crate) mod custom_rule_evidence; -pub(crate) mod dsl_headers; -pub(crate) mod early_builtin_rules; -pub(crate) mod early_sql_lint; -pub(crate) mod evaluation; -pub(crate) mod fact_encoder; -pub(crate) mod fact_replay; -pub(crate) mod fact_tree; -pub(crate) mod findings; -pub(crate) mod fix_facts; -pub(crate) mod fix_session; -pub(crate) mod fix_verdict; -pub(crate) mod fixture_formatting; -pub(crate) mod fixture_schemas; -pub(crate) mod getters; -pub(crate) mod guidance; -pub(crate) mod lint; -pub(crate) mod lint_expansion; -pub(crate) mod lint_headers; -pub(crate) mod lint_inputs; -pub(crate) mod lint_pipeline; -pub(crate) mod lint_preparation; -pub(crate) mod lint_sql_bodies; -pub(crate) mod lint_text; -pub(crate) mod native_header_parser; -pub(crate) mod project_tree_queries; pub(crate) mod registration; -pub(crate) mod results; -pub(crate) mod rows; -pub(crate) mod rule_api; -pub(crate) mod rule_cache; -pub(crate) mod rule_dispatch; -pub(crate) mod rule_plan; -pub(crate) mod rule_run; -pub(crate) mod rules_config; -pub(crate) mod run_lint; -pub(crate) mod selection; -pub(crate) mod skills; -pub(crate) mod sql_document; -pub(crate) mod sql_rules; -pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 27d01ff28b..bdb1ba144f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -1,55 +1,72 @@ //! Register the native rules, lint and DSL header functions. -use crate::bindings::_helpers::rules::{fix_facts::compile_facts, fix_verdict::fix_verdict}; use pyo3::prelude::{Bound, PyModule, PyResult}; -use pyo3::{types::PyModuleMethods, wrap_pyfunction}; +use pyo3::types::PyModuleMethods; -use crate::bindings::_helpers::rules::{ - catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, early_sql_lint, evaluation, - guidance, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_api, rule_cache, - rule_plan, rule_run, rules_config, run_lint, selection, sql_document, sql_rules, -}; +use sqlbuild_python_compile::rules::main::checking::register_dsl_headers::register_dsl_headers; +use sqlbuild_python_compile::rules::main::checking::register_evaluation::register_evaluation; +use sqlbuild_python_compile::rules::main::checking::register_lint::register_lint; +use sqlbuild_python_compile::rules::main::checking::register_project_tree_queries::register_project_tree_queries; +use sqlbuild_python_compile::rules::main::checking::register_results::register_results; +use sqlbuild_python_compile::rules::main::configuration::register_catalogue::register_catalogue; +use sqlbuild_python_compile::rules::main::configuration::register_guidance::register_guidance; +use sqlbuild_python_compile::rules::main::configuration::register_rules_config::register_rules_config; +use sqlbuild_python_compile::rules::main::configuration::register_selection::register_selection; +use sqlbuild_python_compile::rules::main::custom::register_custom_evaluation::register_custom_evaluation; +use sqlbuild_python_compile::rules::main::custom::register_custom_host_observation::register_custom_host_observation; +use sqlbuild_python_compile::rules::main::custom::register_custom_rule_cache::register_custom_rule_cache; +use sqlbuild_python_compile::rules::main::custom::register_custom_rule_evidence::register_custom_rule_evidence; +use sqlbuild_python_compile::rules::main::fact::register_fact_encoder::register_fact_encoder; +use sqlbuild_python_compile::rules::main::fix::compile_facts_function::compile_facts_function; +use sqlbuild_python_compile::rules::main::fix::fix_verdict_function::fix_verdict_function; +use sqlbuild_python_compile::rules::main::fix::plan_rule_fixes_function::plan_rule_fixes_function; +use sqlbuild_python_compile::rules::main::lint::lint_file_role_function::lint_file_role_function; +use sqlbuild_python_compile::rules::main::lint::prepare_lint_body_function::prepare_lint_body_function; +use sqlbuild_python_compile::rules::main::lint::register_early_sql_lint::register_early_sql_lint; +use sqlbuild_python_compile::rules::main::lint::register_lint_expansion::register_lint_expansion; +use sqlbuild_python_compile::rules::main::lint::register_lint_sql_bodies::register_lint_sql_bodies; +use sqlbuild_python_compile::rules::main::lint::register_lint_text::register_lint_text; +use sqlbuild_python_compile::rules::main::lint::register_run_lint::register_run_lint; +use sqlbuild_python_compile::rules::main::rule::register_rule_api::register_rule_api; +use sqlbuild_python_compile::rules::main::rule::register_rule_cache::register_rule_cache; +use sqlbuild_python_compile::rules::main::rule::register_rule_dispatch::register_rule_dispatch; +use sqlbuild_python_compile::rules::main::rule::register_rule_plan::register_rule_plan; +use sqlbuild_python_compile::rules::main::rule::register_rule_run::register_rule_run; +use sqlbuild_python_compile::rules::main::sql::register_fixture_formatting::register_fixture_formatting; +use sqlbuild_python_compile::rules::main::sql::register_sql_document::register_sql_document; +use sqlbuild_python_compile::rules::main::sql::register_sql_rules::register_sql_rules; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - rule_api::register(module)?; - sql_document::register(module)?; - crate::bindings::_helpers::rules::project_tree_queries::register(module)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_headers::lint_file_role, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::prepare_lint_body, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::fix_session::plan_rule_fixes, - module - )?)?; - module.add_function(wrap_pyfunction!(compile_facts, module)?)?; - module.add_function(wrap_pyfunction!(fix_verdict, module)?)?; - evaluation::register(module)?; - lint::register(module)?; - lint_text::register(module)?; - crate::bindings::_helpers::rules::fixture_formatting::register(module)?; - lint_expansion::register(module)?; - lint_sql_bodies::register(module)?; - dsl_headers::register(module)?; - custom_rule_evidence::register(module)?; - rule_cache::register(module)?; - rule_run::register(module)?; - rule_plan::register(module)?; - sql_rules::register(module)?; - early_sql_lint::register(module)?; - custom_rule_cache::register(module)?; - catalogue::register(module)?; - rules_config::register(module)?; - run_lint::register(module)?; - selection::register(module)?; - results::register(module)?; - guidance::register(module)?; - crate::bindings::_helpers::rules::custom_evaluation::register(module)?; - crate::bindings::_helpers::rules::fact_encoder::register(module)?; - crate::bindings::_helpers::rules::custom_host_observation::register(module)?; - crate::bindings::_helpers::rules::rule_dispatch::register(module) + register_rule_api(module)?; + register_sql_document(module)?; + register_project_tree_queries(module)?; + module.add_function(lint_file_role_function(module)?)?; + module.add_function(prepare_lint_body_function(module)?)?; + module.add_function(plan_rule_fixes_function(module)?)?; + module.add_function(compile_facts_function(module)?)?; + module.add_function(fix_verdict_function(module)?)?; + register_evaluation(module)?; + register_lint(module)?; + register_lint_text(module)?; + register_fixture_formatting(module)?; + register_lint_expansion(module)?; + register_lint_sql_bodies(module)?; + register_dsl_headers(module)?; + register_custom_rule_evidence(module)?; + register_rule_cache(module)?; + register_rule_run(module)?; + register_rule_plan(module)?; + register_sql_rules(module)?; + register_early_sql_lint(module)?; + register_custom_rule_cache(module)?; + register_catalogue(module)?; + register_rules_config(module)?; + register_run_lint(module)?; + register_selection(module)?; + register_results(module)?; + register_guidance(module)?; + register_custom_evaluation(module)?; + register_fact_encoder(module)?; + register_custom_host_observation(module)?; + register_rule_dispatch(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs deleted file mode 100644 index eece6e5c03..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Rules results rendered for the CLI, and SQLCLEAN044 help naming files that repeat a literal. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use sqlbuild_rules::rule_results::main::format_result::format_result; -use sqlbuild_rules::rule_results::main::format_rules_progress::format_rules_progress; -use sqlbuild_rules::rule_results::main::format_skipped_rules::format_skipped_rules; -use sqlbuild_rules::rule_results::models::{ResultCounts, RuleFinding}; - -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::main::fields::path_posix::path_posix; - -pub(crate) fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { - Ok(RuleFinding { - code: field(finding, "code")?.extract()?, - path: path_posix(&field(finding, "path")?)?, - line: field(finding, "line")?.extract()?, - column: field(finding, "column")?.extract()?, - message: field(finding, "message")?.extract()?, - remediation: field(finding, "remediation")?.extract()?, - unevaluated: field(finding, "unevaluated")?.is_truthy()?, - affected_rules: field(finding, "affected_rules")?.extract()?, - fixable: field(finding, "fixable")?.is_truthy()?, - }) -} - -fn rule_findings(findings: &Bound<'_, PyAny>) -> PyResult> { - findings - .try_iter()? - .map(|finding| rule_finding(&finding?)) - .collect() -} - -/// One rules result for the requested CLI surface. -#[pyfunction] -#[pyo3(signature = (*, result, json_output))] -pub(crate) fn format_rules_result( - result: &Bound<'_, PyAny>, - json_output: bool, -) -> PyResult { - let counts = ResultCounts { - evaluated_models: field(result, "evaluated_models")?.extract()?, - unevaluated_resources: field(result, "unevaluated_resources")?.extract()?, - cache_hits: field(result, "cache_hits")?.extract()?, - cache_misses: field(result, "cache_misses")?.extract()?, - }; - Ok(format_result( - &rule_findings(&field(result, "findings")?)?, - counts, - json_output, - )) -} - -/// The rules phase completion line for one `RulesRunResult`, optionally noting skipped rules. -pub(crate) fn rules_progress( - elapsed_seconds: f64, - result: &Bound<'_, PyAny>, - note_skipped_rules: bool, -) -> PyResult { - let skipped: Option> = if note_skipped_rules { - Some(field(result, "skipped_type_proof_rules")?.extract()?) - } else { - None - }; - Ok(format_rules_progress( - elapsed_seconds, - ( - field(result, "built_in_ms")?.extract()?, - field(result, "custom_ms")?.extract()?, - ), - skipped.as_deref(), - )) -} - -/// Describe type-proof rules skipped because SQL analysis is disabled, if any. -#[pyfunction] -#[pyo3(signature = (*, codes))] -pub(crate) fn format_skipped_type_proof_rules(codes: Vec) -> Option { - format_skipped_rules(&codes) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_rules_result, module)?)?; - module.add_function(wrap_pyfunction!(format_skipped_type_proof_rules, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index e7c239a10b..0dc66fee49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -24,8 +24,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -39,6 +37,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::run_compile_only_pipeline::run_compile_only_pipeline; +use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; use sqlbuild_python_front::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 3ab6a93648..ba5946bdb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -1,3 +1,2 @@ pub(crate) mod registration; pub(crate) mod scope_command; -pub(crate) mod scope_loading; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs index 745c924fe0..c0558c9166 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::scopes::{scope_command, scope_loading}; +use crate::bindings::_helpers::scopes::scope_command; +use sqlbuild_python_compile::scopes::main::register_scope_loading::register_scope_loading; use sqlbuild_python_front::scopes::main::checks::register_scope_validation::register_scope_validation; use sqlbuild_python_front::scopes::main::checks::register_scope_visibility::register_scope_visibility; use sqlbuild_python_front::scopes::main::declarations::register_declaration_contexts::register_declaration_contexts; @@ -20,7 +21,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_compile_usages(module)?; register_scope_lookup(module)?; register_scope_cache(module)?; - scope_loading::register(module)?; + register_scope_loading(module)?; register_scope_reports(module)?; scope_command::register(module)?; register_scope_validation(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 58eb210cb7..9cd852dc71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -13,7 +13,6 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; -use crate::bindings::_helpers::scopes::scope_loading::load_or_build_scope_index; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -22,6 +21,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::scopes::main::load_or_build_scope_index::load_or_build_scope_index; use sqlbuild_python_front::scopes::main::index::build_scope_lookup::build_scope_lookup; use sqlbuild_python_front::scopes::main::reports::read_result::read_result; use sqlbuild_python_front::scopes::main::reports::scope_browse::scope_browse; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 28c9cb55df..9a8f3cf4b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -19,7 +19,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; @@ -35,6 +34,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs deleted file mode 100644 index 8061ca065b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The semantic diagnostic help catalogue, for Python's tests and corpus tooling. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::semantic_checks::main::semantic_help_catalogue::semantic_help_catalogue as catalogue; - -/// Each semantic diagnostic code and its help. -#[pyfunction] -fn semantic_help_catalogue() -> Vec<(&'static str, &'static str)> { - catalogue().to_vec() -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(semantic_help_catalogue, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 692d959381..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,12 +1 @@ -pub(crate) mod completion; -pub(crate) mod constants; -pub(crate) mod failures; -pub(crate) mod help_catalogue; -pub(crate) mod inputs; -pub(crate) mod metadata; pub(crate) mod registration; -pub(crate) mod resource_sql; -pub(crate) mod session_facts; -pub(crate) mod stage; -pub(crate) mod stage_objects; -pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 487d03d3ea..3d2f9b7239 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_checks::{help_catalogue, resource_sql, stage}; +use sqlbuild_python_compile::semantic_checks::main::register_help_catalogue::register_help_catalogue; +use sqlbuild_python_compile::semantic_checks::main::register_resource_sql::register_resource_sql; +use sqlbuild_python_compile::semantic_checks::main::register_stage::register_stage; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - resource_sql::register(module)?; - help_catalogue::register(module)?; - stage::register(module) + register_resource_sql(module)?; + register_help_catalogue(module)?; + register_stage(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index a097921e9d..9ddd412a8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -13,7 +13,6 @@ use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; -use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; use sqlbuild_python_base::build_progress::main::terminal::flush::flush; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::native_host::main::flush_console::flush_console; @@ -22,6 +21,7 @@ use sqlbuild_python_base::native_host::main::python_executable::python_executabl use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 55b78be584..a28e34fba2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -38,9 +38,7 @@ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::statement_report::StatementReport; use crate::bindings::_helpers::result_output::documents::test_execution_json; @@ -70,6 +68,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; +use sqlbuild_python_compile::compile_outputs::main::write_test_runtime_target::write_test_runtime_target; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_planner::planner_entries::main::plans::build_test_command_plan::build_test_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index a2b5a62391..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,3 +1 @@ -pub(crate) mod assembly; -pub(crate) mod compiled_tests; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 52e8ff7e43..4da1fe6783 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::sql_test_glue::compiled_tests; +use sqlbuild_python_compile::sql_test_glue::main::register_compiled_tests::register_compiled_tests; use sqlbuild_python_front::sql_test_glue::main::register_authored_ctes::register_authored_ctes; use sqlbuild_python_project::sql_test_glue::main::plan_compiled_sql_test_artifacts_function::plan_compiled_sql_test_artifacts_function; use sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_function::plan_sql_tests_function; @@ -15,5 +15,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(plan_compiled_sql_test_artifacts_function(module)?)?; module.add_function(sql_test_plan_error_messages_function(module)?)?; register_authored_ctes(module)?; - compiled_tests::register(module) + register_compiled_tests(module) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 2eb99d1af5..93ad5e4aec 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -3,13 +3,13 @@ use pyo3::pymodule; use sqlbuild_rules::constants::API_VERSION; use crate::bindings::_helpers::{ - analysis, analysis_session, attachments, auditing, authored_values, cache, cli, - compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, - contracts, cursor, dag_artifact, diagnostics, discovery, exported_constants, fingerprints, - frontier, graph, lifecycle_events, lineage, macro_bridge, manifest, microbatches, model_config, - native_host, plan_output, planner_registration, project_assembly, record_behaviours, refactor, - references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - state_rows, type_system, + analysis_session, attachments, auditing, authored_values, cache, cli, compile_attachment, + compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, + dag_artifact, diagnostics, discovery, exported_constants, fingerprints, frontier, graph, + lifecycle_events, lineage, macro_bridge, manifest, microbatches, model_config, native_host, + plan_output, planner_registration, project_assembly, record_behaviours, refactor, references, + render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use sqlbuild_python_base::boundary::main::register_expected_errors::register_expected_errors; use sqlbuild_python_base::boundary::main::register_native_compiler_error::register_native_compiler_error; @@ -18,6 +18,9 @@ use sqlbuild_python_base::records::main::registry::capture::capture; use sqlbuild_python_base::records::main::registry::register_records::register_records; use sqlbuild_python_base::records::main::registry::verify_attributes::verify_attributes; use sqlbuild_python_base::records::main::registry::verify_tables::verify_tables; +use sqlbuild_python_compile::analysis::main::register_queries::register_queries; +use sqlbuild_python_compile::analysis::main::register_sql_test_comparisons::register_sql_test_comparisons; +use sqlbuild_python_compile::analysis::main::register_sql_tests::register_sql_tests; use sqlbuild_python_project::analysis::main::register_normalization::register_normalization; use sqlbuild_python_project::analysis::models::BindingPositions; use sqlbuild_python_project::analysis::models::CompactAnalysisJob; @@ -36,8 +39,8 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { rules::registration::register(module)?; cli::registration::register(module)?; register_normalization(module)?; - analysis::queries::register(module)?; - analysis::sql_tests::register(module)?; + register_queries(module)?; + register_sql_tests(module)?; sqltext::registration::register(module)?; register_oracles(module)?; register_expected_errors(module)?; @@ -70,7 +73,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { microbatches::registration::register(module)?; lifecycle_events::registration::register(module)?; diagnostics::registration::register(module)?; - analysis::sql_test_comparisons::register(module)?; + register_sql_test_comparisons(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 69ffd18969..f5193f059d 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -1,16 +1,8 @@ -#[path = "tests/test_coverage.rs"] -mod coverage; -#[path = "tests/test_delta_facts.rs"] -mod delta_facts; #[path = "tests/test_failure_cause.rs"] mod failure_cause; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_image.rs"] -mod image; #[path = "tests/test_since_cache.rs"] mod since_cache; -#[path = "tests/test_splice.rs"] -mod splice; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index 948e5a6d98..b60ca65398 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,266 +1,8 @@ -use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; -use sqlbuild_cache::project_reuse::types::ImageSections; - use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; -use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; -use crate::bindings::_helpers::compile_delta::image::{ - FACTS_SECTION, GATE_SECTION, IMAGE_FORMAT, ImageRelations, ImageRules, ProjectImage, - RULE_DIGESTS_SECTION, -}; -use sqlbuild_python_base::compile_context::models::BaseRules; -use sqlbuild_python_base::compile_context::models::CapturedDiagnostic; -use sqlbuild_python_base::compile_context::models::CapturedModel; -use sqlbuild_python_base::compile_context::models::CapturedTest; -use sqlbuild_python_base::compile_context::models::ModelCoverage; -use sqlbuild_python_base::compile_context::models::PendingRules; - -/// The rows of a request without models: a zero model count. -const RULE_ROWS: [u8; 8] = [0; 8]; -const OTHER_FORMAT: &str = "sqlbuild-project-image-v0"; - -pub(crate) fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { - values - .iter() - .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) - .collect() -} - -pub(crate) fn strings(values: &[&str]) -> Vec { - values.iter().map(|value| (*value).to_owned()).collect() -} - -/// An image holding at least one value of every kind it stores. -pub(crate) fn image() -> ProjectImage { - let order_types: Vec<(String, String)> = pairs(&[("order_id", "BIGINT"), ("amount", "DOUBLE")]); - let customer_types: Vec<(String, String)> = pairs(&[("customer_id", "BIGINT")]); - let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(); - ProjectImage { - format: String::new(), - json_output: true, - models: vec![ - CapturedModel { - name: "orders".to_owned(), - relative_path: "models/orders.sql".to_owned(), - upstream: pairs(&[("source", "raw_orders"), ("model", "customers")]), - }, - CapturedModel { - name: "customers".to_owned(), - relative_path: "models/customers.sql".to_owned(), - upstream: Vec::new(), - }, - ], - usages: BTreeMap::from([("models/orders.sql".to_owned(), "digest-1".to_owned())]), - tests: vec![CapturedTest { - key: "orders_test".to_owned(), - relative_path: "tests/unit/orders.sql".to_owned(), - chain: strings(&["orders", "customers"]), - }], - relations: ImageRelations { - types: vec![ - ("orders".to_owned(), order_types.clone()), - ("customers".to_owned(), customer_types.clone()), - ], - nullability: vec![("orders".to_owned(), pairs(&[("order_id", "never")]))], - complete: vec![("orders".to_owned(), order_types.clone())], - schemas: vec![("customers".to_owned(), pairs(&[("customer_id", "INTEGER")]))], - dependency_ordered: true, - cyclic: false, - legacy_models: strings(&["customers"]), - families: vec![( - "order_pivot".to_owned(), - vec![DynamicFamily { - name: "category_amounts".to_owned(), - pivot_column: "category".to_owned(), - value_column: "amount".to_owned(), - aggregate: "MAX".to_owned(), - data_type: "DOUBLE".to_owned(), - name_pattern: Some("^amount_".to_owned()), - }], - )], - }, - fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], - model_subjects: Some(vec![ - ( - "models/orders.sql".to_owned(), - "orders".to_owned(), - Some("table".to_owned()), - ), - ( - "models/customers.sql".to_owned(), - "customers".to_owned(), - None, - ), - ]), - diagnostics: vec![CapturedDiagnostic { - json: "{\"code\": \"W1\"}".to_owned(), - resource_name: Some("orders".to_owned()), - paths: strings(&["models/orders.sql"]), - error: false, - }], - text_models: Some(vec![("orders".to_owned(), true, 3)]), - environment_names: strings(&["SQLBUILD_TARGET"]), - pinned_sources: strings(&["raw_customers"]), - test_sources: BTreeMap::from([( - "tests/unit/orders.sql".to_owned(), - strings(&["raw_orders"]), - )]), - model_shapes: vec![ - ("orders".to_owned(), order_types), - ("customers".to_owned(), pairs(&[("customer_id", "VARCHAR")])), - ], - coverage: Some(vec![ModelCoverage { - name: "orders".to_owned(), - disabled: false, - blocked: true, - partial: strings(&["open source: raw_orders"]), - }]), - lint_model_keys: Some(pairs(&[("models/orders.sql", "lint-key")])), - lint_file_keys: Some(vec![ - ("models/orders.sql".to_owned(), Some("file-key".to_owned())), - ("models/customers.sql".to_owned(), None), - ]), - header_keys: BTreeMap::from([( - "models/orders.sql".to_owned(), - pairs(&[("description", "0123")]), - )]), - audit_references: BTreeMap::from([( - "models/orders.sql".to_owned(), - strings(&["order_quality"]), - )]), - rules: Some(ImageRules::Retained(RetainedRows { - request_json: request, - rows: RULE_ROWS.to_vec(), - digests: Vec::new(), - })), - } -} - -/// Every stored fact of `image` but its rules, in a comparable form. -pub(crate) fn facts(image: &ProjectImage) -> String { - format!( - "{:?}", - ( - ( - &image.json_output, - &image.models, - &image.usages, - &image.tests, - &image.relations, - &image.fact_digests, - &image.model_subjects, - ), - ( - &image.diagnostics, - &image.text_models, - &image.environment_names, - &image.pinned_sources, - &image.test_sources, - &image.model_shapes, - &image.coverage, - &image.lint_model_keys, - &image.lint_file_keys, - &image.header_keys, - &image.audit_references, - ), - ) - ) -} - -pub(crate) fn current_format() -> String { - IMAGE_FORMAT.to_owned() -} - -pub(crate) fn encoded(image: ProjectImage) -> ImageSections { - let stored: ProjectImage = ProjectImage { - format: current_format(), - ..image - }; - stored.encode().unwrap_or_default() -} - -/// The image `image()` without any of its optional facts. -pub(crate) fn sparse_image() -> ProjectImage { - ProjectImage { - model_subjects: None, - text_models: None, - lint_model_keys: None, - lint_file_keys: None, - rules: None, - ..image() - } -} - -/// The rules request a decoded image holds, resolved. -pub(crate) fn decoded_rules(image: ProjectImage) -> Option { - image - .rules - .and_then(ImageRules::pending) - .and_then(PendingRules::resolved) -} - -/// `sections` with the gate of an image of another format. -pub(crate) fn with_other_format(sections: &mut ImageSections) { - let other: ProjectImage = ProjectImage { - format: OTHER_FORMAT.to_owned(), - ..ProjectImage::default() - }; - let gate: Vec = other - .encode() - .unwrap_or_default() - .into_iter() - .find(|(name, _)| name == GATE_SECTION) - .map(|(_, bytes)| bytes) - .unwrap_or_default(); - sections - .iter_mut() - .filter(|(name, _)| name == GATE_SECTION) - .for_each(|section| section.1 = gate.clone()); -} - -/// `sections` with their facts cut in half. -pub(crate) fn with_truncated_facts(sections: &mut ImageSections) { - sections - .iter_mut() - .filter(|(name, _)| name == FACTS_SECTION) - .for_each(|(_, bytes)| bytes.truncate(bytes.len() / 2)); -} - -/// `sections` without the rules request's digests. -pub(crate) fn without_rule_digests(sections: &mut ImageSections) { - sections.retain(|(name, _)| name != RULE_DIGESTS_SECTION); -} - -/// The names in `values`, as a set. -pub(crate) fn names(values: &[&str]) -> BTreeSet { - values.iter().map(|value| (*value).to_owned()).collect() -} - -/// One model's semantic coverage facts. -pub(crate) fn coverage( - name: &str, - (disabled, blocked): (bool, bool), - partial: &[&str], -) -> ModelCoverage { - ModelCoverage { - name: name.to_owned(), - disabled, - blocked, - partial: strings(partial), - } -} - -/// Models paired with their upstream names, or with their reasons. -pub(crate) fn order(models: &[(&str, &[&str])]) -> Vec<(String, Vec)> { - models - .iter() - .map(|(name, upstream)| ((*name).to_owned(), strings(upstream))) - .collect() -} /// A connection failure's cause, as `sqb debug` reports it. pub(crate) fn connection_cause() -> FailureCause { diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index aa09bf8593..b5648a3b87 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -1,62 +1,3 @@ -use sqlbuild_cache::project_reuse::types::ImageSections; - -use crate::bindings::_helpers::compile_delta::image::ProjectImage; -use sqlbuild_python_base::compile_context::models::ModelCoverage; - -/// A stored report, a delta report and what splicing the delta into the stored one gives. -pub(crate) struct SpliceTestCase { - pub(crate) description: &'static str, - pub(crate) base: &'static str, - pub(crate) delta: &'static str, - pub(crate) models: &'static [&'static str], - pub(crate) tests: &'static [&'static str], - pub(crate) sources: Option<&'static [&'static str]>, - /// The spliced report, or `None` when the reports must not line up. - pub(crate) expected_report: Option<&'static str>, -} - -/// Stored image bytes damaged before they are read back. -pub(crate) struct DamagedImageTestCase { - pub(crate) description: &'static str, - pub(crate) damage: fn(&mut ImageSections), - pub(crate) expected_gate: bool, - pub(crate) expected_image: bool, -} - -/// An image encoded and decoded again. -pub(crate) struct ImageRoundTripTestCase { - pub(crate) description: &'static str, - pub(crate) image: fn() -> ProjectImage, - pub(crate) expected_model_subjects: bool, - /// The retained rules request's model and digest count, or `None` without one. - pub(crate) expected_rule_models: Option, -} - -/// Stored and fresh semantic coverage facts, and the whole project's partial-check reasons. -pub(crate) struct CoverageTestCase { - pub(crate) description: &'static str, - pub(crate) stored: Vec, - pub(crate) fresh: Vec, - pub(crate) stored_order: Vec<(String, Vec)>, - pub(crate) fresh_order: Vec<(String, Vec)>, - pub(crate) expected_reasons: Vec<(String, Vec)>, -} - -/// An error message a delta round reported. -pub(crate) struct UncompiledModelTestCase { - pub(crate) description: &'static str, - pub(crate) message: &'static str, - pub(crate) expected_uncompiled: bool, -} - -/// A custom-rule fact key a delta compile holding some models may or may not read in full. -pub(crate) struct DeltaFactTestCase { - pub(crate) description: &'static str, - pub(crate) key: &'static [&'static str], - pub(crate) all_sources: bool, - pub(crate) expected_safe: bool, -} - pub(crate) struct FailureCauseTestCase { pub(crate) description: &'static str, pub(crate) report: fn(), diff --git a/fensu.toml b/fensu.toml index 667fedb53c..2fe267411b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1435,7 +1435,7 @@ paths = ["crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs", "crat reason = "ClickHouse and relational backends each list their own view bodies; the view names match because they are one public view contract." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs", "crates/sqlbuild-python/src/bindings/main/register.rs"] +paths = ["crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs", "crates/sqlbuild-python/src/bindings/main/register.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] @@ -1443,7 +1443,7 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgre reason = "PostgreSQL and SQLite execution history each read a page through their own connection, locking and cursor keys; filters and record conversion come from conversion.rs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] +paths = ["crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." [[dupes.allowlist]] @@ -1455,7 +1455,7 @@ paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/constr reason = "The public entry forwards to the crate-internal constructor that the crate's own helpers call." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs"] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs"] reason = "Each loads the members of its own host enum." [[dupes.allowlist]] @@ -1670,12 +1670,12 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 115238 } +thresholds = { max_crate_code_lines = 66831 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] -thresholds = { max_helpers_container_buckets = 96 } +thresholds = { max_helpers_container_buckets = 93 } reason = "The bindings helpers container is above the bucket limit; this caps it at its current bucket count until cohesive groups move into their own modules or crates. Lower the cap as buckets move out." [targets.rust.rule_options.XSB076] @@ -1860,6 +1860,24 @@ path = "crates/sqlbuild-python-project/src/resource_views/models.rs" symbols = ["SeedNamespace::extract_bound"] reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-compile/src/compile_outputs/models.rs" +symbols = ["SettingsRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-compile/src/input_loops/models.rs" +symbols = ["NativeModelLoop"] +reason = "NativeModelLoop implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-compile/src/input_loops/models.rs" +symbols = ["LoopSwitches::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -2023,6 +2041,7 @@ crate_names = [ "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", + "sqlbuild-python-compile", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2048,7 +2067,7 @@ paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] @@ -2135,6 +2154,154 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registration.rs=26", "crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs=2", "crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs=3", + "crates/sqlbuild-python-compile/src/analysis/main/single_comparison_sql.rs=2", + "crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs=10", + "crates/sqlbuild-python-compile/src/analysis_session/_helpers/outcome_objects.rs=2", + "crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs=3", + "crates/sqlbuild-python-compile/src/auditing/_helpers/measurement.rs=1", + "crates/sqlbuild-python-compile/src/auditing/_helpers/schema_audits.rs=1", + "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/declaration_scope.rs=9", + "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs=4", + "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/sql_expansion_context.rs=2", + "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/storage_policies.rs=1", + "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/target.rs=8", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs=41", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/surrogates.rs=2", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs=22", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/test_artifact_cache.rs=39", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs=30", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs=45", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs=27", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/runtime_identity.rs=12", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/startup_inputs.rs=4", + "crates/sqlbuild-python-compile/src/compile_command/main/invocation/interpreter_values.rs=2", + "crates/sqlbuild-python-compile/src/compile_command/main/invocation/print_to.rs=5", + "crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs=2", + "crates/sqlbuild-python-compile/src/compile_command/models.rs=4", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs=12", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_checks.rs=5", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs=6", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_sql_tests.rs=6", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/pipeline_entries.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs=7", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs=7", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs=4", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs=8", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis_cache.rs=20", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs=21", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs=12", + "crates/sqlbuild-python-compile/src/compile_driver/main/setup/compile_adapter_context.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/main/setup/compiled_phase.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/main/setup/progress.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs=4", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/monotonic.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_configured_rule_selection.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolved_request.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/store_and_print.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/main/stages/unit_test_selectors_applied.rs=2", + "crates/sqlbuild-python-compile/src/compile_outputs/_helpers/artifacts.rs=3", + "crates/sqlbuild-python-compile/src/compile_outputs/_helpers/runtime_target.rs=6", + "crates/sqlbuild-python-compile/src/compile_outputs/main/write_python_check_runtime_target.rs=3", + "crates/sqlbuild-python-compile/src/compile_outputs/main/write_runtime_target.rs=1", + "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/artifact.rs=1", + "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/rows.rs=8", + "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/selection.rs=7", + "crates/sqlbuild-python-compile/src/frontier/_helpers/compile_frontier.rs=2", + "crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs=9", + "crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs=5", + "crates/sqlbuild-python-compile/src/hand_built/_helpers/compiled_model.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs=4", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_calls.rs=4", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_external.rs=4", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_header.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_hooks.rs=6", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs=6", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_named.rs=5", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs=7", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_python_hooks.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_schema.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs=5", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs=10", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs=10", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs=2", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs=10", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs=5", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/scenario_loop.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs=4", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs=10", + "crates/sqlbuild-python-compile/src/input_loops/main/attach_checks.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/main/model_loop_classes.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/models.rs=2", + "crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs=3", + "crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_lineage.rs=10", + "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs=20", + "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs=5", + "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_objects.rs=4", + "crates/sqlbuild-python-compile/src/lineage/main/build_project_column_lineage.rs=1", + "crates/sqlbuild-python-compile/src/lineage/main/graph_index.rs=3", + "crates/sqlbuild-python-compile/src/lineage/main/read_lineage_cache.rs=1", + "crates/sqlbuild-python-compile/src/lineage/main/write_lineage_cache.rs=7", + "crates/sqlbuild-python-compile/src/lineage/models.rs=18", + "crates/sqlbuild-python-compile/src/manifest/_helpers/builder.rs=1", + "crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs=4", + "crates/sqlbuild-python-compile/src/manifest/main/build_manifest.rs=7", + "crates/sqlbuild-python-compile/src/project_assembly/_helpers/declared_relations.rs=1", + "crates/sqlbuild-python-compile/src/project_assembly/_helpers/native_resources.rs=3", + "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs=1", + "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/findings.rs=6", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs=8", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs=5", + "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/builtin_catalogue.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs=7", + "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/guidance.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/rules_config.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/selection.rs=8", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs=5", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_observation.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs=10", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_cache.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs=20", + "crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_encoder.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_facts.rs=8", + "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs=8", + "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_verdict.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_expansion.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_headers.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_inputs.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs=4", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_preparation.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs=3", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_plan.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_run.rs=5", + "crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_rules.rs=10", + "crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_test_objects.rs=1", + "crates/sqlbuild-python-compile/src/rules/main/configuration/install_rules_skills.rs=4", + "crates/sqlbuild-python-compile/src/rules/main/fix/plan_rule_fixes.rs=1", + "crates/sqlbuild-python-compile/src/rules/main/sql/format_redundant_fixture_nulls.rs=2", + "crates/sqlbuild-python-compile/src/rules/models.rs=1", + "crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs=1", + "crates/sqlbuild-python-compile/src/scopes/main/load_or_build_scope_index.rs=1", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/completion.rs=1", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs=6", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs=6", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage_objects.rs=2", + "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/assembly.rs=2", + "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs=4", "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=0", "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=2", "crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs=1", @@ -2454,13 +2621,7 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs=2", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=1", "crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", @@ -2495,48 +2656,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs=45", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=38", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", @@ -2553,9 +2675,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs=3", @@ -2608,32 +2727,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_python_hooks.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=6", @@ -2642,30 +2737,18 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", @@ -2699,38 +2782,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=3", @@ -2738,15 +2789,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=97", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", ] @@ -2780,6 +2824,67 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs=1", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/error_classes.rs=1", "crates/sqlbuild-python-base/src/records/models.rs=3", + "crates/sqlbuild-python-compile/src/analysis_session/models.rs=8", + "crates/sqlbuild-python-compile/src/compile_attachment/models.rs=5", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs=7", + "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs=6", + "crates/sqlbuild-python-compile/src/compile_command/models.rs=17", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs=5", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/inputs.rs=15", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs=8", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs=3", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/target_defaults.rs=4", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs=12", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs=7", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs=5", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs=1", + "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs=2", + "crates/sqlbuild-python-compile/src/compile_driver/models.rs=11", + "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs=19", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_config.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs=2", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_header.rs=5", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_hooks.rs=15", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs=6", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs=6", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs=2", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_python_hooks.rs=3", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_schema.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs=14", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs=1", + "crates/sqlbuild-python-compile/src/input_loops/models.rs=3", + "crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs=4", + "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs=2", + "crates/sqlbuild-python-compile/src/lineage/models.rs=18", + "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs=3", + "crates/sqlbuild-python-compile/src/project_assembly/models.rs=18", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/findings.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs=4", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_observation.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs=2", + "crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_encoder.rs=3", + "crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs=4", + "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs=10", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_expansion.rs=5", + "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs=1", + "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs=6", + "crates/sqlbuild-python-compile/src/rules/_helpers/sql/fixture_schemas.rs=4", + "crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_rules.rs=3", + "crates/sqlbuild-python-compile/src/rules/models.rs=6", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs=3", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs=4", + "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage_objects.rs=14", + "crates/sqlbuild-python-compile/src/semantic_checks/models.rs=4", + "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/assembly.rs=2", + "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs=8", "crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs=3", "crates/sqlbuild-python-front/src/authored_values/models.rs=6", "crates/sqlbuild-python-front/src/compile_driver/models.rs=3", @@ -2870,7 +2975,6 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=4", "crates/sqlbuild-python-project/src/sql_test_glue/models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=6", @@ -2893,32 +2997,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs=25", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=2", @@ -2936,28 +3020,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_python_hooks.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", @@ -2966,9 +3032,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs=2", @@ -2979,32 +3042,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=8", ] [targets.rust.rule_options.XSB075] @@ -3012,10 +3055,10 @@ layer_order = ["infra", "front", "project", "plan|compile", "output", "commands" module_layers = [ "*/registration=registry", "analysis=compile", - "analysis_session=compile", + "analysis_session=registry", "attachments=registry", "audit_command=commands", - "auditing=compile", + "auditing=registry", "authored_values=registry", "build_command=commands", "build_progress=output", @@ -3025,14 +3068,14 @@ module_layers = [ "cli=commands", "clone=commands", "command_runtime=commands", - "compile_attachment=compile", - "compile_command=compile", + "compile_attachment=registry", + "compile_command=registry", "compile_delta=compile", "compile_driver=compile", "compile_driver/plan_pipeline=commands", "compile_driver/plan_result=commands", "compile_driver/static_context=commands", - "compile_outputs=compile", + "compile_outputs=registry", "compiled_project=registry", "compiled_views=registry", "compute_logs=output", @@ -3043,7 +3086,7 @@ module_layers = [ "cost_report=output", "cost_telemetry=output", "cursor=registry", - "dag_artifact=compile", + "dag_artifact=registry", "dagster_invocation=commands", "dbt_command=commands", "dbt_events=commands", @@ -3059,12 +3102,11 @@ module_layers = [ "format_command=commands", "freshness=plan", "freshness/freshness_command=commands", - "frontier=compile", + "frontier=registry", "graph=registry", "hand_built=commands", - "hand_built/compiled_model=compile", "init_command=commands", - "input_loops=compile", + "input_loops=registry", "janitor=commands", "lifecycle_events=registry", "lineage=compile", @@ -3072,7 +3114,7 @@ module_layers = [ "load=commands", "location_command=commands", "macro_bridge=registry", - "manifest=compile", + "manifest=registry", "microbatches=registry", "model_config=registry", "native_host=front", @@ -3096,7 +3138,7 @@ module_layers = [ "planner_strategy=registry", "planner_warehouse=registry", "presentation_text=registry", - "project_assembly=compile", + "project_assembly=registry", "query_command=commands", "query_command/text_output=output", "recall=output", @@ -3110,16 +3152,15 @@ module_layers = [ "render=registry", "result_output=output", "review_command=commands", - "rules=compile", + "rules=registry", "scenario_command=commands", "scopes=front", "scopes/scope_command=commands", - "scopes/scope_loading=compile", "seed_command=commands", - "semantic_checks=compile", + "semantic_checks=registry", "semantic_shapes=registry", "sql_test_command=commands", - "sql_test_glue=compile", + "sql_test_glue=registry", "sqltext=registry", "state_rows=registry", "type_system=registry", @@ -3160,11 +3201,11 @@ paths = ["crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs", reason = "Each is a short stack walk over a JSON syntax tree with its own stop and collect rule: cast_targets collects every cast's target type, branch_selects stops at each set-operation branch, and walked lists every object. A shared visitor would need a callback for each rule and save no logic." [[dupes.allowlist]] -paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs"] +paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs"] reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3188,7 +3229,7 @@ paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sq reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/getters.rs"] +paths = ["crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs"] reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." [[dupes.allowlist]] @@ -3228,11 +3269,11 @@ paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sql reason = "The transient status line and the build progress callbacks each own their spinner thread and stop event; merging the two spinners is a separate change." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs"] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/recovery/resource_sql.rs"] reason = "Each reads its own Python mapping into its own native pairs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs"] +paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/rows.rs"] reason = "Each reads its own Python record's fields into its own native record." [[dupes.allowlist]] @@ -3268,7 +3309,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3312,7 +3353,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/pro reason = "Each compiled view reads its own field set from the Python boundary." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/rows.rs"] reason = "Each converts the declaration row of its own consumer into its own declaration type." [[dupes.allowlist]] @@ -3340,11 +3381,11 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_visibility.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3352,11 +3393,11 @@ paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_ reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs"] reason = "Each reads a resource key's type and name into its own native key; consolidating the key readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3364,7 +3405,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/in reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] +paths = ["crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies.rs"] reason = "Each reads its own Python record's fields into its own native record." [[dupes.allowlist]] @@ -3388,7 +3429,7 @@ paths = ["crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_ reason = "Each builds its own index from its own node list; only the index-building loop is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." [[dupes.allowlist]] @@ -3396,23 +3437,23 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/alw reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python-compile/src/analysis/_helpers/sql_test_comparisons.rs", "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs", "crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs", "crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs", "crates/sqlbuild-python-project/src/references/main/register_relations.rs", "crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs"] +paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs", "crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs", "crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs", "crates/sqlbuild-python-project/src/references/main/register_relations.rs", "crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_run.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python-compile/src/analysis_session/_helpers/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python-compile/src/hand_built/_helpers/compiled_model.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage_objects.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3420,11 +3461,11 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/resource_reads/ reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs", "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/builtin_catalogue.rs", "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/test_artifact_cache.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis_cache.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] reason = "Each reads the installed package version for its own output with its own fallback; only the metadata lookup is alike." [[dupes.allowlist]] @@ -3444,7 +3485,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/audits. reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs"] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." [[dupes.allowlist]] @@ -3452,11 +3493,11 @@ paths = ["crates/sqlbuild-python-front/src/model_config/main/config/config_curso reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs"] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_checks.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs", "crates/sqlbuild-python-project/src/graph/main/project/register_views.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs", "crates/sqlbuild-python-project/src/graph/main/project/register_views.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3464,11 +3505,11 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/plain_t reason = "One converts a typed SQL value to plain Python values, the other to JSON-ready values; only the kind dispatch is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs"] reason = "Each reads graph edges into its own key type; only the mapping walk is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/graph/column_lineage_graph.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." [[dupes.allowlist]] @@ -3488,15 +3529,15 @@ paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.r reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs"] +paths = ["crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_pipeline.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs", "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs", "crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +paths = ["crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs", "crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs", "crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/metadata.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3504,7 +3545,7 @@ paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_co reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/native_resources.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3540,7 +3581,7 @@ paths = ["crates/sqlbuild-python-front/src/scopes/main/index/register_scope_inde reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs"] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_cache.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3556,7 +3597,7 @@ paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_r reason = "Each parses its own kind of SQL test file with the shared reader; only the call sequence is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs"] +paths = ["crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs"] reason = "Each crate strips a source expression to its relation name; consolidating these copies across the binding crates is a separate change." [[dupes.allowlist]] @@ -3604,7 +3645,7 @@ paths = ["crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.r reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3612,7 +3653,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs"] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/relation_targets.rs"] reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." [[dupes.allowlist]] @@ -3620,7 +3661,7 @@ paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs"] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_schema.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3628,23 +3669,23 @@ paths = ["crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-p reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python-compile/src/analysis/_helpers/sql_test_comparisons.rs", "crates/sqlbuild-python-compile/src/dag_artifact/_helpers/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-planner/src/plan_output/main/dict.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-planner/src/plan_output/main/dict.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python-compile/src/auditing/_helpers/identity.rs", "crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/configuration/builtin_catalogue.rs", "crates/sqlbuild-python-compile/src/sql_test_glue/_helpers/compiled_tests.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] @@ -3656,11 +3697,11 @@ paths = ["crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs", reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/getters.rs"] reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs"] reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." [[dupes.allowlist]] @@ -3668,7 +3709,7 @@ paths = ["crates/sqlbuild-python-planner/src/microbatches/main/register_statemen reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs"] reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." [[dupes.allowlist]] @@ -3700,7 +3741,7 @@ paths = ["crates/sqlbuild-python-planner/src/planner_resolution/_helpers/referen reason = "Each Python entry reads its own keyword arguments and calls its own resolver; only the keyword parsing is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs"] +paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs"] reason = "Each writes its own record's fields as a JSON dict; only building the dict is alike." [[dupes.allowlist]] @@ -3723,6 +3764,98 @@ reason = "One observes one configured source and the other a batch of adapter so paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_outputs/models.rs", "crates/sqlbuild-python-compile/src/input_loops/models.rs", "crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs", "crates/sqlbuild-python-project/src/graph/models.rs", "crates/sqlbuild-python-project/src/resource_views/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-compile/src/analysis/main/register_sql_test_comparisons.rs", "crates/sqlbuild-python-compile/src/dag_artifact/main/register_projection.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_api.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_selection_phases.rs", "crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_checks.rs", "crates/sqlbuild-python-compile/src/rules/main/configuration/register_catalogue.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_cache.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_run.rs", "crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs", "crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs", "crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs", "crates/sqlbuild-python-project/src/references/main/register_relations.rs", "crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs", "crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs", "crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs", "crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/main/checking/register_dsl_headers.rs", "crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs", "crates/sqlbuild-python-project/src/graph/main/project/register_views.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/analysis/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/project_assembly/main/model_syntax_checks.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs"] +reason = "Each reads its own optional text value; only the downcast-to-string shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/lineage/models.rs", "crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/lineage/main/lineage_key.rs", "crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_command/main/artifacts/write_compile_target.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/diff.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/gather_deferred_relations.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_deferred_target_config.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs"] +reason = "Each Python entry reads its own keyword arguments and calls its own implementation; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_recording.rs"] +reason = "Each records its own artifact event for the compile reuse recorder; only the skip-when-not-recording shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_pipeline_entries.rs", "crates/sqlbuild-python-compile/src/rules/main/lint/register_lint_text.rs", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-pysource/src/rule_evidence/_helpers/evidence.rs", "crates/sqlbuild-python-compile/src/compile_delta/_helpers/verify.rs", "crates/sqlbuild-rules/src/lint_text/_helpers/fixtures.rs"] +reason = "Each walks a directory tree for its own kind of file; only the recursive walk is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/assembly.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/attached_inputs.rs"] +reason = "Each builds its own result inside the compile diagnostics collector and appends what it collected; only the collect-and-chain shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python-compile/src/project_assembly/models.rs", "crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs"] +reason = "Each replaces or appends a value in its own ordered entry list; the types are separate and live in separate crates." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs", "crates/sqlbuild-python-compile/src/frontier/main/write_plan_capture.rs"] +reason = "Each writes its own capture file into the directory its own environment variable names; only building the numbered path is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/models.rs"] +reason = "Each reads its own request's fields from keyword arguments; only the field reads are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs", "crates/sqlbuild-python-compile/src/lineage/_helpers/lineage/lineage_index.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_delta/_helpers/image.rs", "crates/sqlbuild-python-compile/src/compile_delta/models.rs"] +reason = "The two conversions copy the relation tables between the stored image and the driver in opposite directions; only the field copy is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs"] +reason = "Each decides which errors its own loader treats as project input errors; the error sets differ." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 351c64faf4..8b831998b9 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -701,7 +701,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs", "crates/sqlbuild-analysis/src/contracts/constants.rs", "crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", + "crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop.rs", ) NATIVE_CODE_TEST_DIRECTORY: str = "tests" RENDER_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( @@ -709,7 +709,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "crates/sqlbuild-render/src", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops", + "crates/sqlbuild-python-compile/src/input_loops", ) NATIVE_CODE_CONSTANT_PATTERN: re.Pattern[str] = re.compile( r'const [A-Z0-9_]*_CODE: &str = "([A-Z][0-9]{3})";' diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 695d7a15a3..565dd36937 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -289,6 +289,7 @@ "crates/sqlbuild-python-front/src/", "crates/sqlbuild-python-project/src/", "crates/sqlbuild-python-planner/src/", + "crates/sqlbuild-python-compile/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index e426c4f2b7..cdbb904c12 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -7,6 +7,7 @@ "crates/sqlbuild-python-front/src", "crates/sqlbuild-python-project/src", "crates/sqlbuild-python-planner/src", + "crates/sqlbuild-python-compile/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 08510b752e..1dd5e028ee 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -1,9 +1,10 @@ # Native field reads that still fall back to Python on SQLBuild's own classes. # Generated by a strict record run; entries may only be removed. -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.prepare_analysis -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.schemas -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs .schemas +crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/analysis_session/main/analyze_models.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs BindingCatalog.prepare_analysis +crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs BindingCatalog.schemas +crates/sqlbuild-python-compile/src/analysis_session/_helpers/model_analyses.rs .schemas crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs InferredNullability.value crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.expression_shapes crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.native @@ -93,20 +94,22 @@ crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs InvocationOutc crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs NativeProgressProjector.consume crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs TerminalEventIndex.consume crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs WarehouseGroup.value -crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs NativeScopeIndex.grant_relationships -crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs DbtCompileReferenceResolver.validate_model_names -crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs PreparedCompileArtifacts.max_prepared_models -crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs PreparedCompileArtifacts.min_prepared_models -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.exit_code -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.json_output -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.native -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.outcome -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.project_dir -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs CompileReuseAttempt.selected_target -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.check_ms -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.entry_path -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.native -crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.outcome +crates/sqlbuild-python-compile/src/compile_attachment/_helpers/declaration_scope.rs NativeScopeIndex.grant_relationships +crates/sqlbuild-python-compile/src/compile_attachment/_helpers/model_inputs.rs DbtCompileReferenceResolver.validate_model_names +crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs PreparedCompileArtifacts.max_prepared_models +crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs PreparedCompileArtifacts.min_prepared_models +crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs CompileReuseAttempt.exit_code +crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs CompileReuseAttempt.json_output +crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs CompileReuseAttempt.native +crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs CompileReuseAttempt.outcome +crates/sqlbuild-python-compile/src/compile_command/main/invocation/run_compile.rs CompileReuseAttempt.outcome +crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs CompileReuseAttempt.project_dir +crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/reuse_attempt.rs CompileReuseAttempt.selected_target +crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs CompileReuseAttempt.check_ms +crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs CompileReuseAttempt.entry_path +crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs CompileReuseAttempt.native +crates/sqlbuild-python-compile/src/compile_command/main/invocation/compile_reuse_timings.rs CompileReuseAttempt.outcome +crates/sqlbuild-python-compile/src/compile_command/main/invocation/write_reusable_compile.rs CompileReuseAttempt.outcome crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dataclass_fields__ crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dict__ crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs PythonHookEntry.__dataclass_fields__ @@ -123,35 +126,36 @@ crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEn crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEntry.__dict__ crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dataclass_fields__ crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs PreparedCompileArtifacts.start -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.schemas -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs NativeProjectResources.model_deps -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs NativeProjectResources.model_syntax_valid -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs NativeModelFacts.deps -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs NativeModelFacts.dynamic_contract -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs NativeModelFacts.syntax_valid -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectFacts.models -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectFacts.resources -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectResources.audit_scope_deps -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectResources.failure -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectResources.function_deps -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectResources.seed_namespaces -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeProjectResources.source_namespaces -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeResourceFailure.error -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeResourceFailure.index -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs NativeResourceFailure.slot -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs CompileReuseAttempt.started -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs PreparedCompileArtifacts.remove_abandoned_staging -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs CompileInputError.code -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs CompileInputError.help -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs PreparedCompileArtifacts.publish -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs PythonNodeGraph.selected_sql_refs -crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs PlannerScopeResolution.stale_warning_scope +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs PreparedCompileArtifacts.start +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_analysis.rs BindingCatalog.schemas +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_facts.rs NativeProjectResources.model_deps +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_facts.rs NativeProjectResources.model_syntax_valid +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs NativeModelFacts.deps +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs NativeModelFacts.dynamic_contract +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_models.rs NativeModelFacts.syntax_valid +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectFacts.models +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectFacts.resources +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectResources.audit_scope_deps +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectResources.failure +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectResources.function_deps +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectResources.seed_namespaces +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeProjectResources.source_namespaces +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeResourceFailure.error +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeResourceFailure.index +crates/sqlbuild-python-compile/src/compile_driver/_helpers/assembly/assembly_project.rs NativeResourceFailure.slot +crates/sqlbuild-python-compile/src/compile_driver/main/stages/run_compile_command.rs CompileReuseAttempt.started +crates/sqlbuild-python-compile/src/compile_driver/main/stages/unrecorded_compile.rs CompileReuseAttempt.started +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/command.rs PreparedCompileArtifacts.remove_abandoned_staging +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs CompileInputError.code +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs CompileInputError.help +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/phases.rs PreparedCompileArtifacts.publish +crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_read_source_names_for.rs PythonNodeGraph.selected_sql_refs +crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/selection_phases.rs PlannerScopeResolution.stale_warning_scope crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs CollectionRendering.value crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs CollectionRendering.value crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs ConnectionProgressReporter.on_connection_complete -crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs SqlValueKind.value +crates/sqlbuild-python-compile/src/dag_artifact/_helpers/rows.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs DbtInitProgressReporter.complete crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs DbtInitProgressReporter.start crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs DbtInteropCommand.value @@ -235,15 +239,15 @@ crates/sqlbuild-python-base/src/frontier/models.rs TableType.name crates/sqlbuild-python-base/src/frontier/models.rs TableTypeSource.name crates/sqlbuild-python-base/src/frontier/models.rs UsageKind.name crates/sqlbuild-python-base/src/frontier/models.rs VisibilityReason.name -crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs CompilerStage.value -crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs ResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs MacroArgumentError.offset -crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_named.rs ScopeKind.value +crates/sqlbuild-python-compile/src/frontier/_helpers/stage_capture.rs CompilerStage.value +crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs ResourceKind.value +crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/loop_classes.rs DeclarationKind.value +crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_macros.rs MacroArgumentError.offset +crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_named.rs ScopeKind.value crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs OldNameViewHistory.status crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs ColumnLineageMode.value crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs PlannerInputError.code -crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/lineage/_helpers/graph/project_lineage.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs LoadCallbackRelay.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs LoadCallbackRelay.on_connection_error @@ -254,9 +258,9 @@ crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs TransientLine crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs CollectionRendering.value crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs DeclarationKind.value crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs SqlValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs DeclarationKind.value -crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs ScopeKind.value -crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs SourceWriteStrategy.value +crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs DeclarationKind.value +crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs ScopeKind.value +crates/sqlbuild-python-compile/src/manifest/_helpers/rows.rs SourceWriteStrategy.value crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_start @@ -399,7 +403,7 @@ crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs Re crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs ResourceKind.value crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs PlanReason.value crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs SqlValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs RuleSubject.value +crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs RuleSubject.value crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs ScenarioProgressRelay.on_scenario_complete crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs ScenarioProgressRelay.on_scenario_start crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs ScenarioArtifactKind.value @@ -415,7 +419,7 @@ crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs ScopeDi crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs CollectionRendering.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs ScopeKind.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs SqlValueKind.value -crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs NativeScopeIndex.grant_relationships +crates/sqlbuild-python-compile/src/scopes/_helpers/scope_loading.rs NativeScopeIndex.grant_relationships crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs DeclarationKind.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs GrantKind.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs OwnershipRootKind.value @@ -431,8 +435,8 @@ crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgr crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedCompletionRelay.on_seed_complete crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnectionRelay.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnectionRelay.on_connection_error -crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/inputs.rs BindingCatalog.native +crates/sqlbuild-python-compile/src/semantic_checks/_helpers/session/stage.rs BindingCatalog.native crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs BindingCatalog.inferred_schema crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs BindingPositions.position crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs PlanningProgressReporter.on_progress