diff --git a/Cargo.lock b/Cargo.lock index b76db7f99d..79ca179a14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4598,6 +4598,7 @@ dependencies = [ "sqlquack-parse", "sqlquack-syntax", "stacker", + "stats_alloc", "tempfile", "toml", ] @@ -4923,6 +4924,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stats_alloc" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e04424e733e69714ca1bbb9204c1a57f09f5493439520f9f68c132ad25eec" + [[package]] name = "stringprep" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 6dadb24d1f..2861327586 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,6 +206,8 @@ sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git" sqlquack-syntax-sqlite = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } stacker = "0.1" +# Counts heap allocations in growth tests, which measure work instead of time. +stats_alloc = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" tempfile = "3" diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 1c5a31e933..5503cadb5d 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -35,6 +35,7 @@ toml.workspace = true [dev-dependencies] tempfile.workspace = true +stats_alloc.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs index 24ac1db8cf..a5df4b6e76 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs @@ -9,7 +9,7 @@ use crate::python_rules::tests::helpers::{ unit_scaling_project, }; use crate::python_rules::tests::test_types::{ - PurityScalingTestCase, PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase, + PurityScalingTestCase, PythonScalingTestCase, UnitScalingTestCase, }; #[test] @@ -43,46 +43,6 @@ fn given_many_modules_and_dense_findings_when_evaluating_python_rules_then_finis } } -#[test] -fn given_four_times_the_modules_when_finding_unreachable_definitions_then_time_grows_near_linearly() -{ - install_test_identity(); - let _serial = serial_timing(); - let test_cases = [PythonGrowthTestCase { - description: "unreachable definitions walking every import, 500 against 2000 modules", - units: 500, - noisy_files: 5, - functions: 150, - config: json!({ "select": ["SQBRPYTHON", "SQBRPYTHON306"] }), - growth: 4, - pairs: 5, - expected_unreachable_definitions: 2001, - expected_max_ratio: 6.5, - }]; - for test_case in test_cases { - let small = scaling_project(test_case.units, test_case.noisy_files, test_case.functions); - let large = scaling_project( - test_case.growth * test_case.units, - test_case.growth * test_case.noisy_files, - test_case.functions, - ); - let (codes, _) = timed_python_codes(&large, test_case.config.clone()).expect("rules run"); - let unreachable: usize = codes.iter().filter(|code| *code == "SQBRPYTHON306").count(); - assert_eq!( - unreachable, test_case.expected_unreachable_definitions, - "{}", - test_case.description - ); - let ratio: f64 = median_growth(&small, &large, &test_case.config, test_case.pairs); - assert!( - ratio < test_case.expected_max_ratio, - "{}: growing the project {}x multiplied the time by {ratio:.2}", - test_case.description, - test_case.growth - ); - } -} - #[test] fn given_many_linked_node_units_when_evaluating_unit_rules_then_finishes_within_budget_and_grows_near_linearly() { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs index f600b2fd77..0f8ea22335 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs @@ -102,18 +102,6 @@ pub(super) struct UnitScalingTestCase { pub(super) expected_max_ratio: f64, } -pub(super) struct PythonGrowthTestCase { - pub(super) description: &'static str, - pub(super) units: usize, - pub(super) noisy_files: usize, - pub(super) functions: usize, - pub(super) config: serde_json::Value, - pub(super) growth: usize, - pub(super) pairs: usize, - pub(super) expected_unreachable_definitions: usize, - pub(super) expected_max_ratio: f64, -} - pub(super) struct RelativeQueryTestCase { pub(super) description: &'static str, pub(super) project_dir: &'static str, diff --git a/crates/sqlbuild-rules/tests/python_rules.rs b/crates/sqlbuild-rules/tests/python_rules.rs new file mode 100644 index 0000000000..f847e94a2b --- /dev/null +++ b/crates/sqlbuild-rules/tests/python_rules.rs @@ -0,0 +1,8 @@ +//! Python Rules growth guards counting allocations, alone in a binary so no test adds to them. + +#[path = "python_rules/helpers.rs"] +mod helpers; +#[path = "python_rules/test_reachability_work.rs"] +mod test_reachability_work; +#[path = "python_rules/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/tests/python_rules/helpers.rs b/crates/sqlbuild-rules/tests/python_rules/helpers.rs new file mode 100644 index 0000000000..614cf74549 --- /dev/null +++ b/crates/sqlbuild-rules/tests/python_rules/helpers.rs @@ -0,0 +1,65 @@ +use sqlbuild_rules::build_identity::main::install_build_identity::install_build_identity; +use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; +use sqlbuild_rules::models::{EvaluateRequest, RowsRequest}; + +/// Counts the heap allocations of this test binary, which unlike time do not depend on the machine. +#[global_allocator] +static ALLOCATOR: &stats_alloc::StatsAlloc = &stats_alloc::INSTRUMENTED_SYSTEM; + +/// A shared helper, `modules` modules that import it, and one hook file of 150 functions per 100. +pub(crate) fn importing_modules(modules: usize) -> Vec<(String, String)> { + let helper = ( + "python/shared/text.py".to_owned(), + "\"\"\"Shared text.\"\"\"\n\n\ndef trimmed(value: str) -> str:\n \"\"\"Return the value.\"\"\"\n return value\n".to_owned(), + ); + std::iter::once(helper) + .chain((0..modules).map(|module| { + ( + format!("python/area_{}/module_{module}/entry.py", module % 40), + format!( + "\"\"\"Module {module}.\"\"\"\n\nfrom python.shared.text import trimmed\n\n\ndef handle_{module}(value: str) -> str:\n \"\"\"Handle module {module}.\"\"\"\n return trimmed(value)\n" + ), + ) + })) + .chain((0..modules / 100).map(|hook| { + let body: String = (0..150) + .map(|function| format!("def notify_{function}(ctx, reason):\n print(ctx, reason)\n\n\n")) + .collect(); + (format!("hooks/python/notify_{hook}.py"), body) + })) + .collect() +} + +/// The codes found over `files` and the heap allocations and reallocations the evaluation made. +pub(crate) fn counted_codes( + files: &[(String, String)], + config: &serde_json::Value, +) -> (Vec, usize) { + install_build_identity("0123456789abcdef").expect("the test identity installs"); + let directory = tempfile::tempdir().expect("project directory"); + for (path, contents) in files { + let file = directory.path().join(path); + std::fs::create_dir_all(file.parent().expect("parent")).expect("module directory"); + std::fs::write(&file, contents).expect("module file"); + } + let mut config = config.clone(); + config["cache"] = serde_json::json!({ "enabled": false }); + let request = RowsRequest { + request: EvaluateRequest { + project_dir: directory.path().to_string_lossy().into_owned(), + config: serde_json::from_value(config).expect("rules config"), + ..EvaluateRequest::default() + }, + memo: None, + retain: false, + }; + let region = stats_alloc::Region::new(ALLOCATOR); + let evaluation = evaluate_rows(request).expect("rules run"); + let change = region.change(); + let codes = evaluation + .faults + .into_iter() + .map(|fault| fault.code) + .collect(); + (codes, change.allocations + change.reallocations) +} diff --git a/crates/sqlbuild-rules/tests/python_rules/test_reachability_work.rs b/crates/sqlbuild-rules/tests/python_rules/test_reachability_work.rs new file mode 100644 index 0000000000..75119c1f79 --- /dev/null +++ b/crates/sqlbuild-rules/tests/python_rules/test_reachability_work.rs @@ -0,0 +1,38 @@ +use serde_json::json; + +use crate::helpers::{counted_codes, importing_modules}; +use crate::test_types::ReachabilityWorkTestCase; + +#[test] +fn given_four_times_the_modules_when_finding_unreachable_definitions_then_work_grows_near_linearly() +{ + let test_cases = [ReachabilityWorkTestCase { + description: "unreachable definitions walking every import, 500 against 2000 modules", + modules: 500, + growth: 4, + config: json!({ "select": ["SQBRPYTHON", "SQBRPYTHON306"] }), + expected_unreachable_definitions: 2001, + expected_max_ratio: 5.0, + }]; + for test_case in test_cases { + let small = importing_modules(test_case.modules); + let large = importing_modules(test_case.growth * test_case.modules); + + let (_, small_work) = counted_codes(&small, &test_case.config); + let (codes, large_work) = counted_codes(&large, &test_case.config); + + let unreachable: usize = codes.iter().filter(|code| *code == "SQBRPYTHON306").count(); + assert_eq!( + unreachable, test_case.expected_unreachable_definitions, + "{}", + test_case.description + ); + let ratio: f64 = large_work as f64 / small_work as f64; + assert!( + ratio < test_case.expected_max_ratio, + "{}: growing the project {}x multiplied the allocations by {ratio:.2} ({small_work} to {large_work})", + test_case.description, + test_case.growth + ); + } +} diff --git a/crates/sqlbuild-rules/tests/python_rules/test_types.rs b/crates/sqlbuild-rules/tests/python_rules/test_types.rs new file mode 100644 index 0000000000..d4f0154f39 --- /dev/null +++ b/crates/sqlbuild-rules/tests/python_rules/test_types.rs @@ -0,0 +1,8 @@ +pub(crate) struct ReachabilityWorkTestCase { + pub(crate) description: &'static str, + pub(crate) modules: usize, + pub(crate) growth: usize, + pub(crate) config: serde_json::Value, + pub(crate) expected_unreachable_definitions: usize, + pub(crate) expected_max_ratio: f64, +} diff --git a/fensu.toml b/fensu.toml index f6d3b5122e..b87c6ff2a2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1177,10 +1177,8 @@ max_positional_args = 1 [[targets.python.threshold_overrides]] paths = ["src/sqlbuild/__init__.py"] -# Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py). -# Raised from 89,626 for helper-class anchors in Python identity (object_source.py, identity.py). -thresholds = { max_crate_code_lines = 89643 } -reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out." +thresholds = { max_crate_code_lines = 100000 } +reason = "The runtime package is above Fensu's default code-line limit. Kevin set 100,000 on 16 Nov to avoid per-PR raises while the crate split and UI work land; reducing it is parked (backlog)." [targets.python.rule_options.XSB074] pending_budgets = [