Skip to content

Commit bc20006

Browse files
authored
test: count allocations in the reachability scaling guard and fix the runtime package cap (#1521)
* chore: set the runtime package code-line cap to 100,000 A fixed cap avoids raising it in every pull request while the crate split and UI work land. Reducing it again is in the backlog. * test: guard unreachable-definition scaling with an allocation count The guard compared wall time for 500 and 2,000 modules, which flakes on shared runners. It now counts heap allocations in its own test binary, so no other test adds to the count. Today the count grows 3.95x for 4x the modules; copying the dead-code context into every file call grows it 7.58x, and the quadratic parent lookup in fensu 0.31.0 grows it 8.14x. Both fail the bound of 5.
1 parent 50bcf76 commit bc20006

10 files changed

Lines changed: 132 additions & 57 deletions

File tree

‎Cargo.lock‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,8 @@ sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git"
209209
sqlquack-syntax-sqlite = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" }
210210
sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" }
211211
stacker = "0.1"
212+
# Counts heap allocations in growth tests, which measure work instead of time.
213+
stats_alloc = "0.1"
212214
syn = { version = "2", features = ["full", "visit", "extra-traits"] }
213215
subtle = "2"
214216
tempfile = "3"

‎crates/sqlbuild-rules/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ toml.workspace = true
3535

3636
[dev-dependencies]
3737
tempfile.workspace = true
38+
stats_alloc.workspace = true
3839

3940
[lints]
4041
workspace = true

‎crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::python_rules::tests::helpers::{
99
unit_scaling_project,
1010
};
1111
use crate::python_rules::tests::test_types::{
12-
PurityScalingTestCase, PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase,
12+
PurityScalingTestCase, PythonScalingTestCase, UnitScalingTestCase,
1313
};
1414

1515
#[test]
@@ -43,46 +43,6 @@ fn given_many_modules_and_dense_findings_when_evaluating_python_rules_then_finis
4343
}
4444
}
4545

46-
#[test]
47-
fn given_four_times_the_modules_when_finding_unreachable_definitions_then_time_grows_near_linearly()
48-
{
49-
install_test_identity();
50-
let _serial = serial_timing();
51-
let test_cases = [PythonGrowthTestCase {
52-
description: "unreachable definitions walking every import, 500 against 2000 modules",
53-
units: 500,
54-
noisy_files: 5,
55-
functions: 150,
56-
config: json!({ "select": ["SQBRPYTHON", "SQBRPYTHON306"] }),
57-
growth: 4,
58-
pairs: 5,
59-
expected_unreachable_definitions: 2001,
60-
expected_max_ratio: 6.5,
61-
}];
62-
for test_case in test_cases {
63-
let small = scaling_project(test_case.units, test_case.noisy_files, test_case.functions);
64-
let large = scaling_project(
65-
test_case.growth * test_case.units,
66-
test_case.growth * test_case.noisy_files,
67-
test_case.functions,
68-
);
69-
let (codes, _) = timed_python_codes(&large, test_case.config.clone()).expect("rules run");
70-
let unreachable: usize = codes.iter().filter(|code| *code == "SQBRPYTHON306").count();
71-
assert_eq!(
72-
unreachable, test_case.expected_unreachable_definitions,
73-
"{}",
74-
test_case.description
75-
);
76-
let ratio: f64 = median_growth(&small, &large, &test_case.config, test_case.pairs);
77-
assert!(
78-
ratio < test_case.expected_max_ratio,
79-
"{}: growing the project {}x multiplied the time by {ratio:.2}",
80-
test_case.description,
81-
test_case.growth
82-
);
83-
}
84-
}
85-
8646
#[test]
8747
fn given_many_linked_node_units_when_evaluating_unit_rules_then_finishes_within_budget_and_grows_near_linearly()
8848
{

‎crates/sqlbuild-rules/src/python_rules/tests/test_types.rs‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -102,18 +102,6 @@ pub(super) struct UnitScalingTestCase {
102102
pub(super) expected_max_ratio: f64,
103103
}
104104

105-
pub(super) struct PythonGrowthTestCase {
106-
pub(super) description: &'static str,
107-
pub(super) units: usize,
108-
pub(super) noisy_files: usize,
109-
pub(super) functions: usize,
110-
pub(super) config: serde_json::Value,
111-
pub(super) growth: usize,
112-
pub(super) pairs: usize,
113-
pub(super) expected_unreachable_definitions: usize,
114-
pub(super) expected_max_ratio: f64,
115-
}
116-
117105
pub(super) struct RelativeQueryTestCase {
118106
pub(super) description: &'static str,
119107
pub(super) project_dir: &'static str,
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
//! Python Rules growth guards counting allocations, alone in a binary so no test adds to them.
2+
3+
#[path = "python_rules/helpers.rs"]
4+
mod helpers;
5+
#[path = "python_rules/test_reachability_work.rs"]
6+
mod test_reachability_work;
7+
#[path = "python_rules/test_types.rs"]
8+
mod test_types;
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
use sqlbuild_rules::build_identity::main::install_build_identity::install_build_identity;
2+
use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows;
3+
use sqlbuild_rules::models::{EvaluateRequest, RowsRequest};
4+
5+
/// Counts the heap allocations of this test binary, which unlike time do not depend on the machine.
6+
#[global_allocator]
7+
static ALLOCATOR: &stats_alloc::StatsAlloc<std::alloc::System> = &stats_alloc::INSTRUMENTED_SYSTEM;
8+
9+
/// A shared helper, `modules` modules that import it, and one hook file of 150 functions per 100.
10+
pub(crate) fn importing_modules(modules: usize) -> Vec<(String, String)> {
11+
let helper = (
12+
"python/shared/text.py".to_owned(),
13+
"\"\"\"Shared text.\"\"\"\n\n\ndef trimmed(value: str) -> str:\n \"\"\"Return the value.\"\"\"\n return value\n".to_owned(),
14+
);
15+
std::iter::once(helper)
16+
.chain((0..modules).map(|module| {
17+
(
18+
format!("python/area_{}/module_{module}/entry.py", module % 40),
19+
format!(
20+
"\"\"\"Module {module}.\"\"\"\n\nfrom python.shared.text import trimmed\n\n\ndef handle_{module}(value: str) -> str:\n \"\"\"Handle module {module}.\"\"\"\n return trimmed(value)\n"
21+
),
22+
)
23+
}))
24+
.chain((0..modules / 100).map(|hook| {
25+
let body: String = (0..150)
26+
.map(|function| format!("def notify_{function}(ctx, reason):\n print(ctx, reason)\n\n\n"))
27+
.collect();
28+
(format!("hooks/python/notify_{hook}.py"), body)
29+
}))
30+
.collect()
31+
}
32+
33+
/// The codes found over `files` and the heap allocations and reallocations the evaluation made.
34+
pub(crate) fn counted_codes(
35+
files: &[(String, String)],
36+
config: &serde_json::Value,
37+
) -> (Vec<String>, usize) {
38+
install_build_identity("0123456789abcdef").expect("the test identity installs");
39+
let directory = tempfile::tempdir().expect("project directory");
40+
for (path, contents) in files {
41+
let file = directory.path().join(path);
42+
std::fs::create_dir_all(file.parent().expect("parent")).expect("module directory");
43+
std::fs::write(&file, contents).expect("module file");
44+
}
45+
let mut config = config.clone();
46+
config["cache"] = serde_json::json!({ "enabled": false });
47+
let request = RowsRequest {
48+
request: EvaluateRequest {
49+
project_dir: directory.path().to_string_lossy().into_owned(),
50+
config: serde_json::from_value(config).expect("rules config"),
51+
..EvaluateRequest::default()
52+
},
53+
memo: None,
54+
retain: false,
55+
};
56+
let region = stats_alloc::Region::new(ALLOCATOR);
57+
let evaluation = evaluate_rows(request).expect("rules run");
58+
let change = region.change();
59+
let codes = evaluation
60+
.faults
61+
.into_iter()
62+
.map(|fault| fault.code)
63+
.collect();
64+
(codes, change.allocations + change.reallocations)
65+
}
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
use serde_json::json;
2+
3+
use crate::helpers::{counted_codes, importing_modules};
4+
use crate::test_types::ReachabilityWorkTestCase;
5+
6+
#[test]
7+
fn given_four_times_the_modules_when_finding_unreachable_definitions_then_work_grows_near_linearly()
8+
{
9+
let test_cases = [ReachabilityWorkTestCase {
10+
description: "unreachable definitions walking every import, 500 against 2000 modules",
11+
modules: 500,
12+
growth: 4,
13+
config: json!({ "select": ["SQBRPYTHON", "SQBRPYTHON306"] }),
14+
expected_unreachable_definitions: 2001,
15+
expected_max_ratio: 5.0,
16+
}];
17+
for test_case in test_cases {
18+
let small = importing_modules(test_case.modules);
19+
let large = importing_modules(test_case.growth * test_case.modules);
20+
21+
let (_, small_work) = counted_codes(&small, &test_case.config);
22+
let (codes, large_work) = counted_codes(&large, &test_case.config);
23+
24+
let unreachable: usize = codes.iter().filter(|code| *code == "SQBRPYTHON306").count();
25+
assert_eq!(
26+
unreachable, test_case.expected_unreachable_definitions,
27+
"{}",
28+
test_case.description
29+
);
30+
let ratio: f64 = large_work as f64 / small_work as f64;
31+
assert!(
32+
ratio < test_case.expected_max_ratio,
33+
"{}: growing the project {}x multiplied the allocations by {ratio:.2} ({small_work} to {large_work})",
34+
test_case.description,
35+
test_case.growth
36+
);
37+
}
38+
}
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
pub(crate) struct ReachabilityWorkTestCase {
2+
pub(crate) description: &'static str,
3+
pub(crate) modules: usize,
4+
pub(crate) growth: usize,
5+
pub(crate) config: serde_json::Value,
6+
pub(crate) expected_unreachable_definitions: usize,
7+
pub(crate) expected_max_ratio: f64,
8+
}

‎fensu.toml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1177,10 +1177,8 @@ max_positional_args = 1
11771177

11781178
[[targets.python.threshold_overrides]]
11791179
paths = ["src/sqlbuild/__init__.py"]
1180-
# Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py).
1181-
# Raised from 89,626 for helper-class anchors in Python identity (object_source.py, identity.py).
1182-
thresholds = { max_crate_code_lines = 89643 }
1183-
reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out."
1180+
thresholds = { max_crate_code_lines = 100000 }
1181+
reason = "The runtime package is above Fensu's default code-line limit. Kevin set 100,000 on 16 Nov to avoid per-PR raises while the crate split and UI work land; reducing it is parked (backlog)."
11841182

11851183
[targets.python.rule_options.XSB074]
11861184
pending_budgets = [

0 commit comments

Comments
 (0)