Skip to content

Commit f141d96

Browse files
committed
Merge remote-tracking branch 'origin/main' into refactor/python-base-infra
# Conflicts: # fensu.toml
2 parents de6301b + 9ca1f7a commit f141d96

15 files changed

Lines changed: 242 additions & 73 deletions

File tree

‎crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22
33
use std::collections::HashSet;
44

5+
use pyo3::IntoPyObjectExt;
56
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
67
use pyo3::pyfunction;
78
use pyo3::types::{PyBool, PyDict, PyDictMethods};
9+
use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE;
810
use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON;
911
use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name;
1012
use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name;
@@ -123,17 +125,23 @@ fn candidate_records<'py>(candidates: &[Candidate<'py>]) -> PyResult<Vec<Bound<'
123125
candidates.iter().map(Candidate::to_python).collect()
124126
}
125127

128+
/// The active target's database or schema as a location: `preserve` names none, since each model keeps its own.
129+
fn target_location(project: &Bound<'_, PyAny>, field: &str) -> PyResult<Option<String>> {
130+
Ok(native_field(project, field)?
131+
.extract::<Option<String>>()?
132+
.filter(|value| value != PRESERVE_TARGET_VALUE))
133+
}
134+
126135
/// Review schemas whose markers the active target schema holds and that outlived any review run.
127136
fn review_schema_candidates<'py>(
128137
project: &Bound<'py, PyAny>,
129138
warehouse: &Warehouse<'py>,
130139
now: &Bound<'py, PyAny>,
131140
) -> PyResult<Vec<Bound<'py, PyAny>>> {
132141
let py: Python<'py> = warehouse.py();
133-
let schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?;
134-
if schema.is_none() {
142+
let Some(schema) = target_location(project, "effective_target_schema")? else {
135143
return Ok(Vec::new());
136-
}
144+
};
137145
let max_age: Bound<'py, PyAny> = days(py, LEFTOVER_REVIEW_SCHEMA_AGE_DAYS)?;
138146
let leftovers: Bound<'py, PyAny> = call_with(
139147
&host_entry(py, ExecutorHost::LeftoverReviewSchemas)?,
@@ -142,9 +150,9 @@ fn review_schema_candidates<'py>(
142150
("connection", warehouse.connection.clone()),
143151
(
144152
"database",
145-
native_field(project, "effective_target_database")?,
153+
target_location(project, "effective_target_database")?.into_bound_py_any(py)?,
146154
),
147-
("schema", schema),
155+
("schema", schema.into_bound_py_any(py)?),
148156
("now", now.clone()),
149157
("max_age", max_age),
150158
],
@@ -208,11 +216,9 @@ pub(crate) fn build_plan<'py>(
208216
push_artifact_schema(database.clone(), schema.clone());
209217
}
210218
}
211-
if let Some(schema) =
212-
native_field(project, "effective_target_schema")?.extract::<Option<String>>()?
213-
{
219+
if let Some(schema) = target_location(project, "effective_target_schema")? {
214220
push_artifact_schema(
215-
native_field(project, "effective_target_database")?.extract()?,
221+
target_location(project, "effective_target_database")?,
216222
schema,
217223
);
218224
}

‎crates/sqlbuild-recall/src/writer/tests/helpers.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use crate::store::main::open_sqlite_store::open_sqlite_store;
1212
use crate::store::main::query_store::query_store;
1313
use crate::store::main::read_output_transcript::read_output_transcript;
1414
use crate::store::models::OpenMode;
15+
use crate::writer::_helpers::delivery::locked;
1516
use crate::writer::_helpers::queue::WriterState;
1617
use crate::writer::main::start_recall_session::start_recall_session;
1718
use crate::writer::models::{
@@ -175,8 +176,8 @@ pub(crate) fn unwritable_session() -> (bool, bool) {
175176
)
176177
}
177178

178-
/// Seconds `close` took and whether delivery degraded, while another connection holds the write lock.
179-
pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) {
179+
/// While another connection holds the write lock: whether closing set its deadline exactly the budget after it began, the degraded reason, and the seconds `close` took.
180+
pub(crate) fn close_under_held_lock(shutdown: Duration) -> (bool, Option<String>, f64) {
180181
let directory = tempfile::tempdir().expect("a temporary directory can be created");
181182
let path = directory.path().join("logs/recall.sqlite3");
182183
let warm = start_recall_session(config(path.clone(), "warm"));
@@ -191,10 +192,19 @@ pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) {
191192
});
192193
session.append_output(OutputStream::Stdout, b"waiting\n");
193194
let started = std::time::Instant::now();
195+
session.begin_close();
196+
let began = std::time::Instant::now();
197+
let deadline = locked(&session.shared.state)
198+
.deadline
199+
.expect("closing sets a deadline");
194200
let summary = session.close();
195201
let elapsed = started.elapsed().as_secs_f64();
196202
drop(holder);
197-
(elapsed, summary.degraded_reason.is_some())
203+
(
204+
started + shutdown <= deadline && deadline <= began + shutdown,
205+
summary.degraded_reason,
206+
elapsed,
207+
)
198208
}
199209

200210
/// The notices and resulting store state after starting on a file that is not a database.

‎crates/sqlbuild-recall/src/writer/tests/test_session.rs‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ use crate::writer::tests::test_types::{
2424
};
2525

2626
const CLOSE_SLACK_SECONDS: f64 = 0.3;
27+
/// Far beyond any budget a test sets; only a close that waits for the store itself reaches it.
28+
const CLOSE_HANG_GUARD_SECONDS: f64 = 60.0;
2729

2830
#[test]
2931
fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() {
@@ -87,25 +89,28 @@ fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_clos
8789
#[test]
8890
fn given_a_held_write_lock_when_closed_then_the_shutdown_budget_caps_the_delay() {
8991
let test_cases = [HeldLockTestCase {
90-
description: "a locked store costs at most the shutdown budget at exit",
92+
description: "a locked store gives up at the deadline the budget sets when closing begins",
9193
shutdown_ms: 700,
92-
expected_degraded: true,
94+
expected_degraded_reason: Some(
95+
"Recall delivery did not finish within the shutdown timeout",
96+
),
9397
}];
9498

9599
for test_case in test_cases {
96-
let (elapsed, degraded) =
100+
let (deadline_is_the_budget, degraded_reason, elapsed) =
97101
close_under_held_lock(Duration::from_millis(test_case.shutdown_ms));
98-
let budget: f64 = Duration::from_millis(test_case.shutdown_ms).as_secs_f64();
99-
assert!(
100-
elapsed <= budget + CLOSE_SLACK_SECONDS,
101-
"{}: {elapsed:.3}s",
102-
test_case.description
103-
);
102+
assert!(deadline_is_the_budget, "{}", test_case.description);
104103
assert_eq!(
105-
degraded, test_case.expected_degraded,
104+
degraded_reason.as_deref(),
105+
test_case.expected_degraded_reason,
106106
"{}",
107107
test_case.description
108108
);
109+
assert!(
110+
elapsed < CLOSE_HANG_GUARD_SECONDS,
111+
"{}: close hung for {elapsed:.3}s",
112+
test_case.description
113+
);
109114
}
110115
}
111116

‎crates/sqlbuild-recall/src/writer/tests/test_types.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ pub(crate) struct UnwritableTestCase {
5151
pub(crate) struct HeldLockTestCase {
5252
pub(crate) description: &'static str,
5353
pub(crate) shutdown_ms: u64,
54-
pub(crate) expected_degraded: bool,
54+
pub(crate) expected_degraded_reason: Option<&'static str>,
5555
}
5656

5757
pub(crate) struct CorruptStoreTestCase {

‎fensu.toml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1178,7 +1178,8 @@ max_positional_args = 1
11781178
[[targets.python.threshold_overrides]]
11791179
paths = ["src/sqlbuild/__init__.py"]
11801180
# Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py).
1181-
thresholds = { max_crate_code_lines = 89626 }
1181+
# Raised from 89,626 for helper-class anchors in Python identity (object_source.py, identity.py).
1182+
thresholds = { max_crate_code_lines = 89643 }
11821183
reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out."
11831184

11841185
[targets.python.rule_options.XSB074]
@@ -1679,8 +1680,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X
16791680

16801681
[[targets.rust.threshold_overrides]]
16811682
paths = ["crates/sqlbuild-python/Cargo.toml"]
1682-
thresholds = { max_crate_code_lines = 197887 }
1683-
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."
1683+
thresholds = { max_crate_code_lines = 197891 }
1684+
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Set after #1514 moved the shared runtime into sqlbuild-python-base, plus 4 lines for the janitor preserve check from #1512."
16841685

16851686
[[targets.rust.threshold_overrides]]
16861687
paths = ["crates/sqlbuild-python/src/bindings/_helpers"]

‎src/sqlbuild/compiler/python_nodes/_helpers/identity.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from typing import Any, cast
1414

1515
from sqlbuild._native import PythonIdentityDependency, PythonNodeIdentity
16-
from sqlbuild.compiler.python_nodes._helpers.object_source import hash_text
16+
from sqlbuild.compiler.python_nodes._helpers.object_source import class_anchor, hash_text
1717
from sqlbuild.compiler.python_nodes.classes.identity_pass import IdentityPass
1818

1919
_IGNORED_PATH_PARTS: frozenset[str] = frozenset(
@@ -182,8 +182,9 @@ def _visit_object(
182182
collected_dependencies: dict[tuple[str, str], PythonIdentityDependency] = dependencies
183183
collected_dependencies[key] = dependency
184184

185+
unwrapped: object = inspect.unwrap(cast(Any, obj))
185186
globals_map: Mapping[str, object] | None = getattr(
186-
inspect.unwrap(cast(Any, obj)), "__globals__", None
187+
class_anchor(unwrapped) or unwrapped, "__globals__", None
187188
)
188189
if globals_map is None:
189190
module: ModuleType | None = memo.module(obj)

‎src/sqlbuild/compiler/python_nodes/_helpers/object_source.py‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,23 +6,19 @@
66
import hashlib
77
import inspect
88
import linecache
9-
import sys
109
import textwrap
1110
from collections.abc import Callable
1211
from pathlib import Path
13-
from types import ModuleType
12+
from types import FunctionType, ModuleType
1413
from typing import Any, cast
1514

16-
_CLASS_FIRSTLINENO_VERSION: tuple[int, int] = (3, 13)
17-
_FINDSOURCE_PARSES_FILE_PER_CLASS: bool = sys.version_info < _CLASS_FIRSTLINENO_VERSION
18-
1915

2016
def normalized_source(*, obj: object, class_starts: Callable[..., dict[str, int]]) -> str:
21-
"""`inspect.getsource(obj)` dedented and stripped, finding classes through `class_starts`."""
17+
"""`inspect.getsource(obj)` dedented and stripped; classes read from their own file."""
2218

2319
try:
2420
target: object = inspect.unwrap(cast(Any, obj))
25-
if not (_FINDSOURCE_PARSES_FILE_PER_CLASS and inspect.isclass(target)):
21+
if not inspect.isclass(target):
2622
return textwrap.dedent(inspect.getsource(cast(Any, obj))).strip()
2723
lines, start = _class_location(cast(type, target), class_starts=class_starts)
2824
return textwrap.dedent("".join(inspect.getblock(lines[start:]))).strip()
@@ -34,7 +30,7 @@ def source_file(obj: object) -> Path | None:
3430
"""The file `inspect` reads `obj`'s source from, or `None` when it has none."""
3531

3632
try:
37-
raw_path: str | None = inspect.getsourcefile(cast(Any, obj))
33+
raw_path: str | None = inspect.getsourcefile(cast(Any, class_anchor(obj) or obj))
3834
except TypeError:
3935
return None
4036
if raw_path is None:
@@ -66,6 +62,27 @@ def called_references(source: str) -> tuple[tuple[str, ...], ...]:
6662
return tuple(sorted(refs))
6763

6864

65+
def class_anchor(obj: object) -> FunctionType | None:
66+
"""A method of class `obj`; its file and globals outlive the module's sys.modules entry."""
67+
68+
if not inspect.isclass(obj):
69+
return None
70+
value: object
71+
for value in vars(obj).values():
72+
method: object = value.__func__ if isinstance(value, staticmethod | classmethod) else value
73+
if not isinstance(method, FunctionType):
74+
continue
75+
function: object = inspect.unwrap(method)
76+
if (
77+
isinstance(function, FunctionType)
78+
and function.__qualname__.startswith(f"{obj.__qualname__}.")
79+
and function.__module__ == obj.__module__
80+
and not function.__code__.co_filename.startswith("<")
81+
):
82+
return function
83+
return None
84+
85+
6986
def _reference_parts(node: ast.AST) -> tuple[str, ...] | None:
7087
if isinstance(node, ast.Name):
7188
return (node.id,)
@@ -86,21 +103,22 @@ def parse_class_starts(lines: list[str]) -> dict[str, int]:
86103
def _class_location(
87104
cls: type, *, class_starts: Callable[..., dict[str, int]]
88105
) -> tuple[list[str], int]:
89-
"""The lines and start index `inspect.findsource` returns for `cls` on Python 3.12."""
90-
91-
file: str | None = inspect.getsourcefile(cls)
92-
if file:
93-
linecache.checkcache(file)
94-
else:
95-
file = inspect.getfile(cls)
96-
if not (file.startswith("<") and file.endswith(">")):
97-
raise OSError("source code not available")
106+
"""The lines and start index `inspect.findsource` returns for `cls`, read from its own file."""
107+
108+
path: Path | None = source_file(cls)
109+
if path is None:
110+
raise OSError("source code not available")
111+
file: str = str(path)
112+
linecache.checkcache(file)
98113
module: ModuleType | None = inspect.getmodule(cls, file)
99114
lines: list[str] = (
100115
linecache.getlines(file, module.__dict__) if module else linecache.getlines(file)
101116
)
102117
if not lines:
103118
raise OSError("could not get source code")
119+
first_line: object = vars(cls).get("__firstlineno__")
120+
if isinstance(first_line, int):
121+
return lines, first_line - 1
104122
start: int | None = class_starts(file=file, lines=lines).get(cls.__qualname__)
105123
if start is None:
106124
raise OSError("could not find class definition")

‎tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,15 @@ def test_given_one_invocation_lookup_when_planned_then_each_table_is_read_once_b
375375
parent=tmp_path, config=clickhouse_e2e_config, database=recall_database
376376
)
377377
assert run(project_dir, "build").returncode == 0
378+
# A command that ends before its first heartbeat writes none, and ClickHouse plans a table
379+
# with no parts as a read of nothing, so give the heartbeats table a part before planning.
380+
execute_clickhouse_sql(
381+
config=clickhouse_e2e_config,
382+
statements=(
383+
f"INSERT INTO {recall_database}.recall_heartbeats (invocation_id) "
384+
"VALUES ('planning-setup')",
385+
),
386+
)
378387
sql: str = (
379388
f"SELECT * FROM {recall_database}.{test_case.view} WHERE invocation_id = 'any-invocation'"
380389
)

‎tests/integration/src/sqlbuild/cli/recall/_test_types.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ class HeldLockTestCase:
9797
shutdown_seconds: float
9898
builds_before_lock: int
9999
expected_exit_code: int
100+
expected_notice: str
101+
#: Far beyond the budget; only a build that waits for the lock itself reaches it.
102+
hang_guard_seconds: float
100103

101104

102105
@dataclass(frozen=True)

‎tests/integration/src/sqlbuild/cli/recall/helpers.py‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -349,20 +349,12 @@ def _still_running(process: subprocess.Popen[bytes]) -> bool:
349349
return process.poll() is None
350350

351351

352-
def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]:
353-
"""A build with the seconds until its first stderr byte and until it exited."""
352+
def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]:
353+
"""A build with the seconds until it exited."""
354354

355355
started: float = time.monotonic()
356-
process: subprocess.Popen[bytes] = start(project_dir, "build")
357-
assert process.stderr is not None
358-
first_byte: bytes = process.stderr.read(1)
359-
first_output: float = time.monotonic() - started
360-
stdout, stderr = process.communicate(timeout=600)
361-
return (
362-
subprocess.CompletedProcess(process.args, process.returncode, stdout, first_byte + stderr),
363-
first_output,
364-
time.monotonic() - started,
365-
)
356+
completed: subprocess.CompletedProcess[bytes] = run(project_dir, "build")
357+
return completed, time.monotonic() - started
366358

367359

368360
@contextmanager
@@ -395,7 +387,7 @@ def held_store_lock(project_dir: Path) -> Iterator[None]:
395387

396388
def build_while_store_locked(
397389
project_dir: Path,
398-
) -> tuple[subprocess.CompletedProcess[bytes], float, float]:
390+
) -> tuple[subprocess.CompletedProcess[bytes], float]:
399391
"""A timed build while another process holds the store's write lock."""
400392

401393
with held_store_lock(project_dir):

0 commit comments

Comments
 (0)