Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

use std::collections::HashSet;

use pyo3::IntoPyObjectExt;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::pyfunction;
use pyo3::types::{PyBool, PyDict, PyDictMethods};
use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE;
use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON;
use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name;
use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name;
Expand Down Expand Up @@ -123,17 +125,23 @@ fn candidate_records<'py>(candidates: &[Candidate<'py>]) -> PyResult<Vec<Bound<'
candidates.iter().map(Candidate::to_python).collect()
}

/// The active target's database or schema as a location: `preserve` names none, since each model keeps its own.
fn target_location(project: &Bound<'_, PyAny>, field: &str) -> PyResult<Option<String>> {
Ok(native_field(project, field)?
.extract::<Option<String>>()?
.filter(|value| value != PRESERVE_TARGET_VALUE))
}

/// Review schemas whose markers the active target schema holds and that outlived any review run.
fn review_schema_candidates<'py>(
project: &Bound<'py, PyAny>,
warehouse: &Warehouse<'py>,
now: &Bound<'py, PyAny>,
) -> PyResult<Vec<Bound<'py, PyAny>>> {
let py: Python<'py> = warehouse.py();
let schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?;
if schema.is_none() {
let Some(schema) = target_location(project, "effective_target_schema")? else {
return Ok(Vec::new());
}
};
let max_age: Bound<'py, PyAny> = days(py, LEFTOVER_REVIEW_SCHEMA_AGE_DAYS)?;
let leftovers: Bound<'py, PyAny> = call_with(
&host_entry(py, ExecutorHost::LeftoverReviewSchemas)?,
Expand All @@ -142,9 +150,9 @@ fn review_schema_candidates<'py>(
("connection", warehouse.connection.clone()),
(
"database",
native_field(project, "effective_target_database")?,
target_location(project, "effective_target_database")?.into_bound_py_any(py)?,
),
("schema", schema),
("schema", schema.into_bound_py_any(py)?),
("now", now.clone()),
("max_age", max_age),
],
Expand Down Expand Up @@ -208,11 +216,9 @@ pub(crate) fn build_plan<'py>(
push_artifact_schema(database.clone(), schema.clone());
}
}
if let Some(schema) =
native_field(project, "effective_target_schema")?.extract::<Option<String>>()?
{
if let Some(schema) = target_location(project, "effective_target_schema")? {
push_artifact_schema(
native_field(project, "effective_target_database")?.extract()?,
target_location(project, "effective_target_database")?,
schema,
);
}
Expand Down
16 changes: 13 additions & 3 deletions crates/sqlbuild-recall/src/writer/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use crate::store::main::open_sqlite_store::open_sqlite_store;
use crate::store::main::query_store::query_store;
use crate::store::main::read_output_transcript::read_output_transcript;
use crate::store::models::OpenMode;
use crate::writer::_helpers::delivery::locked;
use crate::writer::_helpers::queue::WriterState;
use crate::writer::main::start_recall_session::start_recall_session;
use crate::writer::models::{
Expand Down Expand Up @@ -175,8 +176,8 @@ pub(crate) fn unwritable_session() -> (bool, bool) {
)
}

/// Seconds `close` took and whether delivery degraded, while another connection holds the write lock.
pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) {
/// While another connection holds the write lock: whether closing set its deadline exactly the budget after it began, the degraded reason, and the seconds `close` took.
pub(crate) fn close_under_held_lock(shutdown: Duration) -> (bool, Option<String>, f64) {
let directory = tempfile::tempdir().expect("a temporary directory can be created");
let path = directory.path().join("logs/recall.sqlite3");
let warm = start_recall_session(config(path.clone(), "warm"));
Expand All @@ -191,10 +192,19 @@ pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) {
});
session.append_output(OutputStream::Stdout, b"waiting\n");
let started = std::time::Instant::now();
session.begin_close();
let began = std::time::Instant::now();
let deadline = locked(&session.shared.state)
.deadline
.expect("closing sets a deadline");
let summary = session.close();
let elapsed = started.elapsed().as_secs_f64();
drop(holder);
(elapsed, summary.degraded_reason.is_some())
(
started + shutdown <= deadline && deadline <= began + shutdown,
summary.degraded_reason,
elapsed,
)
}

/// The notices and resulting store state after starting on a file that is not a database.
Expand Down
25 changes: 15 additions & 10 deletions crates/sqlbuild-recall/src/writer/tests/test_session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ use crate::writer::tests::test_types::{
};

const CLOSE_SLACK_SECONDS: f64 = 0.3;
/// Far beyond any budget a test sets; only a close that waits for the store itself reaches it.
const CLOSE_HANG_GUARD_SECONDS: f64 = 60.0;

#[test]
fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() {
Expand Down Expand Up @@ -87,25 +89,28 @@ fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_clos
#[test]
fn given_a_held_write_lock_when_closed_then_the_shutdown_budget_caps_the_delay() {
let test_cases = [HeldLockTestCase {
description: "a locked store costs at most the shutdown budget at exit",
description: "a locked store gives up at the deadline the budget sets when closing begins",
shutdown_ms: 700,
expected_degraded: true,
expected_degraded_reason: Some(
"Recall delivery did not finish within the shutdown timeout",
),
}];

for test_case in test_cases {
let (elapsed, degraded) =
let (deadline_is_the_budget, degraded_reason, elapsed) =
close_under_held_lock(Duration::from_millis(test_case.shutdown_ms));
let budget: f64 = Duration::from_millis(test_case.shutdown_ms).as_secs_f64();
assert!(
elapsed <= budget + CLOSE_SLACK_SECONDS,
"{}: {elapsed:.3}s",
test_case.description
);
assert!(deadline_is_the_budget, "{}", test_case.description);
assert_eq!(
degraded, test_case.expected_degraded,
degraded_reason.as_deref(),
test_case.expected_degraded_reason,
"{}",
test_case.description
);
assert!(
elapsed < CLOSE_HANG_GUARD_SECONDS,
"{}: close hung for {elapsed:.3}s",
test_case.description
);
}
}

Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-recall/src/writer/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ pub(crate) struct UnwritableTestCase {
pub(crate) struct HeldLockTestCase {
pub(crate) description: &'static str,
pub(crate) shutdown_ms: u64,
pub(crate) expected_degraded: bool,
pub(crate) expected_degraded_reason: Option<&'static str>,
}

pub(crate) struct CorruptStoreTestCase {
Expand Down
7 changes: 5 additions & 2 deletions fensu.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1178,7 +1178,8 @@ max_positional_args = 1
[[targets.python.threshold_overrides]]
paths = ["src/sqlbuild/__init__.py"]
# Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py).
thresholds = { max_crate_code_lines = 89626 }
# Raised from 89,626 for helper-class anchors in Python identity (object_source.py, identity.py).
thresholds = { max_crate_code_lines = 89643 }
reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out."

[targets.python.rule_options.XSB074]
Expand Down Expand Up @@ -1703,7 +1704,9 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X

[[targets.rust.threshold_overrides]]
paths = ["crates/sqlbuild-python/Cargo.toml"]
thresholds = { max_crate_code_lines = 206367 }
# Raised from 206,364: +3 for the build identity crate split (#1510), +4 for preserve target
# locations in janitor planning (#1512).
thresholds = { max_crate_code_lines = 206371 }
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."

[[targets.rust.threshold_overrides]]
Expand Down
5 changes: 3 additions & 2 deletions src/sqlbuild/compiler/python_nodes/_helpers/identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from typing import Any, cast

from sqlbuild._native import PythonIdentityDependency, PythonNodeIdentity
from sqlbuild.compiler.python_nodes._helpers.object_source import hash_text
from sqlbuild.compiler.python_nodes._helpers.object_source import class_anchor, hash_text
from sqlbuild.compiler.python_nodes.classes.identity_pass import IdentityPass

_IGNORED_PATH_PARTS: frozenset[str] = frozenset(
Expand Down Expand Up @@ -182,8 +182,9 @@ def _visit_object(
collected_dependencies: dict[tuple[str, str], PythonIdentityDependency] = dependencies
collected_dependencies[key] = dependency

unwrapped: object = inspect.unwrap(cast(Any, obj))
globals_map: Mapping[str, object] | None = getattr(
inspect.unwrap(cast(Any, obj)), "__globals__", None
class_anchor(unwrapped) or unwrapped, "__globals__", None
)
if globals_map is None:
module: ModuleType | None = memo.module(obj)
Expand Down
52 changes: 35 additions & 17 deletions src/sqlbuild/compiler/python_nodes/_helpers/object_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,19 @@
import hashlib
import inspect
import linecache
import sys
import textwrap
from collections.abc import Callable
from pathlib import Path
from types import ModuleType
from types import FunctionType, ModuleType
from typing import Any, cast

_CLASS_FIRSTLINENO_VERSION: tuple[int, int] = (3, 13)
_FINDSOURCE_PARSES_FILE_PER_CLASS: bool = sys.version_info < _CLASS_FIRSTLINENO_VERSION


def normalized_source(*, obj: object, class_starts: Callable[..., dict[str, int]]) -> str:
"""`inspect.getsource(obj)` dedented and stripped, finding classes through `class_starts`."""
"""`inspect.getsource(obj)` dedented and stripped; classes read from their own file."""

try:
target: object = inspect.unwrap(cast(Any, obj))
if not (_FINDSOURCE_PARSES_FILE_PER_CLASS and inspect.isclass(target)):
if not inspect.isclass(target):
return textwrap.dedent(inspect.getsource(cast(Any, obj))).strip()
lines, start = _class_location(cast(type, target), class_starts=class_starts)
return textwrap.dedent("".join(inspect.getblock(lines[start:]))).strip()
Expand All @@ -34,7 +30,7 @@ def source_file(obj: object) -> Path | None:
"""The file `inspect` reads `obj`'s source from, or `None` when it has none."""

try:
raw_path: str | None = inspect.getsourcefile(cast(Any, obj))
raw_path: str | None = inspect.getsourcefile(cast(Any, class_anchor(obj) or obj))
except TypeError:
return None
if raw_path is None:
Expand Down Expand Up @@ -66,6 +62,27 @@ def called_references(source: str) -> tuple[tuple[str, ...], ...]:
return tuple(sorted(refs))


def class_anchor(obj: object) -> FunctionType | None:
"""A method of class `obj`; its file and globals outlive the module's sys.modules entry."""

if not inspect.isclass(obj):
return None
value: object
for value in vars(obj).values():
method: object = value.__func__ if isinstance(value, staticmethod | classmethod) else value
if not isinstance(method, FunctionType):
continue
function: object = inspect.unwrap(method)
if (
isinstance(function, FunctionType)
and function.__qualname__.startswith(f"{obj.__qualname__}.")
and function.__module__ == obj.__module__
and not function.__code__.co_filename.startswith("<")
):
return function
return None


def _reference_parts(node: ast.AST) -> tuple[str, ...] | None:
if isinstance(node, ast.Name):
return (node.id,)
Expand All @@ -86,21 +103,22 @@ def parse_class_starts(lines: list[str]) -> dict[str, int]:
def _class_location(
cls: type, *, class_starts: Callable[..., dict[str, int]]
) -> tuple[list[str], int]:
"""The lines and start index `inspect.findsource` returns for `cls` on Python 3.12."""

file: str | None = inspect.getsourcefile(cls)
if file:
linecache.checkcache(file)
else:
file = inspect.getfile(cls)
if not (file.startswith("<") and file.endswith(">")):
raise OSError("source code not available")
"""The lines and start index `inspect.findsource` returns for `cls`, read from its own file."""

path: Path | None = source_file(cls)
if path is None:
raise OSError("source code not available")
file: str = str(path)
linecache.checkcache(file)
module: ModuleType | None = inspect.getmodule(cls, file)
lines: list[str] = (
linecache.getlines(file, module.__dict__) if module else linecache.getlines(file)
)
if not lines:
raise OSError("could not get source code")
first_line: object = vars(cls).get("__firstlineno__")
if isinstance(first_line, int):
return lines, first_line - 1
start: int | None = class_starts(file=file, lines=lines).get(cls.__qualname__)
if start is None:
raise OSError("could not find class definition")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,15 @@ def test_given_one_invocation_lookup_when_planned_then_each_table_is_read_once_b
parent=tmp_path, config=clickhouse_e2e_config, database=recall_database
)
assert run(project_dir, "build").returncode == 0
# A command that ends before its first heartbeat writes none, and ClickHouse plans a table
# with no parts as a read of nothing, so give the heartbeats table a part before planning.
execute_clickhouse_sql(
config=clickhouse_e2e_config,
statements=(
f"INSERT INTO {recall_database}.recall_heartbeats (invocation_id) "
"VALUES ('planning-setup')",
),
)
sql: str = (
f"SELECT * FROM {recall_database}.{test_case.view} WHERE invocation_id = 'any-invocation'"
)
Expand Down
3 changes: 3 additions & 0 deletions tests/integration/src/sqlbuild/cli/recall/_test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ class HeldLockTestCase:
shutdown_seconds: float
builds_before_lock: int
expected_exit_code: int
expected_notice: str
#: Far beyond the budget; only a build that waits for the lock itself reaches it.
hang_guard_seconds: float


@dataclass(frozen=True)
Expand Down
18 changes: 5 additions & 13 deletions tests/integration/src/sqlbuild/cli/recall/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,20 +349,12 @@ def _still_running(process: subprocess.Popen[bytes]) -> bool:
return process.poll() is None


def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]:
"""A build with the seconds until its first stderr byte and until it exited."""
def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]:
"""A build with the seconds until it exited."""

started: float = time.monotonic()
process: subprocess.Popen[bytes] = start(project_dir, "build")
assert process.stderr is not None
first_byte: bytes = process.stderr.read(1)
first_output: float = time.monotonic() - started
stdout, stderr = process.communicate(timeout=600)
return (
subprocess.CompletedProcess(process.args, process.returncode, stdout, first_byte + stderr),
first_output,
time.monotonic() - started,
)
completed: subprocess.CompletedProcess[bytes] = run(project_dir, "build")
return completed, time.monotonic() - started


@contextmanager
Expand Down Expand Up @@ -395,7 +387,7 @@ def held_store_lock(project_dir: Path) -> Iterator[None]:

def build_while_store_locked(
project_dir: Path,
) -> tuple[subprocess.CompletedProcess[bytes], float, float]:
) -> tuple[subprocess.CompletedProcess[bytes], float]:
"""A timed build while another process holds the store's write lock."""

with held_store_lock(project_dir):
Expand Down
Loading
Loading