Skip to content

Commit 883c887

Browse files
authored
Merge branch 'main' into ci/shared-linux-dist-memory
2 parents 7f54f67 + e8b4ff6 commit 883c887

10 files changed

Lines changed: 117 additions & 7 deletions

File tree

‎crates/sqlbuild-cache/src/artifacts/constants.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
pub const NEW_FILE_MODE: u32 = 0o666;
33
/// The existing SQL-test cache snapshot format, shared by native readers and writers.
44
pub(crate) const SQL_TEST_CACHE_VERSION: u64 = 5;
5-
pub(crate) const SQL_TEST_CACHE_FILE: &str = "sql-test-artifacts.json";
5+
/// The SQL-test cache snapshot's file name inside the compile cache directory.
6+
pub const SQL_TEST_CACHE_FILE: &str = "sql-test-artifacts.json";
67
pub(crate) const MAX_SQL_TEST_CACHE_BYTES: u64 = 10_000_000;
78
pub(crate) const MAX_SQL_TEST_CACHE_RECORDS: usize = 100_000;
89
/// The extension every compiled SQL test artifact carries.

‎crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use pyo3::prelude::{
99
};
1010
use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods};
1111
use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction};
12+
use sqlbuild_cache::artifacts::constants::SQL_TEST_CACHE_FILE;
1213
use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record;
1314
use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record;
1415
use sqlbuild_cache::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss};
@@ -66,6 +67,10 @@ const SQL_FILE_SUFFIX: &str = ".sql";
6667
const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse";
6768
/// The miss reason of a SQL test the cache file holds no record for.
6869
const NOT_RECORDED: &str = "not_recorded";
70+
/// Missed records a cache-miss debug line names before summarising the rest.
71+
const LOGGED_MISSES: usize = 16;
72+
/// Leading characters of a record key that name it in a cache-miss debug line.
73+
const LOGGED_KEY_CHARS: usize = 12;
6974
/// `functions/<language>` path components that already lead a function's path.
7075
const LANGUAGE_DIR_DEPTH: usize = 2;
7176
const PUBLISHED_STAGING_CHANGED: &str = "changed";
@@ -453,20 +458,43 @@ fn test_identities<'py>(
453458
Ok(identities)
454459
}
455460

456-
/// Log why a warm SQL test artifact cache could not keep some artifacts, by reason.
457-
fn log_cache_misses(py: Python<'_>, misses: &BTreeMap<&'static str, usize>) -> PyResult<()> {
461+
/// Log a warm SQL test cache's misses: counts by reason, then each record's key and reason.
462+
fn log_cache_misses(
463+
py: Python<'_>,
464+
misses: &BTreeMap<&'static str, usize>,
465+
missed: &[String],
466+
unreadable: bool,
467+
) -> PyResult<()> {
458468
let reasons: String = misses
459469
.iter()
460470
.map(|(reason, count)| format!("{reason}={count}"))
461471
.collect::<Vec<_>>()
462472
.join(",");
473+
let detail: String = if unreadable {
474+
"the cache file held no usable records".to_owned()
475+
} else {
476+
let mut shown: Vec<String> = missed.iter().take(LOGGED_MISSES).cloned().collect();
477+
if missed.len() > LOGGED_MISSES {
478+
shown.push(format!("{} more", missed.len() - LOGGED_MISSES));
479+
}
480+
format!("records {}", shown.join(","))
481+
};
463482
host::log_debug_message(
464483
py,
465484
SQL_TEST_CACHE_LOGGER,
466-
&format!("SQL test artifact cache missed: {reasons}"),
485+
&format!("SQL test artifact cache missed: {reasons} ({detail})"),
467486
)
468487
}
469488

489+
/// Whether `cache_dir` holds a SQL test cache file, so an empty read means it was unreadable.
490+
fn cache_file_present(cache_dir: &Bound<'_, PyAny>) -> PyResult<bool> {
491+
if cache_dir.is_none() {
492+
return Ok(false);
493+
}
494+
let directory: PathBuf = cache_dir.extract()?;
495+
Ok(directory.join(SQL_TEST_CACHE_FILE).is_file())
496+
}
497+
470498
/// Plan every SQL test whose cached artifact under `target_dir` cannot be reused.
471499
pub(crate) fn plan_tests<'py>(
472500
target_dir: &Bound<'py, PyAny>,
@@ -503,6 +531,7 @@ pub(crate) fn plan_tests<'py>(
503531
let mut cached_paths: Vec<PathBuf> = Vec::new();
504532
let mut cached_records: Vec<(String, SqlTestArtifactRecord)> = Vec::new();
505533
let mut misses: BTreeMap<&'static str, usize> = BTreeMap::new();
534+
let mut missed: Vec<String> = Vec::new();
506535
for (index, (test, identity)) in tests.into_iter().zip(identities).enumerate() {
507536
let Some((record_key, identity)) = identity else {
508537
pending.push((test.unbind(), None, None));
@@ -517,6 +546,8 @@ pub(crate) fn plan_tests<'py>(
517546
});
518547
if let Err(reason) = reusable {
519548
*misses.entry(reason).or_default() += 1;
549+
let short: &str = record_key.get(..LOGGED_KEY_CHARS).unwrap_or(&record_key);
550+
missed.push(format!("{short}:{reason}"));
520551
}
521552
if let Ok(record) = reusable {
522553
let path = tests_root_path.join(&record.relative_path);
@@ -528,8 +559,8 @@ pub(crate) fn plan_tests<'py>(
528559
}
529560
yield_interpreter(py, index);
530561
}
531-
if !cached.is_empty() && !misses.is_empty() {
532-
log_cache_misses(py, &misses)?;
562+
if !misses.is_empty() && (!cached.is_empty() || cache_file_present(&cache_dir)?) {
563+
log_cache_misses(py, &misses, &missed, cached.is_empty())?;
533564
}
534565
let pending_tests: Vec<Bound<'py, PyAny>> = pending
535566
.iter()

‎scripts/delta_compile_series/_helpers/outcomes.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
from __future__ import annotations
44

5+
import json
56
import re
67
from itertools import zip_longest
78
from pathlib import Path
9+
from typing import Any
810

911
from scripts.delta_compile_series.constants import (
1012
AFTER_FAILED_RULES,
@@ -15,7 +17,11 @@
1517
DELTA_MISMATCH_PATTERN,
1618
DELTA_VERIFIED_LINE,
1719
DURATION_PATTERN,
20+
LOG_DIAGNOSTICS_FILE,
21+
LOG_RUNS_GLOB,
22+
MAX_CACHE_LOG_LINES,
1823
MISMATCH_EXIT_CODE,
24+
SQL_TEST_CACHE_LOG_PREFIX,
1925
STALE_PROBLEMS,
2026
)
2127
from scripts.delta_compile_series.models import DeltaCompileEdit, DeltaCompileOutcome
@@ -96,3 +102,31 @@ def full_compile_difference(
96102
)
97103
return f"{stream} differs from a full compile: {first[0]!r} vs {first[1]!r}"
98104
return None
105+
106+
107+
def log_runs(project: Path) -> frozenset[Path]:
108+
"""The per-invocation log directories a project holds."""
109+
110+
return frozenset(path for path in project.glob(LOG_RUNS_GLOB) if path.is_dir())
111+
112+
113+
def sql_test_cache_lines(*, project: Path, earlier: frozenset[Path]) -> str:
114+
"""Cache-miss reasons and record keys logged since `earlier`, never SQL text or environment."""
115+
116+
lines: list[str] = []
117+
for run in sorted(log_runs(project) - earlier):
118+
try:
119+
text: str = (run / LOG_DIAGNOSTICS_FILE).read_text(encoding="utf-8", errors="replace")
120+
rows: list[str] = text.splitlines()
121+
except OSError:
122+
continue
123+
for row in rows:
124+
try:
125+
record: Any = json.loads(row)
126+
except ValueError:
127+
continue
128+
message: Any = record.get("message") if isinstance(record, dict) else None
129+
if isinstance(message, str) and message.startswith(SQL_TEST_CACHE_LOG_PREFIX):
130+
lines.append(message)
131+
shown: list[str] = lines[:MAX_CACHE_LOG_LINES]
132+
return " | ".join(shown) if shown else "none logged"

‎scripts/delta_compile_series/constants.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
PROJECT_CONFIG: str = "sqlbuild_project.toml"
2323
RANDOM_VALUE_LIMIT: int = 10**6
2424
FULL_COPY_SUFFIX: str = "_full_copy"
25+
LOG_RUNS_GLOB: str = "logs/*/*"
26+
LOG_DIAGNOSTICS_FILE: str = "diagnostics.jsonl"
27+
SQL_TEST_CACHE_LOG_PREFIX: str = "SQL test artifact cache"
28+
MAX_CACHE_LOG_LINES: int = 4
2529
DURATION_PATTERN: re.Pattern[str] = re.compile(r"\s*\(\d+(?:\.\d+)?s\)")
2630
COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}')
2731
MAX_FAILURES: int = 4

‎scripts/delta_compile_series/main/run_delta_compile_series.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@
1010

1111
from scripts.delta_compile_series._helpers.edits import edited_file
1212
from scripts.delta_compile_series._helpers.evidence import cache_state, keep_comparison_failure
13-
from scripts.delta_compile_series._helpers.outcomes import full_compile_difference, judged
13+
from scripts.delta_compile_series._helpers.outcomes import (
14+
full_compile_difference,
15+
judged,
16+
log_runs,
17+
sql_test_cache_lines,
18+
)
1419
from scripts.delta_compile_series.constants import (
1520
DELTA_COMPILE_LINE,
1621
DELTA_WEIGHT,
@@ -41,6 +46,7 @@ def _compared(
4146
copy: Path = project_dir.with_name(project_dir.name + FULL_COPY_SUFFIX)
4247
shutil.rmtree(copy, ignore_errors=True)
4348
_ = shutil.copytree(project_dir, copy, symlinks=True)
49+
earlier: tuple[frozenset[Path], frozenset[Path]] = (log_runs(project_dir), log_runs(copy))
4450
before: dict[str, Any] = (
4551
{"delta": cache_state(project_dir), "full": cache_state(copy)} if diagnostics_dir else {}
4652
)
@@ -68,6 +74,12 @@ def _compared(
6874
)
6975
except OSError:
7076
pass
77+
if problem is not None:
78+
problem += (
79+
f"; SQL test cache: delta "
80+
f"{sql_test_cache_lines(project=project_dir, earlier=earlier[0])}"
81+
f"; full {sql_test_cache_lines(project=copy, earlier=earlier[1])}"
82+
)
7183
return outcome, problem
7284

7385

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5573,3 +5573,11 @@ def flip_compiled_sql_test_bytes(project_dir: Path) -> None:
55735573

55745574
for artifact in sorted((project_dir / "target/compiled/tests").rglob("*.sql")):
55755575
_ = artifact.write_text(artifact.read_text().replace("SELECT", "select", 1))
5576+
5577+
5578+
def truncate_sql_test_cache(project_dir: Path) -> None:
5579+
"""Cut every SQL test cache file to its first half, as a write cut short would leave it."""
5580+
5581+
for cache in sorted((project_dir / "target/cache").rglob("sql-test-artifacts.json")):
5582+
text: str = cache.read_text()
5583+
_ = cache.write_text(text[: len(text) // 2])

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
append_to_compiled_sql_tests,
1616
flip_compiled_sql_test_bytes,
1717
run_installed_sqb,
18+
truncate_sql_test_cache,
1819
)
1920

2021
_MISS_LINE: str = "SQL test artifact cache missed"
@@ -34,6 +35,11 @@
3435
change=flip_compiled_sql_test_bytes,
3536
expected_debug_line=f"{_MISS_LINE}: content_changed=",
3637
),
38+
SqlTestCacheMissE2ETestCase(
39+
description="a cut-short cache file logs that it held no usable records",
40+
change=truncate_sql_test_cache,
41+
expected_debug_line="(the cache file held no usable records)",
42+
),
3743
],
3844
ids=lambda case: case.description,
3945
)

‎tests/unit/scripts/delta_compile_series/main/_test_types.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ class ComparisonEvidenceTestCase:
1313
steps: int = 1
1414
expected_mismatches: int = 1
1515
cache_relative_path: str = "orders/check.sql"
16+
full_cache_log: str = (
17+
"SQL test artifact cache missed: not_recorded=1 (the cache file held no usable records)"
18+
)

‎tests/unit/scripts/delta_compile_series/main/helpers.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,14 @@ def compile_project() -> DeltaCompileOutcome:
5252

5353
def compile_full(copy: Path) -> DeltaCompileOutcome:
5454
_ = (copy / cache.relative_to(project)).write_text("{")
55+
run: Path = copy / "logs/2026-11-10/full-run"
56+
run.mkdir(parents=True)
57+
_ = (run / "diagnostics.jsonl").write_text(
58+
json.dumps({"message": "Delta compile fell back: disabled"})
59+
+ "\n"
60+
+ json.dumps({"message": test_case.full_cache_log})
61+
+ "\n"
62+
)
5563
return DeltaCompileOutcome(
5664
0,
5765
test_case.stdout,

‎tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ def test_given_mismatch_when_capturing_then_keeps_private_paired_evidence(
3232
"stderr differs from a full compile: '' vs 'SQL test planning START'"
3333
in result.unexpected[0]
3434
)
35+
assert result.unexpected[0].endswith(
36+
f"; SQL test cache: delta none logged; full {test_case.full_cache_log}"
37+
)
3538
directory: Path = destination / "failure-00"
3639
manifest: dict[str, Any] = json.loads((directory / "manifest.json").read_text())
3740
before: dict[str, Any] = json.loads((directory / "cache-before.json").read_text())

0 commit comments

Comments
 (0)