Skip to content

Commit 78a6706

Browse files
committed
fix(native): trust stored macro calls only for code identified by content
Module stamps were read when the store was saved, so a module imported before its file changed could be stored under the new file's stamps and replayed by later processes; unreadable modules (zip members, moved files) were skipped, and timestamps can be preserved or coarse. Only the first compile of a process now uses and saves the store, so code loaded by earlier compiles can no longer differ from disk. Loaded modules outside the interpreter and SQLBuild are identified by a BLAKE3 digest of their backing file, at attach, after recorded calls and at save; a module whose code cannot be read from a plain file disables the store for that compile. Installed compiled extensions are covered by their distributions' RECORD files, and the project fingerprint must be unchanged between attach and save. The process start time is no longer read, so the store also works without /proc.
1 parent 2d8d622 commit 78a6706

25 files changed

Lines changed: 975 additions & 113 deletions

File tree

‎Cargo.lock‎

Lines changed: 37 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0a
6565
polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "9ff0af282829d92efeeb36b6a820be2271e1da49", default-features = false, features = [
6666
"dialect-duckdb",
6767
] }
68+
blake3 = { version = "1", features = ["rayon"] }
6869
quote = "1"
6970
rayon = "1"
7071
saphyr-parser = "0.1"

‎crates/sqlbuild-cache/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ publish.workspace = true
88
rust-version.workspace = true
99

1010
[dependencies]
11+
blake3.workspace = true
1112
rayon.workspace = true
1213
sha2.workspace = true
1314
tempfile.workspace = true
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
//! BLAKE3 content digests of individual files, read and hashed in parallel.
2+
3+
use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
4+
use std::path::PathBuf;
5+
6+
use crate::digest::types::ContentDigest;
7+
8+
/// The digest of each file in `paths`; BLAKE3 keeps large compiled modules cheap to hash.
9+
pub fn digest_files(paths: &[PathBuf]) -> Vec<std::io::Result<ContentDigest>> {
10+
paths.par_iter().map(file_digest).collect()
11+
}
12+
13+
fn file_digest(path: &PathBuf) -> std::io::Result<ContentDigest> {
14+
let bytes: Vec<u8> = std::fs::read(path)?;
15+
let mut hasher = blake3::Hasher::new();
16+
let _ = hasher.update_rayon(&bytes);
17+
Ok(*hasher.finalize().as_bytes())
18+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
pub mod content_digest;
2+
pub mod digest_files;
23
pub mod fingerprint_project_files;
34
pub mod hex_digest;

‎crates/sqlbuild-cache/src/digest/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#[path = "tests/test_content_digest.rs"]
22
mod content_digest;
3+
#[path = "tests/test_digest_files.rs"]
4+
mod digest_files;
35
#[path = "tests/test_fingerprint_project_files.rs"]
46
mod fingerprint_project_files;
57
#[path = "tests/helpers.rs"]
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
use std::path::PathBuf;
2+
3+
use crate::digest::main::digest_files::digest_files;
4+
use crate::digest::tests::helpers::write_file;
5+
use crate::digest::tests::test_types::DigestFilesTestCase;
6+
7+
#[test]
8+
fn given_paths_when_digesting_files_then_readable_files_digest_their_bytes() {
9+
let test_cases = [
10+
DigestFilesTestCase {
11+
description: "regular file",
12+
path: |root| root.join("macros/cents.py"),
13+
expected_digest_of: Some("def cents(amount):\n return amount\n"),
14+
},
15+
DigestFilesTestCase {
16+
description: "missing file",
17+
path: |root| root.join("macros/missing.py"),
18+
expected_digest_of: None,
19+
},
20+
DigestFilesTestCase {
21+
description: "member path inside an archive",
22+
path: |root| root.join("macros/cents.py/inner.py"),
23+
expected_digest_of: None,
24+
},
25+
DigestFilesTestCase {
26+
description: "directory",
27+
path: |root| root.join("macros"),
28+
expected_digest_of: None,
29+
},
30+
];
31+
32+
for test_case in test_cases {
33+
let project = tempfile::tempdir().expect("temporary project");
34+
write_file(
35+
project.path(),
36+
"macros/cents.py",
37+
"def cents(amount):\n return amount\n",
38+
);
39+
let path: PathBuf = (test_case.path)(project.path());
40+
41+
let digests: Vec<Option<[u8; 32]>> =
42+
digest_files(&[path]).into_iter().map(Result::ok).collect();
43+
44+
assert_eq!(
45+
digests,
46+
vec![
47+
test_case
48+
.expected_digest_of
49+
.map(|text| *blake3::hash(text.as_bytes()).as_bytes())
50+
],
51+
"{}",
52+
test_case.description
53+
);
54+
}
55+
}

‎crates/sqlbuild-cache/src/digest/tests/test_types.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,9 @@ pub(crate) struct FingerprintFailureTestCase {
2727
pub(crate) root: fn(&Path) -> PathBuf,
2828
pub(crate) expected_failure: bool,
2929
}
30+
31+
pub(crate) struct DigestFilesTestCase {
32+
pub(crate) description: &'static str,
33+
pub(crate) path: fn(&Path) -> PathBuf,
34+
pub(crate) expected_digest_of: Option<&'static str>,
35+
}

‎crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use pyo3::exceptions::{PyOSError, PyRuntimeError};
44
use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python};
55
use pyo3::{pyfunction, wrap_pyfunction};
66
use sqlbuild_cache::digest::main::content_digest::content_digest as digest;
7+
use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each;
78
use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint;
89
use sqlbuild_cache::digest::main::hex_digest::hex_digest;
910
use sqlbuild_cache::digest::types::ContentDigest;
@@ -33,8 +34,25 @@ pub(crate) fn fingerprint_project_files(
3334
Ok(hex_digest(&fingerprinted))
3435
}
3536

37+
/// Hexadecimal content digest of each file in `paths`, or `None` where it cannot be read.
38+
#[pyfunction]
39+
pub(crate) fn digest_files(py: Python<'_>, paths: Vec<PathBuf>) -> PyResult<Vec<Option<String>>> {
40+
let digests: Vec<std::io::Result<ContentDigest>> = py
41+
.compiler_detach(|| Ok(digest_each(&paths)))
42+
.map_err(PyRuntimeError::new_err)?;
43+
Ok(digests.iter().map(readable_digest).collect())
44+
}
45+
46+
fn readable_digest(digest: &std::io::Result<ContentDigest>) -> Option<String> {
47+
match digest {
48+
Ok(digest) => Some(hex_digest(digest)),
49+
Err(_unreadable) => None,
50+
}
51+
}
52+
3653
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
3754
module.add_function(wrap_pyfunction!(content_digest, module)?)?;
3855
module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?;
56+
module.add_function(wrap_pyfunction!(digest_files, module)?)?;
3957
Ok(())
4058
}

‎src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,10 @@ SQLBuild relies on this contract to reuse macro results:
228228
- Macro call results may be kept between compiles under `target/cache/`. They are recomputed when
229229
macro or other Python files, configuration, declarations, seeds, sources, variables, the target,
230230
the adapter, the installed packages, or the SQLBuild version change. Editing only model SQL keeps
231-
them, and a changed call is always run again.
231+
them, and a changed call is always run again. Only the first compile in a process uses and saves
232+
them: later compiles in the same process, such as in a notebook or a long-running server, run their
233+
macros again. A compile that has loaded Python code it cannot read from a plain file, such as a
234+
module imported from a zip archive, neither uses nor saves them.
232235

233236
Pass configuration through what SQLBuild compares: macro arguments (including `@@ENV:NAME` and
234237
`@@var` values written in the calling SQL), `ctx.vars`, `ctx.target_name`, `ctx.adapter_name`,

0 commit comments

Comments
 (0)