From 5b5492ca88aa40132110323b81d00b2fba207a1d Mon Sep 17 00:00:00 2001 From: Mahesh Vaikri Date: Tue, 17 Nov 2026 07:53:09 -0600 Subject: [PATCH 1/7] Snapshot interchange: golden fixture written by Rust and a checker for other ports' output snapshot_fixture writes tests/fixtures/snapshot/golden.* (graph, JSON sidecar, binary snapshot) with the awkward cases in it; every port must load it from the snapshot. snapshot_check loads a graph saved by any port and reports whether the snapshot was accepted and whether the store equals a JSON-only load. Co-Authored-By: Claude Sonnet 5.5 --- .gitattributes | 4 + .gitignore | 1 + .../examples/snapshot_check.rs | 52 ++++++++++++ .../examples/snapshot_fixture.rs | 75 ++++++++++++++++++ tests/fixtures/snapshot/golden.ison | 8 ++ .../snapshot/golden.ison.embeddings.json | 1 + .../snapshot/golden.ison.embeddings.snap | Bin 0 -> 3319 bytes 7 files changed, 141 insertions(+) create mode 100644 isongraph-vector-rs/examples/snapshot_check.rs create mode 100644 isongraph-vector-rs/examples/snapshot_fixture.rs create mode 100644 tests/fixtures/snapshot/golden.ison create mode 100644 tests/fixtures/snapshot/golden.ison.embeddings.json create mode 100644 tests/fixtures/snapshot/golden.ison.embeddings.snap diff --git a/.gitattributes b/.gitattributes index 0b48069..e58c4f6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,3 +23,7 @@ *.db binary *.png binary *.ison text + +# Byte-exact fixtures: the snapshot stores the sidecar's length and a hash of its bytes, so +# any line-ending rewrite on checkout would make it (correctly) stale. +tests/fixtures/snapshot/* -text diff --git a/.gitignore b/.gitignore index 21a265c..b3c86cb 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ obj/ .idea/ .DS_Store Thumbs.db +!tests/fixtures/snapshot/*.embeddings.json diff --git a/isongraph-vector-rs/examples/snapshot_check.rs b/isongraph-vector-rs/examples/snapshot_check.rs new file mode 100644 index 0000000..04e6048 --- /dev/null +++ b/isongraph-vector-rs/examples/snapshot_check.rs @@ -0,0 +1,52 @@ +//! Check a graph saved by any port: is its binary snapshot accepted, and is the +//! store it yields the same as a JSON-only load? +//! +//! Usage: cargo run --example snapshot_check -- [dim] +//! +//! Prints `source= skipped= count= equal_to_json=` +//! and exits non-zero if the snapshot was not used or the stores differ. The +//! comparison is the store's JSON export, which carries every vector bit for +//! bit (shortest round-trip float text). + +use isongraph_vector_rs::{ + snapshot_path_for, LoadOptions, LoadSource, MockEncoder, SemanticGraph, +}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let graph = args.next().ok_or("usage: snapshot_check [dim]")?; + let dim: usize = args.next().map(|d| d.parse()).transpose()?.unwrap_or(12); + let enc = || Box::new(MockEncoder::new(dim)); + + let (g, report) = SemanticGraph::load_with_report(&graph, enc(), &LoadOptions::default())?; + + // A JSON-only twin: copy graph + sidecar, leave the snapshot behind. + let twin = format!("{graph}.jsononly.ison"); + std::fs::copy(&graph, &twin)?; + std::fs::copy( + SemanticGraph::embeddings_path_for(&graph), + SemanticGraph::embeddings_path_for(&twin), + )?; + let _ = std::fs::remove_file(snapshot_path_for(&twin)); + let (j, jreport) = SemanticGraph::load_with_report(&twin, enc(), &LoadOptions::default())?; + let equal = g.embedding_store().to_json()? == j.embedding_store().to_json()?; + for p in [ + twin.clone(), + SemanticGraph::embeddings_path_for(&twin).display().to_string(), + ] { + let _ = std::fs::remove_file(p); + } + + println!( + "source={:?} skipped={} count={} json_count={} equal_to_json={}", + report.source, + report.snapshot_skipped.as_deref().unwrap_or("none"), + report.embeddings, + jreport.embeddings, + equal + ); + if report.source != LoadSource::Snapshot || !equal { + std::process::exit(1); + } + Ok(()) +} diff --git a/isongraph-vector-rs/examples/snapshot_fixture.rs b/isongraph-vector-rs/examples/snapshot_fixture.rs new file mode 100644 index 0000000..f6473f2 --- /dev/null +++ b/isongraph-vector-rs/examples/snapshot_fixture.rs @@ -0,0 +1,75 @@ +//! Write the golden snapshot fixture the other ports must read. +//! +//! Usage: cargo run --example snapshot_fixture -- +//! +//! Produces `golden.ison`, `golden.ison.embeddings.json` and +//! `golden.ison.embeddings.snap`: a small graph whose store holds ids with +//! colons and unicode, empty / quoted / multi-line / unicode text, an exact +//! duplicate vector (a tie), a zero vector, two encoder models, updates and +//! tombstones. Every port's test loads the three files, expects the snapshot +//! to be used, and expects the same store as a JSON-only load. + +use isongraph_vector_rs::{MockEncoder, NodeId, SemanticGraph}; + +const DIM: usize = 12; + +struct Rng(u64); +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn f(&mut self) -> f32 { + ((self.next_u64() >> 40) as f32 / (1u64 << 23) as f32) - 1.0 + } +} + +fn main() -> Result<(), Box> { + let dir = std::env::args().nth(1).ok_or("usage: snapshot_fixture ")?; + std::fs::create_dir_all(&dir)?; + let n = 40usize; + let mut g = SemanticGraph::new("golden", Box::new(MockEncoder::new(DIM))); + g.add_node_with_embed("person", "1", vec![("name", "Alice")], None)?; + g.add_node_with_embed("person", "2", vec![("name", "Bob")], None)?; + g.add_edge("KNOWS", ("person", "1"), ("person", "2"), vec![])?; + let mut rng = Rng(7); + let types = ["doc", "note", "a:b"]; + let id_of = |i: usize| { + if matches!(i % 9, 0) { format!("id:with:colons:{i}") } else { format!("{i:05}") } + }; + let mut vectors: Vec> = Vec::new(); + for i in 0..n { + let v: Vec = if i % 17 == 5 && !vectors.is_empty() { + vectors[i / 2].clone() + } else if i % 41 == 7 || i == 7 { + vec![0.0; DIM] + } else { + (0..DIM).map(|_| rng.f()).collect() + }; + vectors.push(v.clone()); + if i == n / 2 { + g.set_encoder(Box::new(MockEncoder::new(DIM).with_model_name("second-model"))); + } + let text = match i % 5 { + 0 => String::new(), + 1 => format!("quote \" and \\ and\nnewline {i}"), + 2 => format!("unicode \u{1F600} \u{00E9}\u{4E2D} {i}"), + _ => format!("text {i}"), + }; + g.embedding_store_mut().add(NodeId::new(types[i % 3], id_of(i)), &text, Some(v))?; + } + for i in (0..n).step_by(11) { + let v: Vec = (0..DIM).map(|_| rng.f()).collect(); + g.embedding_store_mut().add(NodeId::new(types[i % 3], id_of(i)), "updated", Some(v))?; + } + for i in (3..n).step_by(7) { + g.embedding_store_mut().remove(&NodeId::new(types[i % 3], id_of(i))); + } + let path = std::path::Path::new(&dir).join("golden.ison"); + g.save_with_snapshot(&path, None)?; + println!("wrote {} (+ .embeddings.json, .embeddings.snap)", path.display()); + Ok(()) +} diff --git a/tests/fixtures/snapshot/golden.ison b/tests/fixtures/snapshot/golden.ison new file mode 100644 index 0000000..7ebf9be --- /dev/null +++ b/tests/fixtures/snapshot/golden.ison @@ -0,0 +1,8 @@ +edges.KNOWS +source target +:person:1 :person:2 + +nodes.person +id name +"1" Alice +"2" Bob \ No newline at end of file diff --git a/tests/fixtures/snapshot/golden.ison.embeddings.json b/tests/fixtures/snapshot/golden.ison.embeddings.json new file mode 100644 index 0000000..d180ab0 --- /dev/null +++ b/tests/fixtures/snapshot/golden.ison.embeddings.json @@ -0,0 +1 @@ +{"format":"ison-embeddings","version":1,"dimension":12,"count":36,"embeddings":[{"type":"a:b","id":"00002","id_type":"str","text":"unicode 😀 é中 2","model":"mock-12","vector":[0.8050792,0.920658,-0.8466375,-0.18591201,0.8036902,-0.16993725,0.9422715,-0.89151955,-0.23363316,-0.4348215,0.11494386,0.21354151]},{"type":"a:b","id":"00005","id_type":"str","text":"","model":"mock-12","vector":[0.8050792,0.920658,-0.8466375,-0.18591201,0.8036902,-0.16993725,0.9422715,-0.89151955,-0.23363316,-0.4348215,0.11494386,0.21354151]},{"type":"a:b","id":"00008","id_type":"str","text":"text 8","model":"mock-12","vector":[0.054136753,-0.04568243,0.31082678,0.88051426,-0.5584425,-0.28445935,0.056880116,0.41676652,0.24353206,-0.37112403,0.5382987,-0.347458]},{"type":"a:b","id":"00011","id_type":"str","text":"updated","model":"second-model","vector":[0.3610896,-0.41867912,0.20953977,-0.60416067,0.11171162,-0.8276391,0.10333073,0.8598863,0.29210448,-0.92103255,-0.5724354,0.6287266]},{"type":"a:b","id":"00014","id_type":"str","text":"text 14","model":"mock-12","vector":[0.27783287,-0.84936905,-0.35543454,0.20684361,0.99827456,0.71238995,-0.83142555,0.14356387,0.6728343,0.009392381,0.36652577,0.6857302]},{"type":"a:b","id":"00020","id_type":"str","text":"","model":"second-model","vector":[-0.1752863,0.10126865,0.33329463,0.20989013,-0.68514454,-0.9313737,-0.43688715,-0.9079877,-0.3751434,0.029261827,0.6142359,0.8258182]},{"type":"a:b","id":"00023","id_type":"str","text":"text 23","model":"second-model","vector":[-0.45782876,-0.028327227,0.0003670454,-0.5847249,-0.77432334,0.7069124,-0.07635343,-0.10006595,-0.4363159,0.03194499,0.6826962,-0.832093]},{"type":"a:b","id":"00026","id_type":"str","text":"quote \" and \\ and\nnewline 26","model":"second-model","vector":[-0.9391041,-0.37151802,0.326069,-0.6785368,-0.2089442,0.026219964,0.58033013,-0.49688852,-0.20048594,0.98545873,0.6546161,-0.2565918]},{"type":"a:b","id":"00029","id_type":"str","text":"text 29","model":"second-model","vector":[-0.31227958,0.3897463,0.29373598,0.98497486,0.031829596,0.9324832,-0.44791818,-0.7190802,-0.7298348,0.042829275,-0.07338023,-0.08227646]},{"type":"a:b","id":"00032","id_type":"str","text":"unicode 😀 é中 32","model":"second-model","vector":[0.7757329,0.95608735,-0.25136828,-0.7373495,0.51037014,0.6613275,-0.41591692,-0.5077833,-0.27747536,0.6015326,-0.67231905,0.53601134]},{"type":"a:b","id":"00035","id_type":"str","text":"","model":"second-model","vector":[0.80203736,0.38069558,0.002922058,-0.97697747,0.4997301,0.35036075,-0.63711333,-0.15405083,-0.44697797,-0.14527142,-0.035750985,-0.30918717]},{"type":"doc","id":"00006","id_type":"str","text":"quote \" and \\ and\nnewline 6","model":"mock-12","vector":[-0.28959537,-0.25882542,0.7484455,-0.19686449,-0.07312417,-0.28342497,0.59338236,0.55575,0.96993935,0.9081713,-0.27167714,-0.9499024]},{"type":"doc","id":"00012","id_type":"str","text":"unicode 😀 é中 12","model":"mock-12","vector":[0.09344816,0.2275275,-0.58322704,0.26236343,0.23475718,-0.6592735,0.8216685,0.2681986,0.16742682,-0.21788251,0.973492,0.6773857]},{"type":"doc","id":"00015","id_type":"str","text":"","model":"mock-12","vector":[-0.5074742,-0.59585667,-0.48235846,0.54851604,-0.85029674,-0.39373708,-0.45806122,-0.6107091,-0.40986788,0.90221643,0.57111275,-0.041509748]},{"type":"doc","id":"00021","id_type":"str","text":"quote \" and \\ and\nnewline 21","model":"second-model","vector":[0.38113558,0.45426464,-0.31960034,-0.36907363,-0.80885303,-0.14571488,-0.5527477,0.6733229,-0.14699888,0.01282382,0.33394575,0.38454962]},{"type":"doc","id":"00030","id_type":"str","text":"","model":"second-model","vector":[-0.9024378,0.42825615,0.8775512,-0.75369155,0.2352668,-0.73756075,-0.5741323,0.16092896,0.40547323,-0.04142654,0.2231096,-0.4365443]},{"type":"doc","id":"00033","id_type":"str","text":"updated","model":"second-model","vector":[0.27036119,0.16490233,-0.7077726,0.58693814,-0.44947875,0.97008705,-0.7092167,0.22691059,-0.47880685,-0.8993223,0.39195716,-0.67719126]},{"type":"doc","id":"00039","id_type":"str","text":"text 39","model":"second-model","vector":[0.5104972,0.05613768,-0.5862682,-0.94853413,0.72464406,-0.546612,0.104543924,0.7945465,-0.98715365,-0.5010102,-0.29698777,0.4021902]},{"type":"doc","id":"id:with:colons:0","id_type":"str","text":"updated","model":"second-model","vector":[-0.9099302,-0.42507136,-0.9188955,0.10306871,0.59478986,-0.03611064,-0.37001157,-0.1484077,-0.43493235,0.5916629,-0.29600787,-0.97522414]},{"type":"doc","id":"id:with:colons:18","id_type":"str","text":"text 18","model":"mock-12","vector":[0.22471523,-0.12138164,-0.7979369,0.020962,0.78609014,0.555807,-0.9494059,0.17288172,-0.19451845,-0.36982405,-0.16182888,-0.7368301]},{"type":"doc","id":"id:with:colons:27","id_type":"str","text":"unicode 😀 é中 27","model":"second-model","vector":[0.67481494,0.49631917,-0.8106291,0.2687688,-0.15264654,0.9744662,0.6807655,0.13180721,0.39953113,0.8217685,-0.6411215,-0.11971867]},{"type":"doc","id":"id:with:colons:36","id_type":"str","text":"quote \" and \\ and\nnewline 36","model":"second-model","vector":[0.7279917,0.38292813,-0.9827241,0.98499227,-0.2494477,0.7903837,0.51370585,0.61004496,-0.043566823,0.23106015,0.6899421,0.87346625]},{"type":"doc","id":"id:with:colons:9","id_type":"str","text":"text 9","model":"mock-12","vector":[-0.97117984,-0.6807561,0.27949727,-0.41214204,0.6645837,-0.4928758,0.9091841,-0.86614037,-0.11689603,0.9145397,0.39121926,0.14136231]},{"type":"note","id":"00001","id_type":"str","text":"quote \" and \\ and\nnewline 1","model":"mock-12","vector":[0.83603907,0.7426635,0.7280153,0.09657478,0.7592274,-0.3472774,0.2382412,0.5146439,0.34913337,-0.78661144,-0.31111443,-0.15245497]},{"type":"note","id":"00004","id_type":"str","text":"text 4","model":"mock-12","vector":[0.6906998,0.28766084,0.34114742,-0.2768376,-0.8703468,-0.49489057,0.14704335,-0.5557364,-0.0027575493,-0.3286034,0.14117074,0.23255193]},{"type":"note","id":"00007","id_type":"str","text":"unicode 😀 é中 7","model":"mock-12","vector":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]},{"type":"note","id":"00013","id_type":"str","text":"text 13","model":"mock-12","vector":[-0.46734905,0.39290595,0.46246362,-0.7422104,-0.41532254,0.10225761,-0.07882273,-0.52962136,0.79460955,-0.81512153,-0.46347654,0.23978949]},{"type":"note","id":"00016","id_type":"str","text":"quote \" and \\ and\nnewline 16","model":"mock-12","vector":[-0.2452445,-0.49679494,0.9667933,-0.32720673,-0.9838953,0.005624771,-0.87283313,-0.2964561,0.49146068,-0.41747272,-0.18786919,-0.52862453]},{"type":"note","id":"00019","id_type":"str","text":"text 19","model":"mock-12","vector":[0.5104972,0.05613768,-0.5862682,-0.94853413,0.72464406,-0.546612,0.104543924,0.7945465,-0.98715365,-0.5010102,-0.29698777,0.4021902]},{"type":"note","id":"00022","id_type":"str","text":"updated","model":"second-model","vector":[-0.2443732,0.754719,-0.6513661,0.68599236,0.68520486,-0.6713158,-0.77512777,0.6166973,0.71289194,-0.62710774,-0.29552865,0.3338461]},{"type":"note","id":"00025","id_type":"str","text":"","model":"second-model","vector":[-0.7104616,-0.83380055,0.066331625,0.9244201,-0.96211946,0.41779757,0.47778165,0.5672618,0.5437058,0.10284054,-0.8021743,-0.34949064]},{"type":"note","id":"00028","id_type":"str","text":"text 28","model":"second-model","vector":[0.9924885,0.95433974,-0.73094046,0.5899153,-0.86320734,-0.8835025,-0.4023708,0.45610523,0.36763,-0.47795796,0.30989468,0.18956292]},{"type":"note","id":"00034","id_type":"str","text":"text 34","model":"second-model","vector":[0.64239836,-0.21120417,-0.6285505,-0.25374186,0.74021244,-0.80122614,-0.08350146,-0.836689,-0.5168791,-0.70095,-0.9048079,-0.7049277]},{"type":"note","id":"00037","id_type":"str","text":"unicode 😀 é中 37","model":"second-model","vector":[0.03839004,-0.12043488,0.9889339,0.53021,-0.48735857,-0.07312477,-0.786935,-0.54349077,0.17874408,0.44422984,-0.7151289,-0.61311424]},{"type":"person","id":"1","id_type":"str","text":"Alice","model":"mock-12","vector":[-0.8610729,-0.33433855,0.8408419,-0.2771846,0.40870202,-0.35926688,-0.39656878,0.49080622,0.54880023,0.45217478,0.8313228,0.20298672]},{"type":"person","id":"2","id_type":"str","text":"Bob","model":"mock-12","vector":[-0.22434258,0.78542376,-0.014674783,0.6692934,0.35582304,-0.77352357,0.23580599,-0.83886147,-0.46037877,0.14397812,0.7654649,0.0029057264]}],"encoder_model":"second-model","graph_digest":"sha256:a33458853f63a977962395b84c42fb347ed0f6ed66bc00eabb92ef69bb83026c","graph_digest_basis":"nodes-edges-json-v1"} \ No newline at end of file diff --git a/tests/fixtures/snapshot/golden.ison.embeddings.snap b/tests/fixtures/snapshot/golden.ison.embeddings.snap new file mode 100644 index 0000000000000000000000000000000000000000..cf8442f9069f0294b73f25fe1cc93f7365c075ae GIT binary patch literal 3319 zcmb7E3sjR=5{^g_m1QFeqQV*}tEfDZKOuxj?x2eZmWK<9s3;*M5CoGT5mBjTDXY{c zT1D2fkXox0ty1e!j*>f+QWsl__&l_FG^e_X_(B(y78myq=Lib19_O6<-~Y@vGxy&4 zW+pr~Iw~$UYTghIhhtd|?BdvRIG&cBl{xInQYSI~Vri04DDsy`{DmoEVNzf~K%kfr z2Kp-`qGS=1q7aG#R7!q|Sfvz;70Gtko!eRMwW!7wHyH_}zPnS+qAn{Vsynoh#_0=q2M``p)zV~bhE|g$x(vraCi6fRD*hzWa zYFUQj63LOTAzec$ef;Bau)Xyqtua>8%(2KVyCBP(lmOM+ z_mJ-!(vi2t38fu8a0}i}f^~~9zx!Xbxha<1+r9z#iDHOqnU89l(c~KO2E#AofVNfA)hx6)KC{VXi zgJWwAM^_tIQ0a`0(oht4iP$qrO!9|~rbgv;TH9hH^Zm{j9BdaLFHMIf0hH!HK1hmA z_+U=w0vi0o<)Hk+G2pH4q5OzQYBwhr?(Loe9Qg;-?M4~Bxb@$pP+3IgP05AYL>rmp zYYh}E_zykcEJ6KcDb}v=lgY!1OofJWlcp#XJnQrj|CNEZ*9BqeAseWRmVG9E^AO#u1X?5EBy#1#NYNV}KdVgtchA`kdq{z9Lu0y^98Prx8Ejr1o`V zs8Q==YKSbQEwgvi`uu##+Xv866i#_{drV{8chc6|V`2YnM|d2W18wiCA!OVZ+T@*u zwMi)JdZs4JqMXq%e1pvAn4IPV7w-%~?q*l4 zeENiLO?Lo?FBW0z-ZPZ5sf}tDx}oUjbF}`@jxG)E#JNz+tiJEv% z?xfEGQ?TS{15qdV;NxR9U~E^(jySuZ$Ay&`nimN54@M$y!ev=X>U%K0Di=kwIN0D= zOwD(HB>YEe?AZSktqDrUO6Lo*$sw*#+dA6hwbDfSU1O>K*flCyJr?4(@1=9g*I@00 z6w+SQWh$N>frjeCK_Xc=EDLvp5Qi*m9VQ?xesk#VMJAe4^=Ep^7=%Y^y54)tESAkl81 zMR9XMnl}_$r@H|!i!s@MoIxCne8~Mc5lC4M+K2jL$oL&3%=QBH@|Xsa#n0&6S~bW| zEk%CnLsRI)<#02yfrNd-CFOQ;z)U0{PLu-Y%y+Vr39}%+=}T(p%pxanD3pA2oGeQW zN9Eiy>gDc6Pn_dJSlCCzTyd0?o3>Ghv&%3-@jqIy%Z1PoN90tCPMsj-l)HmVTs<_z zGPB-BzTc@#@J1s(Mz;J9!%R~hrM+@bOo)YA^tPr z0ZvBdpMFr3<_C+fOvlrjP&8k62N#1FJT~*NAbKw~->)FeW`FF+T7c%3O|r&=8IT>B zjfHnEk;bwP;v^XbQT&m>Ej=%bSQ3f4_T%*A==D$%t%tTh&A>|AvEbF6fD#`CZXMc0 zjT`R}jsQYUha|DY;)KYEQmvXj4fto z3UU1>spy$4LNjrf6&`$IIz8=uXgM_uxSM{ZI(A0v(ubqiV6%Gkg!ziO9r*+8nA=Uf*E@jzgWF`+x%p6gC7N(gs%XKe za4=|eHO+C;LFZdV4!jeJyw#b+h;dk7b(eCqqe))pUx1?_Gj>%Ub Date: Tue, 17 Nov 2026 08:26:11 -0600 Subject: [PATCH 2/7] C#: one copy of every vector, parallel batch search, optional binary snapshot VectorTable holds each vector once in 1024-row chunks; the IVF index keeps only cell lists and follows table compaction; pending rows fold at write time. Searches share a ReaderWriterLockSlim and run concurrently; SimilaritySearchBatch runs queries in parallel with results bit-identical to the single-query path. Optional .embeddings.snap per Rust DESIGN.md section 11 (reproduces the Rust golden snapshot byte for byte). SaveEmbeddings streams instead of building one giant string. Fixes the competitive bench's recall_before 0.000 (a bench bug: reference recall unset when the sweep never reached 0.95). Breaking: Get/GetEmbedding return a copy of the vector, Add copies the caller's array, AnnStats.VectorBytes is 0, pending rows fold below 256 at write time. Co-Authored-By: Claude Sonnet 5.5 --- .../bench/IsonGraph.Vector.Bench/Program.cs | 4 +- .../Program.cs | 15 +- .../IsonGraph.Vector/EmbeddingStore.Ann.cs | 244 +++-- .../src/IsonGraph.Vector/EmbeddingStore.cs | 930 ++++++++++++++++++ .../src/IsonGraph.Vector/GraphDigest.cs | 21 +- .../src/IsonGraph.Vector/IsonGraphVector.cs | 739 ++------------ .../src/IsonGraph.Vector/IvfIndex.cs | 704 ++++++------- .../src/IsonGraph.Vector/Scan.cs | 363 +++++++ .../src/IsonGraph.Vector/VectorSnapshot.cs | 499 ++++++++++ .../src/IsonGraph.Vector/VectorTable.cs | 325 ++++++ .../tests/IsonGraph.Vector.Tests/AnnTests.cs | 10 +- .../ReadOnlyGraphTests.cs | 3 +- .../SemanticGraphTests.cs | 16 +- .../IsonGraph.Vector.Tests/StageDTests.cs | 834 ++++++++++++++++ 14 files changed, 3577 insertions(+), 1130 deletions(-) create mode 100644 isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.cs create mode 100644 isongraph-vector-csharp/src/IsonGraph.Vector/Scan.cs create mode 100644 isongraph-vector-csharp/src/IsonGraph.Vector/VectorSnapshot.cs create mode 100644 isongraph-vector-csharp/src/IsonGraph.Vector/VectorTable.cs create mode 100644 isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/StageDTests.cs diff --git a/isongraph-vector-csharp/bench/IsonGraph.Vector.Bench/Program.cs b/isongraph-vector-csharp/bench/IsonGraph.Vector.Bench/Program.cs index ffa7a9b..fdfd559 100644 --- a/isongraph-vector-csharp/bench/IsonGraph.Vector.Bench/Program.cs +++ b/isongraph-vector-csharp/bench/IsonGraph.Vector.Bench/Program.cs @@ -58,8 +58,8 @@ static void RunOne(int n, string regimeName, Options o, Action say) say(""); say($"* exact scan: warm median {Pct(exactMs, 0.5):F2} ms, p95 {Pct(exactMs, 0.95):F2} ms"); say($"* index build: {stats.BuildSeconds:F2} s for nlist={stats.Nlist} (automatic nprobe={stats.Nprobe}); " + - $"cells {stats.IndexBytes / 1048576.0:F1} MB plus {stats.VectorBytes / 1048576.0:F0} MB for the index's " + - $"contiguous copy of the vectors (the store's own {(long)n * o.Dim * 4 / 1048576} MB is separate); " + + $"index {stats.IndexBytes / 1048576.0:F1} MB (centroids and cell lists only: it keeps no copy of the " + + $"vectors, the store's {(long)n * o.Dim * 4 / 1048576} MB matrix is the only one); " + $"managed heap {gcAfter / 1048576} MB"); say($"* ingest {ingest:N0} vectors/s (the index is built lazily; ingest is unaffected)"); say(""); diff --git a/isongraph-vector-csharp/bench/IsonGraph.Vector.CompetitiveBench/Program.cs b/isongraph-vector-csharp/bench/IsonGraph.Vector.CompetitiveBench/Program.cs index 016c9e1..7a36fb5 100644 --- a/isongraph-vector-csharp/bench/IsonGraph.Vector.CompetitiveBench/Program.cs +++ b/isongraph-vector-csharp/bench/IsonGraph.Vector.CompetitiveBench/Program.cs @@ -202,7 +202,9 @@ double Ingest(EmbeddingStore s, byte[]? grp) if (refKnob < 0 && (double)p["recall"]! >= 0.95) { refKnob = kn; refRecall = (double)p["recall"]!; } pts.Add(p); } - if (refKnob < 0) refKnob = Knobs().Last(); + // No knob reached 0.95: use the widest and report ITS recall (the mutation section's "before" figure + // used to print 0.000 here because refRecall was only ever assigned when a knob reached 0.95). + if (refKnob < 0) { refKnob = Knobs().Last(); refRecall = (double)pts[^1]["recall"]!; } iv["sweep"] = pts; iv["ref_knob"] = refKnob; iv["ref_recall_pre"] = refRecall; iv["mem_after_sweep"] = Mem(); if (exps.Contains("mutation")) @@ -259,6 +261,17 @@ double Ingest(EmbeddingStore s, byte[]? grp) Search(s2, queries[0], false, null); double fq = (Now() - t) * 1000; sl["first_query_after_load_ms"] = fq; sl["time_to_ready_s"] = loadS + fq / 1000; + // the opt-in binary snapshot of the same store (DESIGN.md section 11) + var snap = System.IO.Path.ChangeExtension(path, ".snap"); + t = Now(); + st.SaveSnapshot(snap); + sl["snap_save_s"] = Now() - t; + sl["snap_mb"] = new FileInfo(snap).Length / 1048576.0; + var s3 = new EmbeddingStore(ann: AnnMode.Ivf, annMinSize: 0, annNprobe: refKnob); + t = Now(); + s3.LoadSnapshot(snap); + sl["snap_load_s"] = Now() - t; + File.Delete(snap); File.Delete(path); } catch (Exception e) { sl["error"] = e.Message; } diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.Ann.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.Ann.cs index 2a4a2d0..c57f3b0 100644 --- a/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.Ann.cs +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.Ann.cs @@ -1,65 +1,83 @@ namespace IsonGraph.Vector; // Opt-in approximate (IVF) search for the in-memory store. The exact scan in -// IsonGraphVector.cs is untouched; everything here runs under the same -// coarse lock as every other store operation. +// EmbeddingStore.cs is untouched; the index is built and retrained under the +// store's exclusive lock and read under the shared one. public sealed partial class EmbeddingStore { - private const int AnnFoldMin = 512; // fold pending rows into cells at >= this many ... - private const double AnnFoldFraction = 0.02; // ... and at least this fraction of the indexed rows private const double AnnRetrainFraction = 0.5; // retrain once writes+deletes exceed this x trained size private static int AnnAutoNlist(int n) => (int)Math.Min(Math.Max(Math.Round(4 * Math.Sqrt(n)), 16), 65536); + // Caller holds a lock. + private bool AnnBelowMin() => (_table?.Live ?? 0) < Math.Max(2, _annMinSize); + + /// + /// Whether a search wanting the index can proceed under the shared lock: the + /// index exists and is current, or the store is under the minimum size and + /// there is none (so the answer is exact). Caller holds a lock. + /// + private bool IndexReady() + { + if (_ann == AnnMode.None) return true; + if (AnnBelowMin()) return _ivf is null; + return _ivf is not null && !_ivf.Stale; + } + /// - /// The index, built or maintained as the policy says; null means "answer - /// exactly". Caller holds the lock. + /// Build, retrain or drop the index as the policy says. Caller holds the + /// exclusive lock. /// /// Policy: built lazily by the first qualifying search (never persisted), /// k-means on a sample of up to 40 x nlist vectors, then every row - /// assigned. New and updated rows are scored exactly until at least - /// max(512, 2%) of the indexed rows are pending, then folded into their - /// nearest cell. Deletes just drop the row. Once writes + deletes since + /// assigned. New and updated rows are scored exactly until 256 are + /// pending, when the write that completes them folds them into their + /// nearest cell. Deletes detach the row. Once writes + deletes since /// training exceed 50% of the trained size the index is retrained from the /// current data (with an automatic nlist, re-chosen for the new size). A /// store that shrinks below the minimum size, or is cleared, drops it. /// - private IvfIndex? AnnIndexForSearch() + private void EnsureIndex() { - if (_ann == AnnMode.None) return null; - int live = _embeddings.Count; - if (live < Math.Max(2, _annMinSize)) + if (_ann == AnnMode.None) return; + if (AnnBelowMin()) { _ivf = null; - return null; - } - if (_ivf is null) - { - _ivf = IvfIndex.FromRecords(_embeddings.Values, _dimension!.Value, live); - TrainIndex(_ivf); - } - else if (_ivf.Churn > AnnRetrainFraction * _ivf.TrainedN) - { - TrainIndex(_ivf); + return; } - else if (_ivf.PendingCount >= Math.Max(AnnFoldMin, AnnFoldFraction * _ivf.TrainedN)) + if (_ivf is null || _ivf.Stale) { - _ivf.Fold(); + var table = _table!; + int nlist = _annNlist ?? AnnAutoNlist(table.Live); + _ivf = IvfIndex.Build(table, nlist, _annSeed); + Interlocked.Increment(ref _annBuilds); } - return _ivf; } - private void TrainIndex(IvfIndex ivf) + /// + /// Take the shared lock with the index ready if : + /// when it has to be built or retrained, the exclusive lock is taken for + /// that (searches wait) and the shared lock is then re-acquired. The caller + /// releases with _rw.ExitReadLock(). + /// + private void EnterReadReady(bool wantIndex) { - int nlist = _annNlist ?? AnnAutoNlist(ivf.LiveCount); - ivf.Train(nlist, _annSeed); - _annBuilds++; + for (;;) + { + _rw.EnterReadLock(); + if (!wantIndex || IndexReady()) return; + _rw.ExitReadLock(); + _rw.EnterWriteLock(); + try { EnsureIndex(); } + finally { _rw.ExitWriteLock(); } + } } private int AnnProbe(IvfIndex ivf) { - int wanted = _annNprobe ?? Math.Max(1, (int)Math.Round(0.05 * ivf.Nlist)); + int configured = Volatile.Read(ref _annNprobe); + int wanted = configured != 0 ? configured : Math.Max(1, (int)Math.Round(0.05 * ivf.Nlist)); return Math.Min(wanted, ivf.Nlist); } @@ -79,27 +97,38 @@ private int AnnProbe(IvfIndex ivf) /// /// Top-k from the index, or null to fall back to the exact scan. Caller - /// holds the lock. Filters apply to the probed rows; if fewer than topK - /// survive, the probe is widened 4x and tried again, ending in the exact - /// scan, so a rare type or a high threshold costs time rather than - /// silently returning a short list. + /// holds the shared lock with the index ready. Filters apply to the probed + /// rows; if fewer than topK survive, the probe is widened 4x and tried + /// again, ending in the exact scan, so a rare type or a high threshold + /// costs time rather than silently returning a short list. /// private IReadOnlyList? AnnSearch( float[] queryVector, int topK, string? nodeType, float threshold) { - var ivf = AnnIndexForSearch(); - if (ivf is null) return null; + var ivf = _ivf; + var table = _table; + if (ivf is null || table is null) return null; + float qnorm = QueryNorm(queryVector); + if (qnorm == 0f) return null; // a zero query has no direction to rank cells by + int code = -1; + if (nodeType is not null) + { + code = table.CodeOf(nodeType); + if (code < 0) return null; // no row has this type: the exact scan says so + } + int k = Math.Min(topK, table.Live); for (long scale = 1; ; scale *= 4) { var lists = AnnLists(ivf, scale); if (lists is null) return null; - if (!ivf.Scan(queryVector, nodeType, threshold, lists.Value, out int passing)) - return null; // a zero query has no direction to rank cells by - if (passing >= topK) + var heap = new TopKHeap(k); + var visitor = new TopKVisitor { Heap = heap, Norms = table.Norms, QueryNorm = qnorm, Threshold = threshold }; + ivf.Scan(table, queryVector, code, lists.Value, ref visitor); + if (heap.Count >= topK) { - _annApproximate++; - if (scale > 1) _annWidened++; - return ivf.TopK(topK); + Interlocked.Increment(ref _annApproximate); + if (scale > 1) Interlocked.Increment(ref _annWidened); + return ToResults(table, heap.ToSortedArray()); } } } @@ -115,20 +144,52 @@ private int AnnProbe(IvfIndex ivf) internal (IEnumerable Ranked, bool Approximate) RankedForGate( float[] queryVector, string? nodeType, float threshold, long scale) { - lock (_lock) + EnterReadReady(_ann != AnnMode.None); + try { CheckQuery(queryVector); - var ivf = AnnIndexForSearch(); - var lists = ivf is null ? null : AnnLists(ivf, scale); - if (ivf is not null && lists is not null && - ivf.Scan(queryVector, nodeType, threshold, lists.Value, out _)) + var ivf = _ivf; + var table = _table; + if (ivf is not null && table is not null) { - _annApproximate++; - if (scale > 1) _annWidened++; - return (ivf.RankedSnapshot(), true); + var lists = AnnLists(ivf, scale); + float qnorm = QueryNorm(queryVector); + int code = nodeType is null ? -1 : table.CodeOf(nodeType); + if (lists is not null && qnorm != 0f && !(nodeType is not null && code < 0)) + { + var found = new HitList(); + var visitor = new CollectVisitor { List = found, Norms = table.Norms, QueryNorm = qnorm, Threshold = threshold }; + ivf.Scan(table, queryVector, code, lists.Value, ref visitor); + Interlocked.Increment(ref _annApproximate); + if (scale > 1) Interlocked.Increment(ref _annWidened); + return (Snapshot(table, found.ToSortedArray()), true); + } } - return (SimilaritySearchVector(queryVector, null, nodeType, threshold, exact: true), false); + Interlocked.Increment(ref _annExact); + return (ExactSearch(queryVector, null, nodeType, threshold, ScanThreads), false); } + finally { _rw.ExitReadLock(); } + } + + /// The hits as a lazily built sequence that no longer touches the table. + private static IEnumerable Snapshot(VectorTable t, Hit[] hits) + { + var refs = new IsonGraph.NodeRef[hits.Length]; + var texts = new string[hits.Length]; + var scores = new float[hits.Length]; + for (int i = 0; i < hits.Length; i++) + { + refs[i] = t.RefAt(hits[i].Slot); + texts[i] = t.TextAt(hits[i].Slot); + scores[i] = hits[i].Score; + } + return Enumerate(refs, texts, scores); + } + + private static IEnumerable Enumerate(IsonGraph.NodeRef[] refs, string[] texts, float[] scores) + { + for (int i = 0; i < refs.Length; i++) + yield return new SimilarityResult(refs[i], scores[i], texts[i]); } /// Resolve a text query to its vector (encoder, model check and dimension check). @@ -146,34 +207,73 @@ public AnnStats BuildAnnIndex(bool force = false) if (_ann == AnnMode.None) throw new EmbeddingException( "this store has no approximate index; create it with ann: AnnMode.Ivf"); - lock (_lock) + _rw.EnterWriteLock(); + try { if (force) _ivf = null; - AnnIndexForSearch()?.Fold(); - return GetAnnStats(); + EnsureIndex(); + if (_ivf is not null && _table is not null) _ivf.Fold(_table); + return StatsLocked(); } + finally { _rw.ExitWriteLock(); } } /// What the approximate index looks like right now. Never builds anything. public AnnStats GetAnnStats() { - lock (_lock) + _rw.EnterReadLock(); + try { return StatsLocked(); } + finally { _rw.ExitReadLock(); } + } + + private AnnStats StatsLocked() + { + var ivf = _ivf; + int configured = Volatile.Read(ref _annNprobe); + return new AnnStats( + Enabled: _ann != AnnMode.None, + Mode: _ann, + Active: ivf is not null, + Nlist: ivf?.Nlist, + Nprobe: ivf is not null ? AnnProbe(ivf) : (configured == 0 ? null : configured), + TrainedN: ivf?.TrainedN, + Pending: ivf?.PendingCount ?? 0, + Churn: ivf?.Churn ?? 0, + BuildSeconds: ivf?.BuildSeconds, + IndexBytes: ivf?.IndexBytes ?? 0, + VectorBytes: ivf?.VectorBytes ?? 0, + MinSize: _annMinSize, + Searches: new AnnSearchCounts( + Interlocked.Read(ref _annApproximate), Interlocked.Read(ref _annExact), + Interlocked.Read(ref _annWidened), Interlocked.Read(ref _annBuilds))); + } + + /// + /// Check the store's invariants (DESIGN.md section 7: table, model counts, + /// index lists and positions). Returns null when they all hold, else a + /// description of the first one that does not. For tests and debugging; + /// O(n). + /// + internal string? VerifyInvariants() + { + _rw.EnterReadLock(); + try { - var ivf = _ivf; - return new AnnStats( - Enabled: _ann != AnnMode.None, - Mode: _ann, - Active: ivf is not null, - Nlist: ivf?.Nlist, - Nprobe: ivf is not null ? AnnProbe(ivf) : _annNprobe, - TrainedN: ivf?.TrainedN, - Pending: ivf?.PendingCount ?? 0, - Churn: ivf?.Churn ?? 0, - BuildSeconds: ivf?.BuildSeconds, - IndexBytes: ivf?.IndexBytes ?? 0, - VectorBytes: ivf?.VectorBytes ?? 0, - MinSize: _annMinSize, - Searches: new AnnSearchCounts(_annApproximate, _annExact, _annWidened, _annBuilds)); + var t = _table; + if (t is null) return _ivf is null ? null : "index without a table"; + return t.Verify() ?? _ivf?.Verify(t); + } + finally { _rw.ExitReadLock(); } + } + + /// Bytes held by the vector matrix chunks (0 while empty); what the vectors cost, one copy. + internal long MatrixBytes + { + get + { + _rw.EnterReadLock(); + try { return _table?.MatrixBytes ?? 0; } + finally { _rw.ExitReadLock(); } } } @@ -226,7 +326,7 @@ public RecallReport RecallCheck( exactTimes.Add(System.Diagnostics.Stopwatch.GetElapsedTime(t0).TotalMilliseconds); if (truth.Count > 0) { - var wanted = new HashSet(truth.Select(r => r.NodeRef)); + var wanted = new HashSet(truth.Select(r => r.NodeRef)); int hit = found.Count(r => wanted.Contains(r.NodeRef)); recalls.Add((double)hit / wanted.Count); } diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.cs new file mode 100644 index 0000000..c607863 --- /dev/null +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/EmbeddingStore.cs @@ -0,0 +1,930 @@ +using System.Globalization; +using System.Text.Json; +using IsonGraph; + +namespace IsonGraph.Vector; + +// ============================================================================= +// Embedding Store +// ============================================================================= + +/// +/// In-memory embedding storage with cosine similarity search. +/// +/// Storage. Every vector is kept once, in a chunked contiguous matrix +/// (1024 rows per chunk); a record is a row number plus its key, text and model +/// (see DESIGN.md section 1). Nothing else owns the floats: the exact scan, the +/// approximate index and the save paths all read the same rows. +/// +/// Concurrency contract. Searches (SimilaritySearch*, +/// , , , exports and +/// saves) take a shared lock and touch no shared mutable state, so any number +/// run at the same time and a batch sees one state for all its queries. +/// Writes (Add*, , , imports, +/// ) take the exclusive lock: a search sees the store +/// before a write or after it, never in between. The one thing a search can do +/// besides read is build or retrain the approximate index, which it does by +/// taking the exclusive lock first (so concurrent searches wait for the +/// build). A write that makes 256 rows pending folds them into the index while +/// it holds the exclusive lock (about 1-3 ms at 1M x 384). +/// +public sealed partial class EmbeddingStore +{ + private readonly ReaderWriterLockSlim _rw = new(LockRecursionPolicy.NoRecursion); + private VectorTable? _table; + private IEmbeddingEncoder? _encoder; + private int _dimension = -1; // pinned vector length, -1 while none + private string? _version; + private ModelMismatchMode _onModelMismatch; + private readonly HashSet _warnedMismatch = new(); + private volatile IReadOnlyDictionary _lastImportMeta = new Dictionary(); + private int _scanThreads; + + // Opt-in approximate index (see AnnMode). _ivf is built lazily by the first + // search that qualifies, kept current by Add/Remove/Clear, and never saved. + private readonly AnnMode _ann; + private readonly int? _annNlist; + private int _annNprobe; // 0 = automatic + private readonly int _annMinSize; + private readonly int _annSeed; + private IvfIndex? _ivf; + private long _annApproximate, _annExact, _annWidened, _annBuilds; + + /// Below this many vectors the index is not used by default. + public const int DefaultAnnMinSize = 50_000; + + /// Encoder used when a text is added or queried without a vector. + /// + /// Free-form version of the encoder/model that made these vectors, + /// recorded in the sidecar. Falls back to the encoder's own Version. + /// + /// + /// What a text query does when the encoder's ModelName differs from the + /// model that produced the stored vectors. Vectors whose model is + /// "unknown"/"provided" never trigger it, and vector queries are never + /// checked. + /// + /// + /// (default) keeps search exact, exactly as it + /// always was. answers bounded top-k searches + /// from an in-memory IVF-flat index: vectors are clustered into + /// cells and only the + /// cells nearest the query are scored. It is approximate - a true neighbour + /// in an unprobed cell is missed - so measure it with + /// . The + /// index is built lazily by the first search, never persisted, and kept + /// current through writes and deletes. It holds no copy of the vectors + /// (only centroids and a cell number per row). + /// + /// Cells in the index. Null picks about 4 * sqrt(N) when it is built. + /// + /// Cells scanned per query. Null picks about 5% of nlist. More cells = higher + /// recall, slower; nprobe >= nlist is the exact scan. Adjustable later via + /// . + /// + /// Below this many vectors search stays exact. Default 50,000. + /// Seed for the k-means initialisation, so the same data gives the same index. + public EmbeddingStore( + IEmbeddingEncoder? encoder = null, + string? version = null, + ModelMismatchMode onModelMismatch = ModelMismatchMode.Warn, + AnnMode ann = AnnMode.None, + int? annNlist = null, + int? annNprobe = null, + int annMinSize = DefaultAnnMinSize, + int annSeed = 0) + { + CheckMode(onModelMismatch); + if (!Enum.IsDefined(ann)) + throw new ArgumentOutOfRangeException(nameof(ann), ann, "unknown AnnMode"); + if (annNlist is < 1) + throw new ArgumentOutOfRangeException(nameof(annNlist), annNlist, "annNlist must be >= 1"); + if (annNprobe is < 1) + throw new ArgumentOutOfRangeException(nameof(annNprobe), annNprobe, "annNprobe must be >= 1"); + if (annMinSize < 0) + throw new ArgumentOutOfRangeException(nameof(annMinSize), annMinSize, "annMinSize must be >= 0"); + _encoder = encoder; + _version = version; + _onModelMismatch = onModelMismatch; + _ann = ann; + _annNlist = annNlist; + _annNprobe = annNprobe ?? 0; + _annMinSize = annMinSize; + _annSeed = annSeed; + } + + /// The approximate index in use, or for exact search. + public AnnMode Ann => _ann; + + /// Cells scanned per query; null means automatic (about 5% of nlist). + public int? AnnNprobe + { + get { int v = Volatile.Read(ref _annNprobe); return v == 0 ? null : v; } + set + { + if (value is < 1) + throw new ArgumentOutOfRangeException(nameof(value), value, "AnnNprobe must be >= 1"); + Volatile.Write(ref _annNprobe, value ?? 0); + } + } + + /// + /// Threads one exact scan may use: 0 (default) picks automatically (up to 8, + /// fewer for small stores), 1 never runs a scan in parallel. The answer is + /// the same for any value; only the time changes. + /// + public int ScanThreads + { + get => Volatile.Read(ref _scanThreads); + set + { + if (value < 0) throw new ArgumentOutOfRangeException(nameof(value), value, "ScanThreads must be >= 0"); + Volatile.Write(ref _scanThreads, value); + } + } + + private static void CheckMode(ModelMismatchMode mode) + { + if (!Enum.IsDefined(mode)) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "unknown ModelMismatchMode"); + } + + /// + /// Raised with a human-readable message when the store has something to + /// warn about (a model mismatch, a version mismatch on load). With no + /// subscriber the message goes to standard error. + /// + public event Action? Warning; + + internal void RaiseWarning(string message) + { + var handler = Warning; + if (handler is not null) handler(message); + else Console.Error.WriteLine("isongraph-vector: warning: " + message); + } + + public IEmbeddingEncoder? Encoder + { + get => _encoder; + set => _encoder = value; + } + + /// + /// Encoder/model version recorded in the sidecar: the one set explicitly, + /// else the encoder's own, else null. + /// + public string? Version + { + get => _version ?? _encoder?.Version; + set => _version = value; + } + + /// What a text query does when the encoder's model differs from the stored vectors'. + public ModelMismatchMode OnModelMismatch + { + get => _onModelMismatch; + set { CheckMode(value); _onModelMismatch = value; } + } + + /// + /// Metadata (encoder_model, encoder_version, graph_digest, + /// graph_digest_basis) of the most recently imported payload; only the + /// keys it carried are present, and it is empty before any import. + /// + public IReadOnlyDictionary LastImportMeta => _lastImportMeta; + + /// Vector length this store holds, or null while it is empty. + public int? Dimension + { + get + { + _rw.EnterReadLock(); + try { return _dimension < 0 ? null : _dimension; } + finally { _rw.ExitReadLock(); } + } + } + + private static string Key(NodeRef nodeRef) => $"{nodeRef.Type}:{nodeRef.Id}"; + + /// + /// Keep the best results without ordering all of them. + /// + /// Sorting every candidate to return ten is O(n log n) where this is + /// O(n·k) with early rejection — for the small k a search actually asks + /// for, that is a large saving over 20,000 candidates. Insertion goes + /// after equal scores, so the order matches a stable descending sort. + /// + internal static List SelectTopK(List results, int k) + { + if (k <= 0) return new List(); + if (results.Count <= k) + return results.OrderByDescending(r => r.Score).ToList(); + + var top = new List(k); + foreach (var candidate in results) + { + if (top.Count == k && candidate.Score <= top[k - 1].Score) continue; + + int i = top.Count < k ? top.Count : k - 1; + if (top.Count < k) top.Add(candidate); + while (i > 0 && top[i - 1].Score < candidate.Score) + { + top[i] = top[i - 1]; + i--; + } + top[i] = candidate; + } + return top; + } + + // Caller holds a lock. + private void CheckDimension(ReadOnlySpan vector, string what) + { + if (vector.Length == 0) + throw new DimensionMismatchException($"{what} is empty"); + if (_dimension < 0) + _dimension = vector.Length; + else if (vector.Length != _dimension) + throw new DimensionMismatchException( + $"{what} has {vector.Length} dimensions, store holds {_dimension}"); + } + + // Caller holds a lock. + private void CheckQuery(ReadOnlySpan vector) + { + if (vector.Length == 0) + throw new DimensionMismatchException("query vector is empty"); + if (_dimension >= 0 && vector.Length != _dimension) + throw new DimensionMismatchException( + $"query vector has {vector.Length} dimensions, store holds {_dimension}"); + } + + private float[] ResolveQuery(string query) + { + var encoder = _encoder ?? throw new NoEncoderException("No encoder provided for text query"); + CheckModelCompat(); + var vector = encoder.Encode(query); + _rw.EnterReadLock(); + try { CheckQuery(vector); } + finally { _rw.ExitReadLock(); } + return vector; + } + + // -- model lifecycle ------------------------------------------------------- + + /// Model name stamped on a vector written now. + private string StampModel() => ModelCompat.Stamp(_encoder); + + /// The encoder's model name, or null when it has none worth comparing. + private string? CurrentModel() => ModelCompat.Current(_encoder); + + /// + /// Which models produced the stored vectors: model name to count. + /// "unknown" and "provided" appear as-is (vectors imported from a port + /// that records no model, or handed in with no encoder attached). + /// + public IReadOnlyDictionary Models() + { + _rw.EnterReadLock(); + try { return new Dictionary(_table?.ModelCounts ?? new Dictionary()); } + finally { _rw.ExitReadLock(); } + } + + /// + /// Complain when a text query's encoder is not the stored vectors' model. + /// Called before a text query is encoded; vector queries skip it. + /// + internal void CheckModelCompat() + { + if (_onModelMismatch == ModelMismatchMode.Ignore) return; + var current = CurrentModel(); + if (current is null) return; + + var foreign = ModelCompat.Foreign(current, Models().Keys); + if (foreign.Count == 0) return; + + var message = ModelCompat.Message(current, foreign); + if (_onModelMismatch == ModelMismatchMode.Error) + throw new ModelMismatchException(message); + + bool first; + lock (_warnedMismatch) first = _warnedMismatch.Add(ModelCompat.WarnKey(current, foreign)); + if (first) RaiseWarning(message); + } + + /// + /// Node refs whose stored vector does not belong to the current encoder. + /// + /// A vector is stale when its recorded model differs from + /// (default: the encoder's ModelName) or its + /// length differs from (default: the + /// encoder's Dimension). Vectors whose model is "unknown"/"provided" + /// cannot be judged by model and are skipped unless + /// is true (a dimension mismatch still + /// counts). With no encoder and no arguments nothing can be judged and the + /// result is empty. + /// + public IReadOnlyList Stale(string? model = null, int? dimension = null, bool includeUnknown = false) + { + _rw.EnterReadLock(); + try + { + model ??= CurrentModel(); + dimension ??= _encoder?.Dimension; + + var stale = new List(); + var t = _table; + if (t is null) return stale; + for (int s = 0; s < t.Slots; s++) + { + if (!t.IsLive(s)) continue; + if (ModelCompat.IsStale(t.ModelAt(s), t.Dim, model, dimension, includeUnknown)) + stale.Add(t.RefAt(s)); + } + return stale; + } + finally { _rw.ExitReadLock(); } + } + + // -- writes ----------------------------------------------------------------- + + /// + /// Store a vector for a node, copying it into the store (later changes to + /// do not reach the store). Encodes + /// first, outside the lock, when no vector is given. + /// + public string Add(NodeRef nodeRef, string text, float[]? vector = null) + { + if (vector is null) + { + var encoder = _encoder ?? + throw new NoEncoderException("No encoder provided and no vector given"); + vector = encoder.Encode(text); + } + _rw.EnterWriteLock(); + try + { + CheckDimension(vector, "vector"); + return PutLocked(nodeRef, text, vector, StampModel()); + } + finally { _rw.ExitWriteLock(); } + } + + /// Store a vector under an explicit model name. Caller holds the write lock and has checked the dimension. + private string PutLocked(NodeRef nodeRef, string text, ReadOnlySpan vector, string model) + { + var key = Key(nodeRef); + var table = _table ??= new VectorTable(vector.Length); + int slot = table.Put(nodeRef, key, text, vector, model, out _); + if (_ivf is not null && _ivf.NotePut(slot)) _ivf.Fold(table); + return key; + } + + /// Copy a record in, keeping the model it was made by. + internal string AddRecord(EmbeddingRecord record) + { + _rw.EnterWriteLock(); + try + { + CheckDimension(record.Vector, "vector"); + return PutLocked(record.NodeRef, record.Text, record.Vector, record.Model); + } + finally { _rw.ExitWriteLock(); } + } + + /// Add many embeddings, encoding their texts in one batched call. + public int AddBatch(IReadOnlyList<(NodeRef NodeRef, string Text)> entries) + { + if (entries.Count == 0) return 0; + if (_encoder is null) + throw new NoEncoderException("No encoder provided and no vectors given"); + + var vectors = _encoder.EncodeBatch(entries.Select(e => e.Text).ToList()); + if (vectors.Count != entries.Count) + throw new EmbeddingException( + $"encoder returned {vectors.Count} vectors for {entries.Count} texts"); + + for (int i = 0; i < entries.Count; i++) + Add(entries[i].NodeRef, entries[i].Text, vectors[i]); + return entries.Count; + } + + /// + /// The record of a node. The vector is a copy (the store keeps its + /// one copy in a shared matrix), so changing it does not change the store. + /// + public EmbeddingRecord? Get(NodeRef nodeRef) + { + _rw.EnterReadLock(); + try + { + var t = _table; + if (t is null || !t.TryGetSlot(Key(nodeRef), out int slot)) return null; + return new EmbeddingRecord(t.KeyAt(slot), t.RefAt(slot), t.Row(slot).ToArray(), t.TextAt(slot), t.ModelAt(slot)); + } + finally { _rw.ExitReadLock(); } + } + + public bool Remove(NodeRef nodeRef) + { + _rw.EnterWriteLock(); + try + { + var t = _table; + if (t is null || !t.Remove(Key(nodeRef), out int slot)) return false; + _ivf?.NoteDrop(slot); + if (t.ShouldCompact) CompactLocked(); + if (_ivf is not null && t.Live < Math.Max(2, _annMinSize)) _ivf = null; + return true; + } + finally { _rw.ExitWriteLock(); } + } + + // Caller holds the write lock. + private void CompactLocked() + { + var t = _table!; + var remap = t.Compact(); + _ivf?.Compact(remap, t.Slots); + } + + /// + /// Squeeze the tombstones out of the matrix now. It happens by itself once + /// a quarter of the rows are dead; order is preserved, so results do not + /// change. A pause proportional to the store's size. + /// + public void Compact() + { + _rw.EnterWriteLock(); + try { if (_table is { Dead: > 0 }) CompactLocked(); } + finally { _rw.ExitWriteLock(); } + } + + public int Count + { + get + { + _rw.EnterReadLock(); + try { return _table?.Live ?? 0; } + finally { _rw.ExitReadLock(); } + } + } + + /// + /// Clear all embeddings. + /// + /// + /// Also forget the pinned vector length, so the next vector may have a + /// different one (switching to an encoder of a different dimension). True + /// by default, which is how this method has always behaved; pass false to + /// keep the store refusing vectors of the wrong length even when empty (the + /// Python port's default). + /// + public void Clear(bool resetDimension = true) + { + _rw.EnterWriteLock(); + try { ClearLocked(resetDimension); } + finally { _rw.ExitWriteLock(); } + } + + private void ClearLocked(bool resetDimension) + { + _table = null; + _ivf = null; + if (resetDimension) _dimension = -1; + } + + // -- search ----------------------------------------------------------------- + + /// + /// Find the most similar embeddings to a text query. + /// + /// With an approximate index () a bounded + /// on a store of at least the minimum size is answered from the index + /// (scores are the same cosine values, but a true neighbour in an unprobed + /// cell can be missing). forces the full scan. + /// Equal scores are ordered by insertion order of the stored node. + /// + /// Query text. + /// Number of results (null returns every match). + /// Keep only embeddings of this node type. + /// Minimum similarity. + /// Skip the approximate index and scan everything (no effect without one). + public IReadOnlyList SimilaritySearch( + string query, int? topK = 10, string? nodeType = null, float threshold = 0.0f, bool exact = false) + => SimilaritySearchVector(ResolveQuery(query), topK, nodeType, threshold, exact); + + private bool WantsIndex(int? topK, bool exact) => _ann != AnnMode.None && !exact && topK is > 0; + + /// Find the most similar embeddings to a query vector; see . + public IReadOnlyList SimilaritySearchVector( + float[] queryVector, int? topK = 10, string? nodeType = null, float threshold = 0.0f, + bool exact = false) + { + EnterReadReady(WantsIndex(topK, exact)); + try { return SearchCore(queryVector, topK, nodeType, threshold, exact, ScanThreads); } + finally { _rw.ExitReadLock(); } + } + + /// + /// Run many text queries at once; see the vector overload. The texts are + /// encoded in one batched encoder call. + /// + public IReadOnlyList> SimilaritySearchBatch( + IReadOnlyList queries, int? topK = 10, string? nodeType = null, float threshold = 0.0f, + bool exact = false, int maxDegreeOfParallelism = 0) + { + if (queries.Count == 0) return Array.Empty>(); + var encoder = _encoder ?? throw new NoEncoderException("No encoder provided for text query"); + CheckModelCompat(); + var vectors = encoder.EncodeBatch(queries); + if (vectors.Count != queries.Count) + throw new EmbeddingException($"encoder returned {vectors.Count} vectors for {queries.Count} texts"); + return SimilaritySearchBatch(vectors, topK, nodeType, threshold, exact, maxDegreeOfParallelism); + } + + /// + /// Run many queries in parallel: result i is bit-identical (same + /// nodes, same scores, same order) to SimilaritySearchVector(queries[i], ...) + /// at the same moment. The whole batch observes one state of the store + /// (it holds the shared lock throughout, so a concurrent write waits for it). + /// Each query runs on one thread (no nested parallelism); the batch is + /// spread over at most threads + /// (0 = one per logical CPU). + /// + public IReadOnlyList> SimilaritySearchBatch( + IReadOnlyList queries, int? topK = 10, string? nodeType = null, float threshold = 0.0f, + bool exact = false, int maxDegreeOfParallelism = 0) + { + if (maxDegreeOfParallelism < 0) + throw new ArgumentOutOfRangeException(nameof(maxDegreeOfParallelism), maxDegreeOfParallelism, "must be >= 0"); + int n = queries.Count; + if (n == 0) return Array.Empty>(); + + EnterReadReady(WantsIndex(topK, exact)); + try + { + foreach (var q in queries) CheckQuery(q); // fail before any work, deterministically + var results = new IReadOnlyList[n]; + int degree = Math.Min(maxDegreeOfParallelism == 0 ? Environment.ProcessorCount : maxDegreeOfParallelism, n); + if (degree <= 1) + { + for (int i = 0; i < n; i++) + results[i] = SearchCore(queries[i], topK, nodeType, threshold, exact, 1); + } + else + { + Parallel.For(0, n, new ParallelOptions { MaxDegreeOfParallelism = degree }, + i => results[i] = SearchCore(queries[i], topK, nodeType, threshold, exact, 1)); + } + return results; + } + finally { _rw.ExitReadLock(); } + } + + /// One search, with a shared lock held and (if wanted) the index ready. + private IReadOnlyList SearchCore( + float[] queryVector, int? topK, string? nodeType, float threshold, bool exact, int maxThreads) + { + CheckQuery(queryVector); + if (_ann != AnnMode.None) + { + if (exact || topK is null || topK <= 0) + Interlocked.Increment(ref _annExact); + else + { + var answer = AnnSearch(queryVector, topK.Value, nodeType, threshold); + if (answer is not null) return answer; + Interlocked.Increment(ref _annExact); + } + } + return ExactSearch(queryVector, topK, nodeType, threshold, maxThreads); + } + + private static float QueryNorm(float[] q) => MathF.Sqrt(RowScan.SumSq(q)); + + private List ToResults(VectorTable t, Hit[] hits) + { + var list = new List(hits.Length); + foreach (var h in hits) list.Add(new SimilarityResult(t.RefAt(h.Slot), h.Score, t.TextAt(h.Slot))); + return list; + } + + private int ScanThreadsFor(VectorTable t, int maxThreads) + { + if (maxThreads == 1) return 1; + long work = (long)t.Slots * t.Dim; + int auto = (int)Math.Min(Math.Min(Environment.ProcessorCount, 8), work >> 21); + return Math.Max(1, maxThreads > 0 ? Math.Min(maxThreads, t.ChunkCount) : auto); + } + + /// The full scan. Caller holds a lock. + private List ExactSearch( + float[] queryVector, int? topK, string? nodeType, float threshold, int maxThreads) + { + var t = _table; + if (t is null || t.Live == 0 || topK is <= 0) return new List(); + int code = -1; + if (nodeType is not null) + { + code = t.CodeOf(nodeType); + if (code < 0) return new List(); // no row has this type + } + float qnorm = QueryNorm(queryVector); + int chunks = t.ChunkCount; + + if (topK is null || topK >= t.Live) + { + var list = new HitList(); + var visitor = new CollectVisitor { List = list, Norms = t.Norms, QueryNorm = qnorm, Threshold = threshold }; + RowScan.Chunks(t, queryVector, code, 0, chunks, ref visitor); + return ToResults(t, list.ToSortedArray()); + } + + int k = topK.Value; + int threads = ScanThreadsFor(t, maxThreads); + if (threads <= 1) + { + var heap = new TopKHeap(k); + var visitor = new TopKVisitor { Heap = heap, Norms = t.Norms, QueryNorm = qnorm, Threshold = threshold }; + RowScan.Chunks(t, queryVector, code, 0, chunks, ref visitor); + return ToResults(t, heap.ToSortedArray()); + } + + var merged = new TopKHeap(k); + var norms = t.Norms; + Parallel.For(0, chunks, new ParallelOptions { MaxDegreeOfParallelism = threads }, + () => new TopKHeap(k), + (c, _, heap) => + { + var visitor = new TopKVisitor { Heap = heap, Norms = norms, QueryNorm = qnorm, Threshold = threshold }; + RowScan.Chunks(t, queryVector, code, c, c + 1, ref visitor); + return heap; + }, + heap => { lock (merged) merged.MergeFrom(heap); }); + return ToResults(t, merged.ToSortedArray()); + } + + /// + /// Cosine score for every embedded node, keyed by "type:id". + /// + /// One scan serving both seed selection and the per-node relevance that + /// SemanticMultiHop blends in, instead of two. + /// + public IReadOnlyDictionary ScoreMap(float[] queryVector, string? nodeType = null) + { + _rw.EnterReadLock(); + try + { + CheckQuery(queryVector); + var scores = new Dictionary(); + var t = _table; + if (t is null) return scores; + int code = -1; + if (nodeType is not null) + { + code = t.CodeOf(nodeType); + if (code < 0) return scores; + } + var all = new float[t.Slots]; + var visitor = new ScoreAllVisitor { Scores = all, Norms = t.Norms, QueryNorm = QueryNorm(queryVector) }; + RowScan.Chunks(t, queryVector, code, 0, t.ChunkCount, ref visitor); + for (int s = 0; s < t.Slots; s++) + { + if (!t.IsLive(s)) continue; + if (code >= 0 && t.Codes[s] != code) continue; + scores[t.KeyAt(s)] = all[s]; + } + return scores; + } + finally { _rw.ExitReadLock(); } + } + + // -- persistence (JSON sidecar) ------------------------------------------------ + + /// Live slots in the sidecar's order: node type (default string comparer), then id (ordinal). + private int[] SortedLiveSlots(VectorTable t) + { + var slots = new int[t.Live]; + for (int s = 0, j = 0; s < t.Slots; s++) + if (t.IsLive(s)) slots[j++] = s; + + // rank the (few) distinct types once instead of comparing strings per row + var names = t.TypeNames.ToArray(); + var order = Enumerable.Range(0, names.Length).ToArray(); + Array.Sort(order, (a, b) => Comparer.Default.Compare(names[a], names[b])); + var rank = new int[names.Length]; + for (int i = 0, r = 0; i < order.Length; i++) + { + if (i > 0 && Comparer.Default.Compare(names[order[i - 1]], names[order[i]]) != 0) r++; + rank[order[i]] = r; + } + var codes = t.Codes; + Array.Sort(slots, (a, b) => + { + int c = rank[codes[a]].CompareTo(rank[codes[b]]); + return c != 0 ? c : string.CompareOrdinal(t.RefAt(a).Id, t.RefAt(b).Id); + }); + return slots; + } + + /// + /// Serialize every embedding into the portable payload. This copies every + /// vector (the store keeps its one copy in a shared matrix); for a large + /// store prefer , which streams to disk. + /// + /// Besides the embeddings it carries encoder_model (the encoder's + /// ModelName) and encoder_version when known and, when + /// is given, graph_digest and its + /// basis. Absent values are omitted, not written as null. + /// + public ExportedEmbeddings ExportEmbeddings(string? graphDigest = null) + { + _rw.EnterReadLock(); + try + { + var payload = new ExportedEmbeddings { Dimension = _dimension < 0 ? null : _dimension }; + var t = _table; + if (t is not null) + { + foreach (int slot in SortedLiveSlots(t)) + { + var r = t.RefAt(slot); + payload.Embeddings.Add(new ExportedEmbedding + { + Type = r.Type, + // C# node ids are always strings; the other ports use this + // to tell a numeric id from a numeric-looking string one. + Id = JsonSerializer.SerializeToElement(r.Id), + IdType = "str", + Text = t.TextAt(slot), + Model = t.ModelAt(slot), + Vector = t.Row(slot).ToArray(), + }); + } + } + payload.Count = payload.Embeddings.Count; + payload.EncoderModel = CurrentModel(); + payload.EncoderVersion = Version; + if (graphDigest is not null) + { + payload.GraphDigest = graphDigest; + payload.GraphDigestBasis = IsonGraphVectorVersion.GraphDigestBasis; + } + return payload; + } + finally { _rw.ExitReadLock(); } + } + + /// + /// Load a payload produced by ExportEmbeddings (in any port). Fields this + /// build does not know are ignored; the metadata it does know is exposed as + /// . + /// + public int ImportEmbeddings(ExportedEmbeddings payload, bool replace = false) + { + if (payload.Format != IsonGraphVectorVersion.EmbeddingFormat) + throw new EmbeddingException( + $"not an {IsonGraphVectorVersion.EmbeddingFormat} payload: format={payload.Format}"); + if (payload.Version != IsonGraphVectorVersion.EmbeddingFormatVersion) + throw new EmbeddingException( + $"unsupported {IsonGraphVectorVersion.EmbeddingFormat} version {payload.Version} " + + $"(this build reads version {IsonGraphVectorVersion.EmbeddingFormatVersion})"); + + _rw.EnterWriteLock(); + try + { + var meta = new Dictionary(); + if (payload.EncoderModel is not null) meta["encoder_model"] = payload.EncoderModel; + if (payload.EncoderVersion is not null) meta["encoder_version"] = payload.EncoderVersion; + if (payload.GraphDigest is not null) meta["graph_digest"] = payload.GraphDigest; + if (payload.GraphDigestBasis is not null) meta["graph_digest_basis"] = payload.GraphDigestBasis; + _lastImportMeta = meta; + + if (replace) + { + ClearLocked(resetDimension: true); + _dimension = payload.Dimension ?? -1; + } + + int written = 0; + foreach (var item in payload.Embeddings) + { + var nodeRef = new NodeRef(item.Type, item.IdAsString()); + CheckDimension(item.Vector, $"embedding for :{item.Type}:{nodeRef.Id}"); + PutLocked(nodeRef, item.Text, item.Vector, item.Model ?? "unknown"); + written++; + } + return written; + } + finally { _rw.ExitWriteLock(); } + } + + /// Serialize the portable payload to a JSON string. + public string ToJson(string? graphDigest = null) => JsonSerializer.Serialize(ExportEmbeddings(graphDigest)); + + /// Load embeddings from a JSON payload written by any port. + public int FromJson(string json, bool replace = false) + { + ExportedEmbeddings? payload; + try { payload = JsonSerializer.Deserialize(json); } + catch (JsonException ex) { throw new EmbeddingException($"unreadable embeddings payload: {ex.Message}"); } + if (payload is null) throw new EmbeddingException("payload did not deserialize"); + return ImportEmbeddings(payload, replace); + } + + /// + /// Write the JSON payload (exactly the bytes JsonSerializer.Serialize(ExportEmbeddings(...)) + /// gives) straight from the stored rows to : no + /// payload, no vector copies and no giant string. Holds the shared lock + /// while it writes. + /// + internal int WriteSidecar(Stream stream, string? graphDigest) + { + _rw.EnterReadLock(); + try + { + var t = _table; + var slots = t is null ? Array.Empty() : SortedLiveSlots(t); + using var w = new Utf8JsonWriter(stream); + w.WriteStartObject(); + w.WriteString("format", IsonGraphVectorVersion.EmbeddingFormat); + w.WriteNumber("version", IsonGraphVectorVersion.EmbeddingFormatVersion); + if (_dimension < 0) w.WriteNull("dimension"); else w.WriteNumber("dimension", _dimension); + w.WriteNumber("count", slots.Length); + w.WriteStartArray("embeddings"); + for (int i = 0; i < slots.Length; i++) + { + int slot = slots[i]; + var r = t!.RefAt(slot); + w.WriteStartObject(); + w.WriteString("type", r.Type); + w.WriteString("id", r.Id); + w.WriteString("id_type", "str"); + w.WriteString("text", t.TextAt(slot)); + w.WriteString("model", t.ModelAt(slot)); + w.WriteStartArray("vector"); + foreach (float f in t.Row(slot)) w.WriteNumberValue(f); + w.WriteEndArray(); + w.WriteEndObject(); + if ((i & 255) == 255) w.Flush(); + } + w.WriteEndArray(); + var model = CurrentModel(); + if (model is not null) w.WriteString("encoder_model", model); + var version = Version; + if (version is not null) w.WriteString("encoder_version", version); + if (graphDigest is not null) + { + w.WriteString("graph_digest", graphDigest); + w.WriteString("graph_digest_basis", IsonGraphVectorVersion.GraphDigestBasis); + } + w.WriteEndObject(); + w.Flush(); + return slots.Length; + } + finally { _rw.ExitReadLock(); } + } + + /// + /// Write every embedding to a JSON sidecar, atomically. Returns the count. + /// + /// The file is written to a temp file and renamed into place, so a crash + /// never leaves a half-written sidecar. ties + /// the sidecar to one particular graph (see ). The + /// rows are streamed from the store to the file (no intermediate payload), + /// holding the shared lock: searches continue, writes wait. + /// + public int SaveEmbeddings(string path, string? graphDigest = null) + { + int count = 0; + AtomicFile.Write(path, stream => count = WriteSidecar(stream, graphDigest)); + return count; + } + + /// Read a JSON sidecar written by (or by any port). + public int LoadEmbeddings(string path, bool replace = false) + { + var payload = ReadPayload(path); + return ImportEmbeddings(payload, replace); + } + + /// Parse a JSON sidecar from disk without building a string of the whole file (a JsonException propagates). + internal static ExportedEmbeddings? DeserializePayload(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan); + return JsonSerializer.Deserialize(stream); + } + + private static ExportedEmbeddings ReadPayload(string path) + { + try + { + return DeserializePayload(path) ?? throw new EmbeddingException("payload did not deserialize"); + } + catch (JsonException ex) + { + throw new EmbeddingException($"unreadable embeddings payload: {ex.Message}"); + } + } +} diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/GraphDigest.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/GraphDigest.cs index f720730..9411c0b 100644 --- a/isongraph-vector-csharp/src/IsonGraph.Vector/GraphDigest.cs +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/GraphDigest.cs @@ -384,7 +384,21 @@ internal static class AtomicFile /// the target. UTF-8 without a BOM, and no line-ending translation, so a /// file's bytes do not depend on the OS that wrote it. /// - public static void WriteText(string path, string text) + public static void WriteText(string path, string text) => + Write(path, stream => + { + var bytes = new UTF8Encoding(false).GetBytes(text); + stream.Write(bytes, 0, bytes.Length); + }); + + /// + /// Run against a temp file beside + /// , flush it to disk and move it over the target: a + /// reader sees the old file or the new one, never a partial one, and a + /// failure (an exception from included) removes the + /// temp file and leaves the target as it was. + /// + public static void Write(string path, Action fill) { var full = System.IO.Path.GetFullPath(path); var dir = System.IO.Path.GetDirectoryName(full) ?? "."; @@ -392,10 +406,9 @@ public static void WriteText(string path, string text) $".{System.IO.Path.GetFileName(full)}.{Environment.ProcessId}.{Guid.NewGuid().ToString("N")[..8]}.tmp"); try { - using (var stream = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + using (var stream = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 20)) { - var bytes = new UTF8Encoding(false).GetBytes(text); - stream.Write(bytes, 0, bytes.Length); + fill(stream); stream.Flush(flushToDisk: true); } File.Move(tmp, full, overwrite: true); diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/IsonGraphVector.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/IsonGraphVector.cs index 8ff525d..ff7bb95 100644 --- a/isongraph-vector-csharp/src/IsonGraph.Vector/IsonGraphVector.cs +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/IsonGraphVector.cs @@ -38,6 +38,12 @@ public static class IsonGraphVectorVersion /// public const string EmbeddingSidecarSuffix = ".embeddings.json"; + /// + /// Suffix appended to a graph path to locate its optional binary + /// embeddings snapshot (see ). + /// + public const string EmbeddingSnapshotSuffix = ".embeddings.snap"; + /// /// Name of the serialization hashes. Written next /// to the digest so a reader that computes a different digest (or none) can @@ -223,7 +229,7 @@ public IReadOnlyList EncodeBatch(IReadOnlyList texts) /// Surrogate key, "type:id". /// The node the vector belongs to. -/// The embedding. +/// The embedding. The store keeps its one copy of every vector in a shared matrix, so a record handed out by carries a copy of it: changing it does not change the store. /// The text it was made from. /// /// Name of the model that made the vector: the encoder's ModelName, "unknown" @@ -374,626 +380,6 @@ public sealed class ExportedEmbeddings public string? GraphDigestBasis { get; set; } } -// ============================================================================= -// Embedding Store -// ============================================================================= - -/// -/// In-memory embedding storage with brute-force cosine similarity search. -/// -public sealed partial class EmbeddingStore -{ - private readonly Dictionary _embeddings = new(); - private readonly object _lock = new(); - private IEmbeddingEncoder? _encoder; - private int? _dimension; - private string? _version; - private ModelMismatchMode _onModelMismatch; - private Dictionary? _modelsCache; - private readonly HashSet _warnedMismatch = new(); - private IReadOnlyDictionary _lastImportMeta = new Dictionary(); - - // Opt-in approximate index (see AnnMode). _ivf is built lazily by the first - // search that qualifies, kept current by Put/Remove/Clear, and never saved. - private readonly AnnMode _ann; - private readonly int? _annNlist; - private int? _annNprobe; - private readonly int _annMinSize; - private readonly int _annSeed; - private IvfIndex? _ivf; - private long _annApproximate, _annExact, _annWidened, _annBuilds; - - /// Below this many vectors the index is not used by default. - public const int DefaultAnnMinSize = 50_000; - - /// Encoder used when a text is added or queried without a vector. - /// - /// Free-form version of the encoder/model that made these vectors, - /// recorded in the sidecar. Falls back to the encoder's own Version. - /// - /// - /// What a text query does when the encoder's ModelName differs from the - /// model that produced the stored vectors. Vectors whose model is - /// "unknown"/"provided" never trigger it, and vector queries are never - /// checked. - /// - /// - /// (default) keeps search exact, exactly as it - /// always was. answers bounded top-k searches - /// from an in-memory IVF-flat index: vectors are clustered into - /// cells and only the - /// cells nearest the query are scored. It is approximate - a true neighbour - /// in an unprobed cell is missed - so measure it with - /// . The - /// index is built lazily by the first search, never persisted, and kept - /// current through writes and deletes. It holds its own contiguous copy of - /// the vectors (about the size of the vectors again). - /// - /// Cells in the index. Null picks about 4 * sqrt(N) when it is built. - /// - /// Cells scanned per query. Null picks about 5% of nlist. More cells = higher - /// recall, slower; nprobe >= nlist is the exact scan. Adjustable later via - /// . - /// - /// Below this many vectors search stays exact. Default 50,000. - /// Seed for the k-means initialisation, so the same data gives the same index. - public EmbeddingStore( - IEmbeddingEncoder? encoder = null, - string? version = null, - ModelMismatchMode onModelMismatch = ModelMismatchMode.Warn, - AnnMode ann = AnnMode.None, - int? annNlist = null, - int? annNprobe = null, - int annMinSize = DefaultAnnMinSize, - int annSeed = 0) - { - CheckMode(onModelMismatch); - if (!Enum.IsDefined(ann)) - throw new ArgumentOutOfRangeException(nameof(ann), ann, "unknown AnnMode"); - if (annNlist is < 1) - throw new ArgumentOutOfRangeException(nameof(annNlist), annNlist, "annNlist must be >= 1"); - if (annNprobe is < 1) - throw new ArgumentOutOfRangeException(nameof(annNprobe), annNprobe, "annNprobe must be >= 1"); - if (annMinSize < 0) - throw new ArgumentOutOfRangeException(nameof(annMinSize), annMinSize, "annMinSize must be >= 0"); - _encoder = encoder; - _version = version; - _onModelMismatch = onModelMismatch; - _ann = ann; - _annNlist = annNlist; - _annNprobe = annNprobe; - _annMinSize = annMinSize; - _annSeed = annSeed; - } - - /// The approximate index in use, or for exact search. - public AnnMode Ann => _ann; - - /// Cells scanned per query; null means automatic (about 5% of nlist). - public int? AnnNprobe - { - get { lock (_lock) return _annNprobe; } - set - { - if (value is < 1) - throw new ArgumentOutOfRangeException(nameof(value), value, "AnnNprobe must be >= 1"); - lock (_lock) _annNprobe = value; - } - } - - private static void CheckMode(ModelMismatchMode mode) - { - if (!Enum.IsDefined(mode)) - throw new ArgumentOutOfRangeException(nameof(mode), mode, "unknown ModelMismatchMode"); - } - - /// - /// Raised with a human-readable message when the store has something to - /// warn about (a model mismatch, a version mismatch on load). With no - /// subscriber the message goes to standard error. - /// - public event Action? Warning; - - internal void RaiseWarning(string message) - { - var handler = Warning; - if (handler is not null) handler(message); - else Console.Error.WriteLine("isongraph-vector: warning: " + message); - } - - public IEmbeddingEncoder? Encoder - { - get => _encoder; - set => _encoder = value; - } - - /// - /// Encoder/model version recorded in the sidecar: the one set explicitly, - /// else the encoder's own, else null. - /// - public string? Version - { - get => _version ?? _encoder?.Version; - set => _version = value; - } - - /// What a text query does when the encoder's model differs from the stored vectors'. - public ModelMismatchMode OnModelMismatch - { - get => _onModelMismatch; - set { CheckMode(value); _onModelMismatch = value; } - } - - /// - /// Metadata (encoder_model, encoder_version, graph_digest, - /// graph_digest_basis) of the most recently imported payload; only the - /// keys it carried are present, and it is empty before any import. - /// - public IReadOnlyDictionary LastImportMeta - { - get { lock (_lock) return _lastImportMeta; } - } - - /// Vector length this store holds, or null while it is empty. - public int? Dimension - { - get { lock (_lock) return _dimension; } - } - - private static string Key(NodeRef nodeRef) => $"{nodeRef.Type}:{nodeRef.Id}"; - - /// - /// Keep the best results without ordering all of them. - /// - /// Sorting every candidate to return ten is O(n log n) where this is - /// O(n·k) with early rejection — for the small k a search actually asks - /// for, that is a large saving over 20,000 candidates. Insertion goes - /// after equal scores, so the order matches a stable descending sort. - /// - internal static List SelectTopK(List results, int k) - { - if (k <= 0) return new List(); - if (results.Count <= k) - return results.OrderByDescending(r => r.Score).ToList(); - - var top = new List(k); - foreach (var candidate in results) - { - if (top.Count == k && candidate.Score <= top[k - 1].Score) continue; - - int i = top.Count < k ? top.Count : k - 1; - if (top.Count < k) top.Add(candidate); - while (i > 0 && top[i - 1].Score < candidate.Score) - { - top[i] = top[i - 1]; - i--; - } - top[i] = candidate; - } - return top; - } - - - private void CheckDimension(ReadOnlySpan vector, string what) - { - if (vector.Length == 0) - throw new DimensionMismatchException($"{what} is empty"); - if (_dimension is null) - _dimension = vector.Length; - else if (vector.Length != _dimension) - throw new DimensionMismatchException( - $"{what} has {vector.Length} dimensions, store holds {_dimension}"); - } - - private void CheckQuery(ReadOnlySpan vector) - { - if (vector.Length == 0) - throw new DimensionMismatchException("query vector is empty"); - if (_dimension is not null && vector.Length != _dimension) - throw new DimensionMismatchException( - $"query vector has {vector.Length} dimensions, store holds {_dimension}"); - } - - /// - /// Cosine similarity over two vectors. - /// - /// Spans rather than IReadOnlyList: through the interface every element - /// access is a dispatch, which measured roughly twenty times slower than - /// the Rust and C++ ports scoring the same vectors. The stored vectors are - /// float[], so a span costs nothing and the JIT indexes them directly. - /// - private static float CosineSimilarity(ReadOnlySpan v1, ReadOnlySpan v2) - { - float dot = 0f, norm1 = 0f, norm2 = 0f; - int n = Math.Min(v1.Length, v2.Length); - for (int i = 0; i < n; i++) - { - dot += v1[i] * v2[i]; - norm1 += v1[i] * v1[i]; - norm2 += v2[i] * v2[i]; - } - norm1 = MathF.Sqrt(norm1); - norm2 = MathF.Sqrt(norm2); - if (norm1 == 0f || norm2 == 0f) return 0f; - return dot / (norm1 * norm2); - } - - private float[] ResolveQuery(string query) - { - if (_encoder is null) - throw new NoEncoderException("No encoder provided for text query"); - CheckModelCompat(); - var vector = _encoder.Encode(query); - CheckQuery(vector); - return vector; - } - - // -- model lifecycle ------------------------------------------------------- - - /// Model name stamped on a vector written now. - private string StampModel() => ModelCompat.Stamp(_encoder); - - /// The encoder's model name, or null when it has none worth comparing. - private string? CurrentModel() => ModelCompat.Current(_encoder); - - /// - /// Which models produced the stored vectors: model name to count. - /// "unknown" and "provided" appear as-is (vectors imported from a port - /// that records no model, or handed in with no encoder attached). - /// - public IReadOnlyDictionary Models() - { - lock (_lock) - { - if (_modelsCache is null) - { - var counts = new Dictionary(); - foreach (var record in _embeddings.Values) - counts[record.Model] = counts.GetValueOrDefault(record.Model) + 1; - _modelsCache = counts; - } - return new Dictionary(_modelsCache); - } - } - - /// - /// Complain when a text query's encoder is not the stored vectors' model. - /// Called before a text query is encoded; vector queries skip it. - /// - internal void CheckModelCompat() - { - if (_onModelMismatch == ModelMismatchMode.Ignore) return; - var current = CurrentModel(); - if (current is null) return; - - var foreign = ModelCompat.Foreign(current, Models().Keys); - if (foreign.Count == 0) return; - - var message = ModelCompat.Message(current, foreign); - if (_onModelMismatch == ModelMismatchMode.Error) - throw new ModelMismatchException(message); - - bool first; - lock (_lock) first = _warnedMismatch.Add(ModelCompat.WarnKey(current, foreign)); - if (first) RaiseWarning(message); - } - - /// - /// Node refs whose stored vector does not belong to the current encoder. - /// - /// A vector is stale when its recorded model differs from - /// (default: the encoder's ModelName) or its - /// length differs from (default: the - /// encoder's Dimension). Vectors whose model is "unknown"/"provided" - /// cannot be judged by model and are skipped unless - /// is true (a dimension mismatch still - /// counts). With no encoder and no arguments nothing can be judged and the - /// result is empty. - /// - public IReadOnlyList Stale(string? model = null, int? dimension = null, bool includeUnknown = false) - { - lock (_lock) - { - model ??= CurrentModel(); - dimension ??= _encoder?.Dimension; - - var stale = new List(); - foreach (var record in _embeddings.Values) - { - if (ModelCompat.IsStale(record.Model, record.Vector.Length, model, dimension, includeUnknown)) - stale.Add(record.NodeRef); - } - return stale; - } - } - - // -- writes ----------------------------------------------------------------- - - public string Add(NodeRef nodeRef, string text, float[]? vector = null) - { - lock (_lock) - { - if (vector is null) - { - if (_encoder is null) - throw new NoEncoderException("No encoder provided and no vector given"); - vector = _encoder.Encode(text); - } - CheckDimension(vector, "vector"); - return Put(nodeRef, text, vector, StampModel()); - } - } - - /// Store a vector under an explicit model name. Caller holds the lock and has checked the dimension. - private string Put(NodeRef nodeRef, string text, float[] vector, string model) - { - var key = Key(nodeRef); - var record = new EmbeddingRecord(key, nodeRef, vector, text, model); - _embeddings[key] = record; - _modelsCache = null; - _ivf?.Put(record); - return key; - } - - /// Copy a record in, keeping the model it was made by. - internal string AddRecord(EmbeddingRecord record) - { - lock (_lock) - { - CheckDimension(record.Vector, "vector"); - return Put(record.NodeRef, record.Text, record.Vector, record.Model); - } - } - - /// Add many embeddings, encoding their texts in one batched call. - public int AddBatch(IReadOnlyList<(NodeRef NodeRef, string Text)> entries) - { - if (entries.Count == 0) return 0; - if (_encoder is null) - throw new NoEncoderException("No encoder provided and no vectors given"); - - var vectors = _encoder.EncodeBatch(entries.Select(e => e.Text).ToList()); - if (vectors.Count != entries.Count) - throw new EmbeddingException( - $"encoder returned {vectors.Count} vectors for {entries.Count} texts"); - - for (int i = 0; i < entries.Count; i++) - Add(entries[i].NodeRef, entries[i].Text, vectors[i]); - return entries.Count; - } - - public EmbeddingRecord? Get(NodeRef nodeRef) - { - lock (_lock) - return _embeddings.TryGetValue(Key(nodeRef), out var record) ? record : null; - } - - public bool Remove(NodeRef nodeRef) - { - lock (_lock) - { - _modelsCache = null; - var key = Key(nodeRef); - bool removed = _embeddings.Remove(key); - if (removed) _ivf?.Drop(key); - return removed; - } - } - - public int Count - { - get { lock (_lock) return _embeddings.Count; } - } - - /// - /// Clear all embeddings. - /// - /// - /// Also forget the pinned vector length, so the next vector may have a - /// different one (switching to an encoder of a different dimension). True - /// by default, which is how this method has always behaved; pass false to - /// keep the store refusing vectors of the wrong length even when empty (the - /// Python port's default). - /// - public void Clear(bool resetDimension = true) - { - lock (_lock) - { - _embeddings.Clear(); - _modelsCache = null; - _ivf = null; - if (resetDimension) _dimension = null; - } - } - - /// - /// Find the most similar embeddings to a text query. - /// - /// With an approximate index () a bounded - /// on a store of at least the minimum size is answered from the index - /// (scores are the same cosine values, but a true neighbour in an unprobed - /// cell can be missing). forces the full scan. - /// - /// Query text. - /// Number of results (null returns every match). - /// Keep only embeddings of this node type. - /// Minimum similarity. - /// Skip the approximate index and scan everything (no effect without one). - public IReadOnlyList SimilaritySearch( - string query, int? topK = 10, string? nodeType = null, float threshold = 0.0f, bool exact = false) - => SimilaritySearchVector(ResolveQuery(query), topK, nodeType, threshold, exact); - - /// Find the most similar embeddings to a query vector; see . - public IReadOnlyList SimilaritySearchVector( - float[] queryVector, int? topK = 10, string? nodeType = null, float threshold = 0.0f, - bool exact = false) - { - lock (_lock) - { - CheckQuery(queryVector); - - if (_ann != AnnMode.None) - { - if (exact || topK is null || topK <= 0) - _annExact++; - else - { - var answer = AnnSearch(queryVector, topK.Value, nodeType, threshold); - if (answer is not null) return answer; - _annExact++; - } - } - - var results = new List(); - foreach (var record in _embeddings.Values) - { - if (nodeType is not null && record.NodeRef.Type != nodeType) continue; - float score = CosineSimilarity(queryVector, record.Vector); - if (score >= threshold) - results.Add(new SimilarityResult(record.NodeRef, score, record.Text)); - } - - if (topK is null) - return results.OrderByDescending(r => r.Score).ToList(); - return SelectTopK(results, topK.Value); - } - } - - /// - /// Cosine score for every embedded node, keyed by "type:id". - /// - /// One scan serving both seed selection and the per-node relevance that - /// SemanticMultiHop blends in, instead of two. - /// - public IReadOnlyDictionary ScoreMap(float[] queryVector, string? nodeType = null) - { - lock (_lock) - { - CheckQuery(queryVector); - var scores = new Dictionary(); - foreach (var (key, record) in _embeddings) - { - if (nodeType is not null && record.NodeRef.Type != nodeType) continue; - scores[key] = CosineSimilarity(queryVector, record.Vector); - } - return scores; - } - } - - /// - /// Serialize every embedding into the portable payload. - /// - /// Besides the embeddings it carries encoder_model (the encoder's - /// ModelName) and encoder_version when known and, when - /// is given, graph_digest and its - /// basis. Absent values are omitted, not written as null. - /// - public ExportedEmbeddings ExportEmbeddings(string? graphDigest = null) - { - lock (_lock) - { - var payload = new ExportedEmbeddings { Dimension = _dimension }; - foreach (var record in _embeddings.Values.OrderBy(r => r.NodeRef.Type) - .ThenBy(r => r.NodeRef.Id, StringComparer.Ordinal)) - { - payload.Embeddings.Add(new ExportedEmbedding - { - Type = record.NodeRef.Type, - // C# node ids are always strings; the other ports use this - // to tell a numeric id from a numeric-looking string one. - Id = JsonSerializer.SerializeToElement(record.NodeRef.Id), - IdType = "str", - Text = record.Text, - Model = record.Model, - Vector = record.Vector, - }); - } - payload.Count = payload.Embeddings.Count; - payload.EncoderModel = CurrentModel(); - payload.EncoderVersion = Version; - if (graphDigest is not null) - { - payload.GraphDigest = graphDigest; - payload.GraphDigestBasis = IsonGraphVectorVersion.GraphDigestBasis; - } - return payload; - } - } - - /// - /// Load a payload produced by ExportEmbeddings (in any port). Fields this - /// build does not know are ignored; the metadata it does know is exposed as - /// . - /// - public int ImportEmbeddings(ExportedEmbeddings payload, bool replace = false) - { - if (payload.Format != IsonGraphVectorVersion.EmbeddingFormat) - throw new EmbeddingException( - $"not an {IsonGraphVectorVersion.EmbeddingFormat} payload: format={payload.Format}"); - if (payload.Version != IsonGraphVectorVersion.EmbeddingFormatVersion) - throw new EmbeddingException( - $"unsupported {IsonGraphVectorVersion.EmbeddingFormat} version {payload.Version} " + - $"(this build reads version {IsonGraphVectorVersion.EmbeddingFormatVersion})"); - - lock (_lock) - { - var meta = new Dictionary(); - if (payload.EncoderModel is not null) meta["encoder_model"] = payload.EncoderModel; - if (payload.EncoderVersion is not null) meta["encoder_version"] = payload.EncoderVersion; - if (payload.GraphDigest is not null) meta["graph_digest"] = payload.GraphDigest; - if (payload.GraphDigestBasis is not null) meta["graph_digest_basis"] = payload.GraphDigestBasis; - _lastImportMeta = meta; - - if (replace) - { - Clear(resetDimension: true); - _dimension = payload.Dimension; - } - - int written = 0; - foreach (var item in payload.Embeddings) - { - var nodeRef = new NodeRef(item.Type, item.IdAsString()); - CheckDimension(item.Vector, $"embedding for :{item.Type}:{nodeRef.Id}"); - Put(nodeRef, item.Text, item.Vector, item.Model ?? "unknown"); - written++; - } - return written; - } - } - - /// Serialize the portable payload to a JSON string. - public string ToJson(string? graphDigest = null) => JsonSerializer.Serialize(ExportEmbeddings(graphDigest)); - - /// Load embeddings from a JSON payload written by any port. - public int FromJson(string json, bool replace = false) - { - ExportedEmbeddings? payload; - try { payload = JsonSerializer.Deserialize(json); } - catch (JsonException ex) { throw new EmbeddingException($"unreadable embeddings payload: {ex.Message}"); } - if (payload is null) throw new EmbeddingException("payload did not deserialize"); - return ImportEmbeddings(payload, replace); - } - - /// - /// Write every embedding to a JSON sidecar, atomically. Returns the count. - /// - /// The file is written to a temp file and renamed into place, so a crash - /// never leaves a half-written sidecar. ties - /// the sidecar to one particular graph (see ). - /// - public int SaveEmbeddings(string path, string? graphDigest = null) - { - var payload = ExportEmbeddings(graphDigest); - AtomicFile.WriteText(path, JsonSerializer.Serialize(payload)); - return payload.Count; - } - - /// Read a JSON sidecar written by (or by any port). - public int LoadEmbeddings(string path, bool replace = false) => - FromJson(File.ReadAllText(path, System.Text.Encoding.UTF8), replace); -} - // ============================================================================= // Semantic Graph // ============================================================================= @@ -1972,15 +1358,50 @@ public static string EmbeddingsPathFor(string path) => /// /// With false an existing sidecar is left in /// place (and will fail the digest check on load, as it no longer matches). + /// + /// With true a binary snapshot of the + /// embeddings is also written to <path>.embeddings.snap, after + /// and in addition to the JSON sidecar, which stays the portable copy. + /// uses the snapshot when it provably belongs to the + /// graph and sidecar on disk (and its checksum verifies), and the JSON + /// otherwise; see to learn which. A save + /// without a snapshot deletes any existing one. If the snapshot cannot be + /// written the graph and sidecar are saved anyway, a stale snapshot is + /// removed, and the error is thrown. /// - public void Save(string path, bool embeddings = true) + /// Graph file path (.isonl for ISONL, otherwise ISON). + /// Write the embeddings sidecar. + /// Also write the binary snapshot (needs ). + public void Save(string path, bool embeddings = true, bool snapshot = false) { bool isonl = IsIsonl(path); var text = isonl ? _graph.ToIsonl() : _graph.ToIson(); AtomicFile.WriteText(path, text); - if (embeddings) - _store.SaveEmbeddings(EmbeddingsPathFor(path), GraphDigest.OfText(text, isonl)); + if (!embeddings) return; + var digest = GraphDigest.OfText(text, isonl); + var sidecar = EmbeddingsPathFor(path); + var snap = SnapshotPathFor(path); + if (!snapshot) + DeleteQuietly(snap); // the sidecar is about to change: a snapshot of the old one is stale + _store.SaveEmbeddings(sidecar, digest); + if (!snapshot) return; + try + { + _store.SaveSnapshot(snap, new SnapshotIdentity(digest, SidecarId.OfFile(sidecar))); + } + catch + { + DeleteQuietly(snap); // the pair is saved; a stale snapshot must not survive + throw; + } + } + + private static void DeleteQuietly(string path) + { + try { File.Delete(path); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } } /// @@ -2030,6 +1451,40 @@ public void Save(string path, bool embeddings = true) /// Seed for the k-means initialisation. /// Digest mismatch with . public static SemanticGraph Load( + string path, + IEmbeddingEncoder? encoder = null, + bool embeddings = true, + bool autoEmbed = true, + IEnumerable? embedFields = null, + GraphMismatchMode onMismatch = GraphMismatchMode.Error, + string? version = null, + ModelMismatchMode onModelMismatch = ModelMismatchMode.Warn, + Action? onWarning = null, + AnnMode ann = AnnMode.None, + int? annNlist = null, + int? annNprobe = null, + int annMinSize = EmbeddingStore.DefaultAnnMinSize, + int annSeed = 0) => + LoadWithReport(path, encoder, embeddings, autoEmbed, embedFields, onMismatch, version, + onModelMismatch, onWarning, ann, annNlist, annNprobe, annMinSize, annSeed).Graph; + + /// Binary snapshot path for a saved graph (<path>.embeddings.snap). + public static string SnapshotPathFor(string path) => + path + IsonGraphVectorVersion.EmbeddingSnapshotSuffix; + + /// + /// , also saying where the embeddings came from. + /// + /// If a binary snapshot (<path>.embeddings.snap, written by + /// with snapshot: true) sits beside the sidecar, + /// it is used when its header names this graph's digest and the sidecar + /// file as it is now and its checksum verifies; the result is identical to + /// loading the JSON. Otherwise the JSON sidecar is read, exactly as before, + /// and says why the snapshot was + /// not used (a snapshot that fails any check never changes the store). + /// Parameters are those of . + /// + public static (SemanticGraph Graph, LoadReport Report) LoadWithReport( string path, IEmbeddingEncoder? encoder = null, bool embeddings = true, @@ -2048,6 +1503,10 @@ public static SemanticGraph Load( if (!Enum.IsDefined(onMismatch)) throw new ArgumentOutOfRangeException(nameof(onMismatch), onMismatch, "unknown GraphMismatchMode"); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + LoadReport Report(LoadSource source, string? skipped, int count) => + new(source, skipped, count, System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalSeconds); + bool isonl = IsIsonl(path); var graphText = File.ReadAllText(path, System.Text.Encoding.UTF8); var name = System.IO.Path.GetFileNameWithoutExtension(path); @@ -2063,16 +1522,26 @@ public static SemanticGraph Load( foreach (var edge in loaded.Edges()) graph._graph.AddEdge(edge.RelType, edge.Source, edge.Target, edge.Properties); - if (!embeddings) return graph; + if (!embeddings) return (graph, Report(LoadSource.NoEmbeddings, null, 0)); var sidecar = EmbeddingsPathFor(path); - if (!File.Exists(sidecar)) return graph; + if (!File.Exists(sidecar)) return (graph, Report(LoadSource.NoEmbeddings, null, 0)); + + string? snapshotSkipped = null; + var snap = SnapshotPathFor(path); + if (File.Exists(snap)) + { + if (graph._store.TryAdoptSnapshot(snap, sidecar, GraphDigest.OfText(graphText, isonl), out var info, out snapshotSkipped)) + { + graph.WarnOnVersionSkew(); + return (graph, Report(LoadSource.Snapshot, null, info!.Count)); + } + } ExportedEmbeddings? payload; try { - payload = JsonSerializer.Deserialize( - File.ReadAllText(sidecar, System.Text.Encoding.UTF8)); + payload = EmbeddingStore.DeserializePayload(sidecar); } catch (JsonException ex) { @@ -2105,21 +1574,25 @@ public static SemanticGraph Load( if (problem is not null && onMismatch == GraphMismatchMode.Reembed) { graph.EmbedAllNodes(); - return graph; + return (graph, Report(LoadSource.Json, snapshotSkipped, graph._store.Count)); } if (problem is not null) graph._store.RaiseWarning(problem); graph._store.ImportEmbeddings(payload); + graph.WarnOnVersionSkew(); + return (graph, Report(LoadSource.Json, snapshotSkipped, graph._store.Count)); + } - var storedVersion = graph._store.LastImportMeta.GetValueOrDefault("encoder_version"); - var currentVersion = graph._store.Version; + private void WarnOnVersionSkew() + { + var storedVersion = _store.LastImportMeta.GetValueOrDefault("encoder_version"); + var currentVersion = _store.Version; if (storedVersion is not null && currentVersion is not null && storedVersion != currentVersion) { - graph._store.RaiseWarning( + _store.RaiseWarning( $"embeddings were made by encoder version '{storedVersion}', " + $"but the encoder in use is version '{currentVersion}'"); } - return graph; } public override string ToString() => diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/IvfIndex.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/IvfIndex.cs index a3ce1f5..76249c3 100644 --- a/isongraph-vector-csharp/src/IsonGraph.Vector/IvfIndex.cs +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/IvfIndex.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Diagnostics; using System.Numerics; using NVec = System.Numerics.Vector; @@ -6,226 +7,78 @@ namespace IsonGraph.Vector; /// -/// Inverted-file ("IVF-flat") index over a contiguous copy of a store's vectors. +/// Inverted-file ("IVF-flat") index over a . /// /// Spherical k-means partitions the rows into Nlist cells by direction, -/// which is what cosine similarity compares. A query scores the lists -/// cells whose centroids are nearest and only the rows inside them, with the -/// same cosine formula the exact scan uses. +/// which is what cosine similarity compares. A query scores the cells whose +/// centroids are nearest and only the rows inside them, with the same cosine +/// the exact scan uses (the same kernel, so the same bits). /// -/// Storage is struct-of-arrays: one float[] matrix (row stride padded -/// to a whole number of SIMD lanes, so the dot product has no scalar tail), -/// the row norms, a type code per row and the owning record. The cells are a -/// per-row cell number (_assign, -1 = none) plus the rows grouped by -/// cell (_order/_offsets). +/// The index owns no vectors: the table holds the one copy of every row, and +/// the index holds only the centroids and, per slot, which cell it is in: +/// * _assign[slot]: its cell, , or ; +/// * _pos[slot]: its position in that cell's list (or the pending +/// list), so a row leaves its list in O(1) by swapping the last one in; +/// * _lists[cell]: exactly the live folded rows of a cell, no stale entries. /// -/// It follows the store it belongs to instead of being rebuilt on every write: -/// * a new or updated row is "pending": it has no cell, and every query -/// scores the pending rows exactly, so a write is visible immediately; -/// * pending rows are folded into their nearest cell in one pass once there -/// are enough of them (the store decides when); -/// * a delete clears the row's cell; stale entries in _order are -/// skipped at query time (a row is only taken from the cell it is assigned to); -/// * compaction (when a quarter of the slots are dead) remaps row numbers and -/// carries the cells across without retraining; -/// * Churn counts every write and delete since the centroids were -/// trained; the store retrains when it passes a fraction of the trained size. +/// It follows the store instead of being rebuilt on every write: a new or +/// updated row is "pending" and every query scores the pending rows exactly, +/// so a write is visible at once; the write that makes +/// pending rows folds them into their nearest cells. A delete detaches the row. +/// Compaction renumbers slots and the index follows without retraining. +/// Churn counts writes and deletes since training; past a fraction of +/// the trained size the store retrains. /// -/// Not thread-safe: the owning store calls it only under its lock. Queries -/// reuse scratch buffers owned by the index, so a query allocates nothing -/// per scanned row (only its result objects). +/// Mutating members are called only under the store's write lock; searches +/// read the index under the read lock and touch no shared mutable state +/// (all scratch is per call), so any number run at once. /// internal sealed class IvfIndex { - // -- rows ------------------------------------------------------------------- + /// Pending rows that trigger a fold (DESIGN.md section 8). + public const int FoldBatch = 256; + + private const int NoCell = -1; + private const int Pending = -2; + private readonly int _dim; - private readonly int _stride; // dim rounded up to the SIMD width - private float[] _matrix; - private float[] _norms; - private int[] _codes; - private bool[] _alive; - private EmbeddingRecord?[] _recs; - private int[] _assign; - private int _n; // slots in use, tombstones included - private int _dead; - private readonly Dictionary _pos = new(); // record id -> slot - private readonly Dictionary _codeOf = new(); // node type -> code - - // -- cells ------------------------------------------------------------------ - private float[] _centroids = Array.Empty(); - private int[] _order = Array.Empty(); - private int[] _offsets = Array.Empty(); - private readonly HashSet _pending = new(); + private readonly int _stride; // dim rounded up to the SIMD width (centroids only) + private float[] _centroids = Array.Empty(); // nlist x stride + private int[] _assign = Array.Empty(); + private int[] _pos = Array.Empty(); + private int[][] _lists = Array.Empty(); + private int[] _listLen = Array.Empty(); + private int[] _pend = new int[FoldBatch + 16]; + private int _pendLen; public int Nlist { get; private set; } public long Churn { get; private set; } public int TrainedN { get; private set; } public double BuildSeconds { get; private set; } - public int PendingCount => _pending.Count; - public int LiveCount => _n - _dead; - public long VectorBytes => ((long)_matrix.Length + _norms.Length) * sizeof(float); - public long IndexBytes => - ((long)_centroids.Length + _assign.Length + _order.Length + _offsets.Length) * sizeof(int); - - // -- scratch (single-threaded, under the store lock) ------------------------ - private float[] _q; - private float[] _cKey = Array.Empty(); - private int[] _cId = Array.Empty(); - private int[] _candRows = new int[256]; - private float[] _candScores = new float[256]; - private int _candCount; - private int[] _topRows = new int[16]; - private float[] _topScores = new float[16]; - - private IvfIndex(int dim, int capacity) - { - _dim = dim; - int lanes = Vector.Count; - _stride = (dim + lanes - 1) / lanes * lanes; - long cells = (long)capacity * _stride; - if (cells > Array.MaxLength) - throw new EmbeddingException( - $"approximate index would need {cells} floats ({capacity} vectors x {_stride}); " + - "the contiguous copy is limited to about 2 billion values"); - _matrix = new float[cells]; - _norms = new float[capacity]; - _codes = new int[capacity]; - _alive = new bool[capacity]; - _recs = new EmbeddingRecord?[capacity]; - _assign = new int[capacity]; - Array.Fill(_assign, -1); - _q = new float[_stride]; - } - - /// Copy into a fresh, untrained index. Call next. - public static IvfIndex FromRecords(IEnumerable records, int dim, int count) - { - var index = new IvfIndex(dim, count + count / 8 + 16); - foreach (var record in records) index.Append(record); - return index; - } - - // -- writes ----------------------------------------------------------------- - - private int CodeFor(string nodeType) - { - if (!_codeOf.TryGetValue(nodeType, out int code)) - _codeOf[nodeType] = code = _codeOf.Count; - return code; - } - - private void Grow() - { - int capacity = Math.Max(_alive.Length + _alive.Length / 2, _alive.Length + 64); - if ((long)capacity * _stride > Array.MaxLength) - capacity = Array.MaxLength / _stride; - if (capacity <= _alive.Length) - throw new EmbeddingException("approximate index is full (contiguous copy limited to about 2 billion values)"); - int old = _alive.Length; - Array.Resize(ref _matrix, capacity * _stride); - Array.Resize(ref _norms, capacity); - Array.Resize(ref _codes, capacity); - Array.Resize(ref _alive, capacity); - Array.Resize(ref _recs, capacity); - Array.Resize(ref _assign, capacity); - Array.Fill(_assign, -1, old, capacity - old); - } - - private int Append(EmbeddingRecord record) - { - if (_n >= _alive.Length) Grow(); - int i = _n++; - _pos[record.Id] = i; - _alive[i] = true; - Store(i, record); - return i; - } - - private void Store(int i, EmbeddingRecord record) - { - var dst = new Span(_matrix, i * _stride, _stride); - record.Vector.AsSpan(0, _dim).CopyTo(dst); - dst.Slice(_dim).Clear(); - _norms[i] = MathF.Sqrt(Dot(dst, dst)); - _codes[i] = CodeFor(record.NodeRef.Type); - _recs[i] = record; - } + public int PendingCount => _pendLen; + public bool Stale => Churn > 0.5 * TrainedN; - /// A new or replaced vector: visible to the next query, folded into a cell later. - public void Put(EmbeddingRecord record) - { - if (_pos.TryGetValue(record.Id, out int i)) - Store(i, record); // overwrite in place: keeps its position - else - i = Append(record); - _assign[i] = -1; - _pending.Add(i); - Churn++; - } + /// The index keeps no copy of any vector: the table's rows are the only ones. + public long VectorBytes => 0; - /// A deleted vector: the slot becomes a tombstone no query will read. - public void Drop(string id) + public long IndexBytes { - if (!_pos.Remove(id, out int i)) return; - _norms[i] = 0f; - _alive[i] = false; - _recs[i] = null; - _assign[i] = -1; - _pending.Remove(i); - _dead++; - Churn++; - while (_n > 0 && !_alive[_n - 1]) // trailing tombstones just shorten the table + get { - _n--; - _dead--; + long bytes = ((long)_centroids.Length + _assign.Length + _pos.Length + _listLen.Length + _pend.Length) * sizeof(int); + foreach (var l in _lists) bytes += (long)l.Length * sizeof(int); + return bytes; } - if (_dead > 32 && _dead * 4 > _n) Compact(); } - /// Squeeze tombstones out, keeping the live rows in order, and carry the cells across. - private void Compact() + private IvfIndex(int dim) { - var remap = new int[_n]; - int kept = 0; - for (int i = 0; i < _n; i++) - { - if (!_alive[i]) { remap[i] = -1; continue; } - remap[i] = kept; - if (kept != i) // destination never overtakes the source - { - Array.Copy(_matrix, i * _stride, _matrix, kept * _stride, _stride); - _norms[kept] = _norms[i]; - _codes[kept] = _codes[i]; - _recs[kept] = _recs[i]; - _assign[kept] = _assign[i]; - } - kept++; - } - if (_pending.Count > 0) - { - var moved = new List(_pending.Count); - foreach (int old in _pending) moved.Add(remap[old]); - _pending.Clear(); - foreach (int row in moved) _pending.Add(row); - } - for (int i = kept; i < _n; i++) - { - _recs[i] = null; - _alive[i] = false; - _assign[i] = -1; - _norms[i] = 0f; - } - for (int i = 0; i < kept; i++) - { - _alive[i] = true; - _pos[_recs[i]!.Id] = i; - } - _n = kept; - _dead = 0; - Relist(); + _dim = dim; + int lanes = Vector.Count; + _stride = (dim + lanes - 1) / lanes * lanes; } - // -- training --------------------------------------------------------------- + // -- building --------------------------------------------------------------- private struct SplitMix64 { @@ -246,18 +99,27 @@ public ulong Next() } /// - /// Train centroids on a sample, then assign every live row. Deterministic - /// for a given data set, , - /// and SIMD width (the assignment passes run in parallel but every row's - /// answer is independent of the others, and the centroid sums are - /// accumulated in a fixed order). + /// Train centroids on a sample of the table's live rows, then assign every + /// live row. Deterministic for a given data set, , + /// and SIMD width (the assignment passes run in + /// parallel but every row's answer is independent of the others, and the + /// centroid sums are accumulated in a fixed order). /// - public void Train(int nlist, int seed, int iters = 8) + public static IvfIndex Build(VectorTable t, int nlist, int seed, int iters = 8) { var started = Stopwatch.GetTimestamp(); - var live = new int[LiveCount]; - for (int i = 0, j = 0; i < _n; i++) - if (_alive[i]) live[j++] = i; + var ivf = new IvfIndex(t.Dim); + ivf.Train(t, nlist, seed, iters); + ivf.BuildSeconds = Stopwatch.GetElapsedTime(started).TotalSeconds; + return ivf; + } + + private void Train(VectorTable t, int nlist, int seed, int iters) + { + int n = t.Slots; + var live = new int[t.Live]; + for (int i = 0, j = 0; i < n; i++) + if (t.IsLive(i)) live[j++] = i; int total = live.Length; nlist = Math.Max(1, Math.Min(nlist, total)); var rng = new SplitMix64(seed); @@ -279,145 +141,265 @@ public void Train(int nlist, int seed, int iters = 8) Array.Sort(sample); } - // Unit-length sample rows (a zero row stays zero). - var x = new float[(long)sampleN * _stride]; - for (int r = 0; r < sampleN; r++) - { - int row = sample[r]; - float norm = _norms[row]; - float scale = norm > 0f ? 1f / norm : 0f; - var src = new ReadOnlySpan(_matrix, row * _stride, _stride); - var dst = new Span(x, r * _stride, _stride); - for (int d = 0; d < _stride; d++) dst[d] = src[d] * scale; - } + // The sample is never copied out of the table: a unit-length, SIMD-padded copy of one row + // (a zero row stays zero) is produced on demand, so training allocates no sample matrix. + int stride = _stride; // Initial centroids: nlist distinct sample rows. - var cent = new float[(long)nlist * _stride]; + var cent = new float[(long)nlist * stride]; var pick = new int[sampleN]; for (int i = 0; i < sampleN; i++) pick[i] = i; for (int c = 0; c < nlist; c++) { int j = c + rng.NextInt(sampleN - c); (pick[c], pick[j]) = (pick[j], pick[c]); - Array.Copy(x, pick[c] * _stride, cent, c * _stride, _stride); + UnitRow(t, sample[pick[c]], new Span(cent, c * stride, stride)); } var cell = new int[sampleN]; var counts = new int[nlist]; - var sums = new double[(long)nlist * _stride]; + var sums = new double[(long)nlist * stride]; + var unit = new float[stride]; for (int it = 0; it < iters; it++) { - AssignNearest(x, null, sampleN, cent, nlist, cell); + AssignRows(t, sample, sampleN, cent, nlist, cell, BlockRows, unitRows: true); Array.Clear(counts); Array.Clear(sums); for (int r = 0; r < sampleN; r++) // sequential: a fixed summation order { int c = cell[r]; counts[c]++; - var src = new ReadOnlySpan(x, r * _stride, _stride); - var acc = new Span(sums, c * _stride, _stride); - for (int d = 0; d < _stride; d++) acc[d] += src[d]; + UnitRow(t, sample[r], unit); + var acc = new Span(sums, c * stride, stride); + for (int d = 0; d < stride; d++) acc[d] += unit[d]; } for (int c = 0; c < nlist; c++) { - var dst = new Span(cent, c * _stride, _stride); - var acc = new ReadOnlySpan(sums, c * _stride, _stride); - for (int d = 0; d < _stride; d++) dst[d] = (float)acc[d]; + var dst = new Span(cent, c * stride, stride); + var acc = new ReadOnlySpan(sums, c * stride, stride); + for (int d = 0; d < stride; d++) dst[d] = (float)acc[d]; float length = MathF.Sqrt(Dot(dst, dst)); if (counts[c] == 0 || length == 0f) // re-seed a dead centroid on a random point { - Array.Copy(x, rng.NextInt(sampleN) * _stride, cent, c * _stride, _stride); + UnitRow(t, sample[rng.NextInt(sampleN)], dst); length = MathF.Sqrt(Dot(dst, dst)); } if (length > 0f) - for (int d = 0; d < _stride; d++) dst[d] /= length; + for (int d = 0; d < stride; d++) dst[d] /= length; } } _centroids = cent; Nlist = nlist; - _cKey = new float[nlist]; - _cId = new int[nlist]; - - // Every row, in the order of the table. (Raw rows rank centroids the same - // as unit rows: scaling by a positive norm does not change the argmax.) - AssignNearest(_matrix, null, _n, cent, nlist, _assign); - for (int i = 0; i < _n; i++) - if (!_alive[i]) _assign[i] = -1; - _pending.Clear(); + _assign = new int[Math.Max(n, 16)]; + _pos = new int[_assign.Length]; + Array.Fill(_assign, NoCell); + AssignRows(t, null, n, cent, nlist, _assign, BlockRows, unitRows: false); + for (int i = 0; i < n; i++) + if (!t.IsLive(i)) _assign[i] = NoCell; + _pendLen = 0; Churn = 0; TrainedN = total; - Relist(); - BuildSeconds = Stopwatch.GetElapsedTime(started).TotalSeconds; + Relist(n); } - /// Regroup the rows by cell (a stable counting sort of _assign). - private void Relist() + /// Rebuild every cell list from _assign (slot order within a cell) and the positions. + private void Relist(int n) { - if (_offsets.Length != Nlist + 1) _offsets = new int[Nlist + 1]; - else Array.Clear(_offsets); - int assigned = 0; - for (int i = 0; i < _n; i++) + var counts = new int[Nlist]; + for (int i = 0; i < n; i++) { int c = _assign[i]; - if (c >= 0) { _offsets[c + 1]++; assigned++; } + if (c >= 0) counts[c]++; } - for (int c = 0; c < Nlist; c++) _offsets[c + 1] += _offsets[c]; - if (_order.Length < assigned) _order = new int[assigned + assigned / 8 + 16]; - var cursor = new int[Nlist]; - Array.Copy(_offsets, cursor, Nlist); - for (int i = 0; i < _n; i++) + _lists = new int[Nlist][]; + _listLen = new int[Nlist]; + for (int c = 0; c < Nlist; c++) _lists[c] = new int[counts[c] + counts[c] / 8 + 4]; + for (int i = 0; i < n; i++) { int c = _assign[i]; - if (c >= 0) _order[cursor[c]++] = i; + if (c < 0) continue; + _pos[i] = _listLen[c]; + _lists[c][_listLen[c]++] = i; } } - /// Give every pending row its nearest cell and regroup. - public void Fold() + // -- following the store ---------------------------------------------------- + + private void Detach(int slot) + { + if ((uint)slot >= (uint)_assign.Length) return; + int cell = _assign[slot]; + if (cell == NoCell) return; + int at = _pos[slot]; + int[] list; + int last; + if (cell == Pending) { list = _pend; last = --_pendLen; } + else { list = _lists[cell]; last = --_listLen[cell]; } + if (at != last) + { + int moved = list[last]; + list[at] = moved; + _pos[moved] = at; + } + _assign[slot] = NoCell; + } + + /// + /// A row was written (new, or its vector replaced): it waits in the pending + /// list. Returns true when the list is long enough that the caller should + /// now. + /// + public bool NotePut(int slot) { - if (_pending.Count == 0) return; - var ids = new int[_pending.Count]; - _pending.CopyTo(ids); - Array.Sort(ids); - var cells = new int[ids.Length]; - AssignNearest(_matrix, ids, ids.Length, _centroids, Nlist, cells); - for (int i = 0; i < ids.Length; i++) _assign[ids[i]] = cells[i]; - _pending.Clear(); - Relist(); + if (slot >= _assign.Length) + { + int cap = Math.Max(_assign.Length * 2, slot + 1); + int old = _assign.Length; + Array.Resize(ref _assign, cap); + Array.Resize(ref _pos, cap); + Array.Fill(_assign, NoCell, old, cap - old); + } + Detach(slot); + if (_pendLen == _pend.Length) Array.Resize(ref _pend, _pend.Length * 2); + _assign[slot] = Pending; + _pos[slot] = _pendLen; + _pend[_pendLen++] = slot; + Churn++; + return _pendLen >= FoldBatch; + } + + public void NoteDrop(int slot) + { + Detach(slot); + Churn++; + } + + /// Give every pending row its nearest cell. + public void Fold(VectorTable t) + { + if (_pendLen == 0) return; + var slots = new int[_pendLen]; + Array.Copy(_pend, slots, _pendLen); + var cells = new int[slots.Length]; + AssignRows(t, slots, slots.Length, _centroids, Nlist, cells, FoldBlockRows, unitRows: false); + _pendLen = 0; + for (int i = 0; i < slots.Length; i++) + { + int slot = slots[i], c = cells[i]; + if (_listLen[c] == _lists[c].Length) Array.Resize(ref _lists[c], _lists[c].Length * 2 + 4); + _assign[slot] = c; + _pos[slot] = _listLen[c]; + _lists[c][_listLen[c]++] = slot; + } + } + + /// Follow the table's compaction: [old] is the new slot, or -1 for a dead one. + public void Compact(int[] remap, int newSlots) + { + var assign = new int[Math.Max(newSlots, 16)]; + Array.Fill(assign, NoCell); + for (int old = 0; old < remap.Length; old++) + { + int s = remap[old]; + if (s >= 0 && old < _assign.Length) assign[s] = _assign[old]; + } + var pending = new int[Math.Max(_pend.Length, FoldBatch + 16)]; + int p = 0; + for (int i = 0; i < _pendLen; i++) + { + int s = remap[_pend[i]]; + if (s >= 0) pending[p++] = s; + } + _assign = assign; + _pos = new int[assign.Length]; + Relist(newSlots); + _pend = pending; + _pendLen = p; + for (int i = 0; i < p; i++) _pos[pending[i]] = i; + } + + /// Every invariant of DESIGN.md section 7 that concerns the index; null when they hold. + public string? Verify(VectorTable t) + { + if (_assign.Length < t.Slots) return $"index covers {_assign.Length} rows, table has {t.Slots}"; + if (_pendLen >= FoldBatch) return $"{_pendLen} rows pending, the cap is {FoldBatch}"; + int seen = 0; + for (int c = 0; c < Nlist; c++) + for (int at = 0; at < _listLen[c]; at++) + { + int s = _lists[c][at]; + if (_assign[s] != c || _pos[s] != at) return $"slot {s} listed in cell {c} at {at} but assigned {_assign[s]} at {_pos[s]}"; + if (!t.IsLive(s)) return $"dead slot {s} still in cell {c}"; + seen++; + } + for (int at = 0; at < _pendLen; at++) + { + int s = _pend[at]; + if (_assign[s] != Pending || _pos[s] != at || !t.IsLive(s)) return $"pending slot {s} is inconsistent"; + seen++; + } + if (seen != t.Live) return $"{seen} rows indexed, {t.Live} live"; + for (int s = 0; s < t.Slots; s++) + if (!t.IsLive(s) && _assign[s] != NoCell) return $"dead slot {s} is assigned"; + return null; } // -- nearest-centroid kernel ------------------------------------------------ private const int BlockRows = 256; + private const int FoldBlockRows = 32; private const int CentroidTile = 64; + [ThreadStatic] private static float[]? t_block; + + /// + /// A row of the table as the k-means kernel wants it: zero-padded to the + /// SIMD width and, when , scaled to unit length (a + /// zero row stays zero). + /// + private void UnitRow(VectorTable t, int slot, Span dst, bool unit = true) + { + t.Row(slot).CopyTo(dst); + dst.Slice(_dim).Clear(); + if (!unit) return; + float norm = MathF.Sqrt(Dot(dst, dst)); + float scale = norm > 0f ? 1f / norm : 0f; + for (int d = 0; d < _dim; d++) dst[d] *= scale; + } + /// - /// For rows (consecutive from 0, or the rows named - /// by ) of , the index of the - /// centroid with the largest dot product (the lowest index wins a tie). - /// is indexed like the rows, or by the row itself - /// when is null. + /// For table rows (slots 0.. or the named + /// ), the nearest centroid. + /// is indexed like the rows (by slot when is null). + /// Each block of rows is copied into a zero-padded buffer first, so the + /// kernel and its results are the ones the padded layout always had. /// - private void AssignNearest(float[] data, int[]? ids, int count, float[] cent, int nlist, int[] output) + private void AssignRows(VectorTable t, int[]? slots, int count, float[] cent, int nlist, int[] output, int blockRows, bool unitRows) { if (count == 0) return; int stride = _stride; - int blocks = (count + BlockRows - 1) / BlockRows; - if (blocks == 1) + int blocks = (count + blockRows - 1) / blockRows; + void One(int b) { - NearestBlock(data, ids, 0, count, cent, nlist, stride, output); - return; + int start = b * blockRows; + int len = Math.Min(blockRows, count - start); + var buf = t_block; + if (buf is null || buf.Length < len * stride) t_block = buf = new float[Math.Max(len, BlockRows) * stride]; + for (int i = 0; i < len; i++) + { + int slot = slots is null ? start + i : slots[start + i]; + UnitRow(t, slot, new Span(buf, i * stride, stride), unitRows); + } + NearestBlock(buf, null, 0, len, cent, nlist, stride, output, start); } - Parallel.For(0, blocks, b => - { - int start = b * BlockRows; - NearestBlock(data, ids, start, Math.Min(BlockRows, count - start), cent, nlist, stride, output); - }); + if (blocks == 1) One(0); + else Parallel.For(0, blocks, One); } private static void NearestBlock( - float[] data, int[]? ids, int start, int count, float[] cent, int nlist, int stride, int[] output) + float[] data, int[]? ids, int start, int count, float[] cent, int nlist, int stride, int[] output, + int outputStart = -1) { Span best = stackalloc float[BlockRows]; Span bestIdx = stackalloc int[BlockRows]; @@ -444,8 +426,9 @@ private static void NearestBlock( } } } + int o = outputStart >= 0 ? outputStart : start; for (int i = 0; i < count; i++) - output[start + i] = bestIdx[i]; + output[o + i] = bestIdx[i]; } /// Four rows against centroids [from, to): two centroids per step, eight independent accumulators. @@ -504,7 +487,7 @@ private static void Offer(Span best, Span bestIdx, int g, float scor if (score > best[g]) { best[g] = score; bestIdx[g] = c; } } - /// Dot product of two equal-length, lane-padded spans. + /// Dot product of two equal-length, lane-padded spans (centroid ranking and k-means only). private static float Dot(ReadOnlySpan a, ReadOnlySpan b) { var va = MemoryMarshal.Cast>(a); @@ -526,146 +509,45 @@ private static float Dot(ReadOnlySpan a, ReadOnlySpan b) // -- query ------------------------------------------------------------------ /// - /// Score the rows in the cells nearest the query, - /// plus every pending row, into the candidate buffers (rows that pass the - /// type filter and ). Returns false for a zero - /// query, which has no direction to rank cells by. Allocates nothing per row. + /// Feed the rows of the cells nearest the query, + /// plus every pending row, to (rows of the type + /// , or all when it is negative). Allocates only + /// its own scratch, so concurrent calls are safe. The caller has checked + /// the query has a direction (a non-zero norm). /// - public bool Scan(float[] query, string? nodeType, float threshold, int lists, out int passing) + public void Scan(VectorTable t, float[] query, int code, int lists, ref TV visitor) + where TV : struct, IRowVisitor { - passing = 0; - _candCount = 0; - query.AsSpan(0, _dim).CopyTo(_q); - float qnorm = MathF.Sqrt(Dot(_q, _q)); - if (qnorm == 0f) return false; - int code = -1; - if (nodeType is not null && !_codeOf.TryGetValue(nodeType, out code)) - return true; // no row has this type - - int stride = _stride; - int nlist = Nlist; - var centroids = _centroids; - for (int c = 0; c < nlist; c++) + int nlist = Nlist, stride = _stride; + // the query, padded the way the centroids are + var qpad = ArrayPool.Shared.Rent(stride); + var keys = ArrayPool.Shared.Rent(nlist); + try { - _cKey[c] = -Dot(_q, new ReadOnlySpan(centroids, c * stride, stride)); - _cId[c] = c; - } - if (lists < nlist) - MemoryExtensions.Sort(_cKey.AsSpan(0, nlist), _cId.AsSpan(0, nlist)); - else - lists = nlist; - - var matrix = _matrix; - var norms = _norms; - var codes = _codes; - var assign = _assign; - var order = _order; - var offsets = _offsets; - for (int l = 0; l < lists; l++) - { - int cell = _cId[l]; - for (int p = offsets[cell], end = offsets[cell + 1]; p < end; p++) + Array.Clear(qpad, 0, stride); + Array.Copy(query, qpad, _dim); + var q = new ReadOnlySpan(qpad, 0, stride); + for (int c = 0; c < nlist; c++) { - int row = order[p]; - if (assign[row] != cell) continue; // deleted or updated since the last regroup - if (code >= 0 && codes[row] != code) continue; - Consider(row, qnorm, threshold); + float s = Dot(q, new ReadOnlySpan(_centroids, c * stride, stride)); + int bits = BitConverter.SingleToInt32Bits(s); + uint ordered = bits < 0 ? ~(uint)bits : (uint)bits | 0x80000000u; // ascending = ascending score + keys[c] = ((ulong)~ordered << 32) | (uint)c; // best score first, then lowest cell } - } - foreach (int row in _pending) - { - if (code >= 0 && codes[row] != code) continue; - Consider(row, qnorm, threshold); - } - passing = _candCount; - return true; - } - - private void Consider(int row, float qnorm, float threshold) - { - float dot = Dot(_q, new ReadOnlySpan(_matrix, row * _stride, _stride)); - float denom = _norms[row] * qnorm; - float score = denom > 0f ? dot / denom : 0f; - if (!(score >= threshold)) return; - if (_candCount == _candRows.Length) - { - Array.Resize(ref _candRows, _candCount * 2); - Array.Resize(ref _candScores, _candCount * 2); - } - _candRows[_candCount] = row; - _candScores[_candCount++] = score; - } + if (lists < nlist) Array.Sort(keys, 0, nlist); + else lists = nlist; - private static bool Better(float s1, int r1, float s2, int r2) => s1 > s2 || (s1 == s2 && r1 < r2); - - /// - /// The best of the last , best first; - /// equal scores keep row (insertion) order, as in the exact scan. - /// - public List TopK(int k) - { - if (_topRows.Length < k) - { - _topRows = new int[k]; - _topScores = new float[k]; - } - var topRows = _topRows; - var topScores = _topScores; - int m = 0; - for (int i = 0; i < _candCount; i++) - { - float s = _candScores[i]; - int r = _candRows[i]; - if (m == k && !Better(s, r, topScores[k - 1], topRows[k - 1])) continue; - int pos = m < k ? m++ : k - 1; - while (pos > 0 && Better(s, r, topScores[pos - 1], topRows[pos - 1])) + for (int l = 0; l < lists; l++) { - topScores[pos] = topScores[pos - 1]; - topRows[pos] = topRows[pos - 1]; - pos--; + int cell = (int)(keys[l] & 0xFFFFFFFFu); + RowScan.Slots(t, query, code, _lists[cell], 0, _listLen[cell], ref visitor); } - topScores[pos] = s; - topRows[pos] = r; + RowScan.Slots(t, query, code, _pend, 0, _pendLen, ref visitor); } - var results = new List(m); - for (int i = 0; i < m; i++) + finally { - var rec = _recs[topRows[i]]!; - results.Add(new SimilarityResult(rec.NodeRef, topScores[i], rec.Text)); + ArrayPool.Shared.Return(qpad); + ArrayPool.Shared.Return(keys); } - return results; - } - - /// - /// Every candidate of the last , best first, as a - /// snapshot that stays valid however the index changes afterwards. - /// - public IEnumerable RankedSnapshot() - { - int m = _candCount; - var keys = new ulong[m]; - for (int i = 0; i < m; i++) - { - int bits = BitConverter.SingleToInt32Bits(_candScores[i]); - uint ordered = bits < 0 ? ~(uint)bits : (uint)bits | 0x80000000u; // ascending = ascending score - keys[i] = ((ulong)~ordered << 32) | (uint)_candRows[i]; // best score, then lowest row, first - } - Array.Sort(keys); - var recs = new EmbeddingRecord[m]; - var scores = new float[m]; - for (int i = 0; i < m; i++) - { - uint ordered = ~(uint)(keys[i] >> 32); - int bits = (ordered & 0x80000000u) != 0 ? (int)(ordered & 0x7FFFFFFFu) : (int)~ordered; - scores[i] = BitConverter.Int32BitsToSingle(bits); - recs[i] = _recs[(int)(keys[i] & 0xFFFFFFFFu)]!; - } - return Enumerate(recs, scores); - } - - private static IEnumerable Enumerate(EmbeddingRecord[] recs, float[] scores) - { - for (int i = 0; i < recs.Length; i++) - yield return new SimilarityResult(recs[i].NodeRef, scores[i], recs[i].Text); } } diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/Scan.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/Scan.cs new file mode 100644 index 0000000..3af375c --- /dev/null +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/Scan.cs @@ -0,0 +1,363 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace IsonGraph.Vector; + +/// A scored row: the unit every search ranks, ordered by (score desc, slot asc). +internal readonly struct Hit +{ + public readonly float Score; + public readonly int Slot; + public Hit(float score, int slot) { Score = score; Slot = slot; } + + /// Best first: higher score, then lower slot (the one total order every search uses). + public static int BestFirst(Hit a, Hit b) + { + if (a.Score > b.Score) return -1; + if (a.Score < b.Score) return 1; + return a.Slot.CompareTo(b.Slot); + } +} + +/// +/// Bounded collector for the best k hits: a min-heap whose root is the worst +/// kept, so most rows are rejected by one compare against the root. +/// Order is (score desc, slot asc), a total order, so the answer does not +/// depend on the order rows are offered in (threads, tiling, cell order). +/// +internal sealed class TopKHeap +{ + private readonly float[] _scores; + private readonly int[] _slots; + private int _count; + + public TopKHeap(int k) + { + _scores = new float[k]; + _slots = new int[k]; + } + + public int Count => _count; + public bool Full => _count == _scores.Length; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Worse(float s1, int r1, float s2, int r2) => s1 < s2 || (s1 == s2 && r1 > r2); + + public void Offer(float score, int slot) + { + if (_count == _scores.Length) + { + if (score < _scores[0] || !Worse(_scores[0], _slots[0], score, slot)) return; + _scores[0] = score; + _slots[0] = slot; + SiftDown(0); + return; + } + int i = _count++; + _scores[i] = score; + _slots[i] = slot; + while (i > 0) + { + int parent = (i - 1) >> 1; + if (!Worse(_scores[i], _slots[i], _scores[parent], _slots[parent])) break; + Swap(i, parent); + i = parent; + } + } + + private void SiftDown(int i) + { + for (;;) + { + int l = 2 * i + 1, r = l + 1, worst = i; + if (l < _count && Worse(_scores[l], _slots[l], _scores[worst], _slots[worst])) worst = l; + if (r < _count && Worse(_scores[r], _slots[r], _scores[worst], _slots[worst])) worst = r; + if (worst == i) return; + Swap(i, worst); + i = worst; + } + } + + private void Swap(int a, int b) + { + (_scores[a], _scores[b]) = (_scores[b], _scores[a]); + (_slots[a], _slots[b]) = (_slots[b], _slots[a]); + } + + public void MergeFrom(TopKHeap other) + { + for (int i = 0; i < other._count; i++) Offer(other._scores[i], other._slots[i]); + } + + /// The kept hits, best first. + public Hit[] ToSortedArray() + { + var hits = new Hit[_count]; + for (int i = 0; i < _count; i++) hits[i] = new Hit(_scores[i], _slots[i]); + Array.Sort(hits, Hit.BestFirst); + return hits; + } +} + +/// Receives (slot, dot) for every row a scan reaches; turns it into a score and keeps or drops it. +internal interface IRowVisitor +{ + void Visit(int slot, float dot); +} + +/// Cosine score of a dot product: 0 when the norms' product is 0, NaN for a tombstone (NaN norm). +internal static class Cosine +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Score(float dot, float queryNorm, float rowNorm) + { + float denom = queryNorm * rowNorm; + return denom == 0f ? 0f : dot / denom; + } +} + +/// Visitor that keeps the best k rows with score >= threshold. +internal struct TopKVisitor : IRowVisitor +{ + public TopKHeap Heap; + public float[] Norms; + public float QueryNorm; + public float Threshold; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Visit(int slot, float dot) + { + float score = Cosine.Score(dot, QueryNorm, Norms[slot]); + if (score >= Threshold) Heap.Offer(score, slot); + } +} + +/// Visitor that keeps every row with score >= threshold. +internal sealed class HitList +{ + public Hit[] Items = new Hit[256]; + public int Count; + + public void Add(float score, int slot) + { + if (Count == Items.Length) Array.Resize(ref Items, Count * 2); + Items[Count++] = new Hit(score, slot); + } + + public Hit[] ToSortedArray() + { + var hits = new Hit[Count]; + Array.Copy(Items, hits, Count); + Array.Sort(hits, Hit.BestFirst); + return hits; + } +} + +internal struct CollectVisitor : IRowVisitor +{ + public HitList List; + public float[] Norms; + public float QueryNorm; + public float Threshold; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Visit(int slot, float dot) + { + float score = Cosine.Score(dot, QueryNorm, Norms[slot]); + if (score >= Threshold) List.Add(score, slot); + } +} + +/// Visitor that records every score, NaN ones included (the score map). +internal struct ScoreAllVisitor : IRowVisitor +{ + public float[] Scores; + public float[] Norms; + public float QueryNorm; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Visit(int slot, float dot) => Scores[slot] = Cosine.Score(dot, QueryNorm, Norms[slot]); +} + +/// +/// The dot-product kernels and the loops that feed them. +/// +/// One summation order everywhere: every (query, row) dot is accumulated +/// sequentially in ascending dimension, one multiply then one add per element, +/// never fused. The kernels differ only in how many rows they walk at once +/// (independent accumulators, so the adds overlap in the pipeline instead of +/// each waiting on the previous one), so a single query, a batch and any thread +/// count produce the same bits, and they are the bits the store has always +/// produced. +/// +internal static class RowScan +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Dot1(ref float q, ref float r, int dim) + { + float s = 0f; + for (int i = 0; i < dim; i++) s += Unsafe.Add(ref q, i) * Unsafe.Add(ref r, i); + return s; + } + + public static void Dot4(ref float q, ref float r0, ref float r1, ref float r2, ref float r3, int dim, + out float d0, out float d1, out float d2, out float d3) + { + float s0 = 0f, s1 = 0f, s2 = 0f, s3 = 0f; + for (int i = 0; i < dim; i++) + { + float x = Unsafe.Add(ref q, i); + s0 += x * Unsafe.Add(ref r0, i); + s1 += x * Unsafe.Add(ref r1, i); + s2 += x * Unsafe.Add(ref r2, i); + s3 += x * Unsafe.Add(ref r3, i); + } + d0 = s0; d1 = s1; d2 = s2; d3 = s3; + } + + /// Eight consecutive rows (stride ) at once. + public static void Dot8(ref float q, ref float rows, int dim, Span d) + { + ref float r0 = ref rows; + ref float r1 = ref Unsafe.Add(ref rows, dim); + ref float r2 = ref Unsafe.Add(ref rows, 2 * dim); + ref float r3 = ref Unsafe.Add(ref rows, 3 * dim); + ref float r4 = ref Unsafe.Add(ref rows, 4 * dim); + ref float r5 = ref Unsafe.Add(ref rows, 5 * dim); + ref float r6 = ref Unsafe.Add(ref rows, 6 * dim); + ref float r7 = ref Unsafe.Add(ref rows, 7 * dim); + float s0 = 0f, s1 = 0f, s2 = 0f, s3 = 0f, s4 = 0f, s5 = 0f, s6 = 0f, s7 = 0f; + for (int i = 0; i < dim; i++) + { + float x = Unsafe.Add(ref q, i); + s0 += x * Unsafe.Add(ref r0, i); + s1 += x * Unsafe.Add(ref r1, i); + s2 += x * Unsafe.Add(ref r2, i); + s3 += x * Unsafe.Add(ref r3, i); + s4 += x * Unsafe.Add(ref r4, i); + s5 += x * Unsafe.Add(ref r5, i); + s6 += x * Unsafe.Add(ref r6, i); + s7 += x * Unsafe.Add(ref r7, i); + } + d[0] = s0; d[1] = s1; d[2] = s2; d[3] = s3; d[4] = s4; d[5] = s5; d[6] = s6; d[7] = s7; + } + + /// Eight rows anywhere in memory (a cell's rows are scattered across chunks) at once. + public static void Dot8Gather(ref float q, ref float r0, ref float r1, ref float r2, ref float r3, + ref float r4, ref float r5, ref float r6, ref float r7, int dim, Span d) + { + float s0 = 0f, s1 = 0f, s2 = 0f, s3 = 0f, s4 = 0f, s5 = 0f, s6 = 0f, s7 = 0f; + for (int i = 0; i < dim; i++) + { + float x = Unsafe.Add(ref q, i); + s0 += x * Unsafe.Add(ref r0, i); + s1 += x * Unsafe.Add(ref r1, i); + s2 += x * Unsafe.Add(ref r2, i); + s3 += x * Unsafe.Add(ref r3, i); + s4 += x * Unsafe.Add(ref r4, i); + s5 += x * Unsafe.Add(ref r5, i); + s6 += x * Unsafe.Add(ref r6, i); + s7 += x * Unsafe.Add(ref r7, i); + } + d[0] = s0; d[1] = s1; d[2] = s2; d[3] = s3; d[4] = s4; d[5] = s5; d[6] = s6; d[7] = s7; + } + + /// Sequential sum of squares (the norm before the square root). + public static float SumSq(ReadOnlySpan v) + { + float s = 0f; + for (int i = 0; i < v.Length; i++) s += v[i] * v[i]; + return s; + } + + /// + /// Visit every row of chunks [fromChunk, toChunk) whose type code matches + /// ( < 0 = any). + /// + public static void Chunks(VectorTable t, float[] query, int code, int fromChunk, int toChunk, ref TV visitor) + where TV : struct, IRowVisitor + { + int dim = t.Dim, n = t.Slots; + var codes = t.Codes; + ref float q = ref MemoryMarshal.GetArrayDataReference(query); + Span d = stackalloc float[8]; + for (int c = fromChunk; c < toChunk; c++) + { + var data = t.Chunk(c); + int baseSlot = c << 10; + int rows = Math.Min(VectorTable.ChunkRows, n - baseSlot); + ref float chunk = ref MemoryMarshal.GetArrayDataReference(data); + int i = 0; + for (; i + 8 <= rows; i += 8) + { + if (code >= 0 && !AnyCode(codes, baseSlot + i, 8, code)) continue; + Dot8(ref q, ref Unsafe.Add(ref chunk, i * dim), dim, d); + for (int j = 0; j < 8; j++) + { + int slot = baseSlot + i + j; + if (code >= 0 && codes[slot] != code) continue; + visitor.Visit(slot, d[j]); + } + } + for (; i < rows; i++) + { + int slot = baseSlot + i; + if (code >= 0 && codes[slot] != code) continue; + visitor.Visit(slot, Dot1(ref q, ref Unsafe.Add(ref chunk, i * dim), dim)); + } + } + } + + private static bool AnyCode(int[] codes, int start, int count, int code) + { + for (int i = 0; i < count; i++) if (codes[start + i] == code) return true; + return false; + } + + /// Visit the rows named by [from, to) (a cell, or the pending rows). + public static void Slots(VectorTable t, float[] query, int code, int[] slots, int from, int to, ref TV visitor) + where TV : struct, IRowVisitor + { + int dim = t.Dim; + var codes = t.Codes; + ref float q = ref MemoryMarshal.GetArrayDataReference(query); + int i = from; + Span d8 = stackalloc float[8]; + for (; i + 8 <= to; i += 8) + { + int s0 = slots[i], s1 = slots[i + 1], s2 = slots[i + 2], s3 = slots[i + 3]; + int s4 = slots[i + 4], s5 = slots[i + 5], s6 = slots[i + 6], s7 = slots[i + 7]; + if (code >= 0 && codes[s0] != code && codes[s1] != code && codes[s2] != code && codes[s3] != code && + codes[s4] != code && codes[s5] != code && codes[s6] != code && codes[s7] != code) + continue; + Dot8Gather(ref q, ref t.RowRef(s0), ref t.RowRef(s1), ref t.RowRef(s2), ref t.RowRef(s3), + ref t.RowRef(s4), ref t.RowRef(s5), ref t.RowRef(s6), ref t.RowRef(s7), dim, d8); + if (code < 0 || codes[s0] == code) visitor.Visit(s0, d8[0]); + if (code < 0 || codes[s1] == code) visitor.Visit(s1, d8[1]); + if (code < 0 || codes[s2] == code) visitor.Visit(s2, d8[2]); + if (code < 0 || codes[s3] == code) visitor.Visit(s3, d8[3]); + if (code < 0 || codes[s4] == code) visitor.Visit(s4, d8[4]); + if (code < 0 || codes[s5] == code) visitor.Visit(s5, d8[5]); + if (code < 0 || codes[s6] == code) visitor.Visit(s6, d8[6]); + if (code < 0 || codes[s7] == code) visitor.Visit(s7, d8[7]); + } + for (; i + 4 <= to; i += 4) + { + int s0 = slots[i], s1 = slots[i + 1], s2 = slots[i + 2], s3 = slots[i + 3]; + if (code >= 0 && codes[s0] != code && codes[s1] != code && codes[s2] != code && codes[s3] != code) + continue; + Dot4(ref q, ref t.RowRef(s0), ref t.RowRef(s1), ref t.RowRef(s2), ref t.RowRef(s3), dim, + out float d0, out float d1, out float d2, out float d3); + if (code < 0 || codes[s0] == code) visitor.Visit(s0, d0); + if (code < 0 || codes[s1] == code) visitor.Visit(s1, d1); + if (code < 0 || codes[s2] == code) visitor.Visit(s2, d2); + if (code < 0 || codes[s3] == code) visitor.Visit(s3, d3); + } + for (; i < to; i++) + { + int s = slots[i]; + if (code >= 0 && codes[s] != code) continue; + visitor.Visit(s, Dot1(ref q, ref t.RowRef(s), dim)); + } + } +} diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/VectorSnapshot.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/VectorSnapshot.cs new file mode 100644 index 0000000..2a8a203 --- /dev/null +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/VectorSnapshot.cs @@ -0,0 +1,499 @@ +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using IsonGraph; + +namespace IsonGraph.Vector; + +// Optional binary snapshot of an EmbeddingStore, written next to (never +// instead of) the JSON embeddings sidecar. DESIGN.md section 11 is the +// normative byte layout; everything here is little endian explicitly and +// IEEE-754 binary32, whatever the host. + +/// Where a load's embeddings came from. +public enum LoadSource +{ + /// The binary snapshot. + Snapshot, + /// The JSON sidecar. + Json, + /// Nothing was restored (embeddings off, or no sidecar). + NoEmbeddings, +} + +/// What did. +/// Where the embeddings came from. +/// +/// Why a snapshot that exists was not used (and the JSON sidecar was read +/// instead); null when it was used or there is none. +/// +/// Embeddings restored. +/// Wall time of the whole load. +public sealed record LoadReport(LoadSource Source, string? SnapshotSkipped, int Embeddings, double Seconds); + +/// +/// A cheap identity for a sidecar file: its length and a SHA-256 over its +/// length, its first and last 64 KiB and 64 probes of 4 KiB spread over the +/// rest. Any rewrite of the sidecar by a serializer (every float re-rendered) +/// changes it; a hand edit that keeps the length and avoids every sampled byte +/// does not, which is why a save that does not write a snapshot deletes the +/// old one. +/// +public sealed class SidecarId : IEquatable +{ + private const int Edge = 64 * 1024; + private const int Probe = 4 * 1024; + private const int Probes = 64; + + public long Length { get; } + + /// 32 bytes. + public byte[] Fingerprint { get; } + + public SidecarId(long length, byte[] fingerprint) + { + if (fingerprint.Length != 32) throw new ArgumentException("a fingerprint is 32 bytes", nameof(fingerprint)); + Length = length; + Fingerprint = fingerprint; + } + + /// Read the identity of the file at . + public static SidecarId OfFile(string path) + { + using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + long len = file.Length; + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + Span eight = stackalloc byte[8]; + BinaryPrimitives.WriteInt64LittleEndian(eight, len); + hash.AppendData(eight); + var buf = new byte[Edge]; + void Take(long at, int n) + { + n = (int)Math.Min(n, Math.Max(0, len - at)); + file.Seek(at, SeekOrigin.Begin); + file.ReadExactly(buf, 0, n); + hash.AppendData(buf, 0, n); + } + Take(0, Edge); + Take(Math.Max(0, len - Edge), Edge); + for (int i = 1; i <= Probes; i++) + Take((long)((UInt128)(ulong)len * (ulong)i / (Probes + 1)), Probe); + return new SidecarId(len, hash.GetHashAndReset()); + } + + public bool Equals(SidecarId? other) => + other is not null && Length == other.Length && Fingerprint.AsSpan().SequenceEqual(other.Fingerprint); + + public override bool Equals(object? obj) => Equals(obj as SidecarId); + public override int GetHashCode() => HashCode.Combine(Length, BitConverter.ToInt32(Fingerprint, 0)); +} + +/// What a snapshot is tied to. +/// Digest of the graph file (the JSON sidecar records the same). +/// The JSON sidecar written for the same save. +public sealed record SnapshotIdentity(string? GraphDigest, SidecarId? Sidecar); + +/// The header of a snapshot, and its size. +public sealed record SnapshotInfo( + int Version, int Dimension, int Count, string? GraphDigest, string? GraphDigestBasis, + string? EncoderModel, string? EncoderVersion, SidecarId? Sidecar, long Bytes); + +/// A snapshot that cannot be used, with the reason. +internal sealed class SnapshotRejected : Exception +{ + public SnapshotRejected(string message) : base(message) { } +} + +/// Buffers writes, hashing every byte that goes through it. +internal sealed class SnapshotWriter +{ + private readonly Stream _inner; + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + private readonly byte[] _buf = new byte[1 << 22]; + private int _pos; + public long Written { get; private set; } + + public SnapshotWriter(Stream inner) => _inner = inner; + + private void Drain() + { + if (_pos == 0) return; + _hash.AppendData(_buf, 0, _pos); + _inner.Write(_buf, 0, _pos); + Written += _pos; + _pos = 0; + } + + public void Bytes(ReadOnlySpan data) + { + if (data.Length > _buf.Length - _pos) Drain(); + if (data.Length > _buf.Length) + { + _hash.AppendData(data); + _inner.Write(data); + Written += data.Length; + return; + } + data.CopyTo(_buf.AsSpan(_pos)); + _pos += data.Length; + } + + public void U32(uint v) + { + Span b = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(b, v); + Bytes(b); + } + + public void U64(ulong v) + { + Span b = stackalloc byte[8]; + BinaryPrimitives.WriteUInt64LittleEndian(b, v); + Bytes(b); + } + + public void Str(string s) + { + int n = Encoding.UTF8.GetByteCount(s); + U32((uint)n); + if (n <= _buf.Length - _pos) { Encoding.UTF8.GetBytes(s, _buf.AsSpan(_pos)); _pos += n; } + else Bytes(Encoding.UTF8.GetBytes(s)); + } + + public void OptStr(string? s) + { + if (s is null) U32(uint.MaxValue); else Str(s); + } + + public void Floats(ReadOnlySpan v) + { + if (BitConverter.IsLittleEndian) { Bytes(MemoryMarshal.AsBytes(v)); return; } + Span b = stackalloc byte[4]; + foreach (float f in v) + { + BinaryPrimitives.WriteSingleLittleEndian(b, f); + Bytes(b); + } + } + + /// Flush the body and append its SHA-256 (not itself hashed). + public long Finish() + { + Drain(); + var digest = _hash.GetCurrentHash(); + _inner.Write(digest); + Written += digest.Length; + return Written; + } +} + +/// Reads a snapshot body, hashing everything it consumes and never reading past the trailer. +internal sealed class SnapshotReader : IDisposable +{ + private static readonly UTF8Encoding Strict = new(false, true); + private readonly FileStream _file; + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + private long _consumed; + private readonly long _body; + + public long FileLength { get; } + + public SnapshotReader(string path) + { + _file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 22, FileOptions.SequentialScan); + FileLength = _file.Length; + if (FileLength < 32 + 8) + { + _file.Dispose(); + _hash.Dispose(); + throw new SnapshotRejected("file too short"); + } + _body = FileLength - 32; + } + + public long Remaining => Math.Max(0, _body - _consumed); + + public void Exact(Span dest) + { + if (dest.Length > Remaining) throw new SnapshotRejected("truncated"); + _file.ReadExactly(dest); + _hash.AppendData(dest); + _consumed += dest.Length; + } + + public uint U32() + { + Span b = stackalloc byte[4]; + Exact(b); + return BinaryPrimitives.ReadUInt32LittleEndian(b); + } + + public ulong U64() + { + Span b = stackalloc byte[8]; + Exact(b); + return BinaryPrimitives.ReadUInt64LittleEndian(b); + } + + /// A length-prefixed string; lets the length be 0xFFFFFFFF (absent). + public string? Str(bool optional) + { + uint len = U32(); + if (len == uint.MaxValue) + return optional ? null : throw new SnapshotRejected("a required string is marked absent"); + if (len > Remaining) throw new SnapshotRejected("a string is longer than the file"); + Span b = len <= 1024 ? stackalloc byte[(int)len] : new byte[len]; + Exact(b); + try { return Strict.GetString(b); } + catch (DecoderFallbackException) { throw new SnapshotRejected("a string is not UTF-8"); } + } + + /// Read dest.Length little-endian IEEE-754 floats. + public void Floats(Span dest) + { + var bytes = MemoryMarshal.AsBytes(dest); + Exact(bytes); + if (!BitConverter.IsLittleEndian) + for (int i = 0; i < dest.Length; i++) + dest[i] = BitConverter.Int32BitsToSingle(BinaryPrimitives.ReverseEndianness(BitConverter.SingleToInt32Bits(dest[i]))); + } + + /// The trailer must equal the hash of the body, and nothing may be left between them. + public void VerifyTrailer() + { + if (Remaining != 0) throw new SnapshotRejected("bytes left over after the last block"); + var expected = _hash.GetCurrentHash(); + Span stored = stackalloc byte[32]; + _file.ReadExactly(stored); + if (!stored.SequenceEqual(expected)) throw new SnapshotRejected("checksum mismatch"); + } + + public void Dispose() + { + _file.Dispose(); + _hash.Dispose(); + } +} + +public sealed partial class EmbeddingStore +{ + private const int SnapshotBlockRows = 1024; + private static readonly byte[] SnapshotMagic = "ISONVSNP"u8.ToArray(); + + /// + /// Write a binary snapshot of every live embedding to , + /// atomically (a temp file, flushed, renamed over : a + /// failure or crash leaves the previous file untouched). Rows are written in + /// the order the JSON sidecar uses (node type, then id), so loading a + /// snapshot gives the same store, slot for slot, as loading the sidecar. + /// ties it to the graph and sidecar it was saved + /// with; a load that finds a different identity refuses the snapshot. + /// Holds the shared lock while it writes. + /// + public SnapshotInfo SaveSnapshot(string path, SnapshotIdentity? identity = null) + { + SnapshotInfo? info = null; + AtomicFile.Write(path, stream => info = WriteSnapshot(stream, identity ?? new SnapshotIdentity(null, null))); + return info!; + } + + private SnapshotInfo WriteSnapshot(Stream stream, SnapshotIdentity identity) + { + _rw.EnterReadLock(); + try + { + var t = _table; + var slots = t is null ? Array.Empty() : SortedLiveSlots(t); + int dim = t?.Dim ?? 0; + + // models, in order of first appearance among the rows as written + var models = new List(); + var modelIndex = new Dictionary(); + foreach (int s in slots) + { + var m = t!.ModelAt(s); + if (!modelIndex.ContainsKey(m)) { modelIndex[m] = (uint)models.Count; models.Add(m); } + } + + string? basis = identity.GraphDigest is null ? null : IsonGraphVectorVersion.GraphDigestBasis; + string? encoderModel = CurrentModel(), encoderVersion = Version; + var w = new SnapshotWriter(stream); + w.Bytes(SnapshotMagic); + w.U32(1); + w.U32(identity.Sidecar is null ? 0u : 1u); + w.U32((uint)dim); + w.U32(SnapshotBlockRows); + w.U64((ulong)slots.Length); + w.OptStr(identity.GraphDigest); + w.OptStr(basis); + w.OptStr(encoderModel); + w.OptStr(encoderVersion); + if (identity.Sidecar is not null) + { + w.U64((ulong)identity.Sidecar.Length); + w.Bytes(identity.Sidecar.Fingerprint); + } + w.U32((uint)models.Count); + foreach (var m in models) w.Str(m); + + for (int at = 0; at < slots.Length; at += SnapshotBlockRows) + { + int end = Math.Min(slots.Length, at + SnapshotBlockRows); + for (int i = at; i < end; i++) + { + int s = slots[i]; + var r = t!.RefAt(s); + w.Str(r.Type); + w.Str(r.Id); + w.Str(t.TextAt(s)); + w.U32(modelIndex[t.ModelAt(s)]); + } + for (int i = at; i < end; i++) w.Floats(t!.Row(slots[i])); + } + long bytes = w.Finish(); + return new SnapshotInfo(1, dim, slots.Length, identity.GraphDigest, basis, encoderModel, encoderVersion, + identity.Sidecar, bytes); + } + finally { _rw.ExitReadLock(); } + } + + /// + /// Replace the store's embeddings with those of a snapshot file. Nothing + /// changes unless the whole file verifies (structure and checksum); the + /// approximate-index settings are kept (the index is rebuilt lazily). Does + /// not check identity: does. + /// + /// The file is not a valid snapshot. + public SnapshotInfo LoadSnapshot(string path) + { + try + { + var (stage, info) = ReadSnapshot(path, _ => null); + Adopt(stage, info); + return info; + } + catch (SnapshotRejected ex) + { + throw new EmbeddingException($"invalid snapshot: {ex.Message}"); + } + } + + /// + /// Use the snapshot at for the graph whose digest is + /// and whose sidecar is at , + /// if it belongs to them. False, with the reason, leaves the store untouched. + /// + internal bool TryAdoptSnapshot(string snap, string sidecar, string graphDigest, out SnapshotInfo? info, out string? skipped) + { + info = null; + skipped = null; + try + { + var current = SidecarId.OfFile(sidecar); + var (stage, header) = ReadSnapshot(snap, h => + { + if (h.GraphDigest != graphDigest || h.GraphDigestBasis != IsonGraphVectorVersion.GraphDigestBasis) + return "it was saved for a different graph (digest differs)"; + if (h.Sidecar is null) return "it records no sidecar identity"; + if (!h.Sidecar.Equals(current)) + return "the JSON sidecar changed since the snapshot was written (stale snapshot)"; + return null; + }); + Adopt(stage, header); + info = header; + return true; + } + catch (SnapshotRejected ex) { skipped = ex.Message; } + catch (IOException ex) { skipped = ex.Message; } + catch (UnauthorizedAccessException ex) { skipped = ex.Message; } + return false; + } + + private void Adopt(VectorTable? stage, SnapshotInfo info) + { + _rw.EnterWriteLock(); + try + { + _table = stage; + _dimension = stage is null ? -1 : info.Dimension; + _ivf = null; + var meta = new Dictionary(); + if (info.EncoderModel is not null) meta["encoder_model"] = info.EncoderModel; + if (info.EncoderVersion is not null) meta["encoder_version"] = info.EncoderVersion; + if (info.GraphDigest is not null) meta["graph_digest"] = info.GraphDigest; + if (info.GraphDigestBasis is not null) meta["graph_digest_basis"] = info.GraphDigestBasis; + _lastImportMeta = meta; + } + finally { _rw.ExitWriteLock(); } + } + + /// + /// Read and fully verify the snapshot at , after + /// (a reason to refuse, or null) has approved its + /// header. Returns the rows in a fresh staging table; the store is not touched. + /// + private static (VectorTable? Stage, SnapshotInfo Info) ReadSnapshot(string path, Func accept) + { + using var r = new SnapshotReader(path); + Span magic = stackalloc byte[8]; + r.Exact(magic); + if (!magic.SequenceEqual(SnapshotMagic)) throw new SnapshotRejected("not a snapshot (bad magic)"); + uint version = r.U32(); + if (version != 1) + throw new SnapshotRejected($"unsupported snapshot version {version} (this build reads 1)"); + uint flags = r.U32(); + if ((flags & ~1u) != 0) throw new SnapshotRejected($"unknown flags 0x{flags:x}"); + uint dim = r.U32(); + uint blockRows = r.U32(); + ulong count = r.U64(); + if (blockRows == 0 || blockRows > 1u << 20) throw new SnapshotRejected($"implausible block size {blockRows}"); + if (count > int.MaxValue || (count > 0 && dim == 0) || dim > int.MaxValue) + throw new SnapshotRejected($"implausible shape {count} x {dim}"); + if ((UInt128)count * dim * 4 > (ulong)r.Remaining) + throw new SnapshotRejected("fewer bytes than the rows need"); + string? digest = r.Str(true), basis = r.Str(true), encoderModel = r.Str(true), encoderVersion = r.Str(true); + SidecarId? sidecar = null; + if ((flags & 1) != 0) + { + long len = (long)r.U64(); + var fingerprint = new byte[32]; + r.Exact(fingerprint); + sidecar = new SidecarId(len, fingerprint); + } + uint modelCount = r.U32(); + if (modelCount > r.Remaining / 4) throw new SnapshotRejected("more models than the file can hold"); + var models = new string[modelCount]; + for (int i = 0; i < models.Length; i++) models[i] = r.Str(false)!; + var info = new SnapshotInfo(1, (int)dim, (int)count, digest, basis, encoderModel, encoderVersion, sidecar, r.FileLength); + var refusal = accept(info); + if (refusal is not null) throw new SnapshotRejected(refusal); + + VectorTable? stage = count > 0 ? new VectorTable((int)dim) : null; + var meta = new (string Type, string Id, string Text, string Model)[Math.Min(blockRows, (uint)Math.Max(count, 1UL))]; + var floats = new float[(long)meta.Length * (int)dim]; + ulong left = count; + while (left > 0) + { + int rows = (int)Math.Min(left, blockRows); + for (int i = 0; i < rows; i++) + { + string type = r.Str(false)!, id = r.Str(false)!, text = r.Str(false)!; + uint model = r.U32(); + if (model >= models.Length) throw new SnapshotRejected("a row names a model that is not listed"); + meta[i] = (type, id, text, models[model]); + } + if ((ulong)rows * dim * 4 > (ulong)r.Remaining) throw new SnapshotRejected("truncated"); + r.Floats(floats.AsSpan(0, rows * (int)dim)); + for (int i = 0; i < rows; i++) + { + var (type, id, text, model) = meta[i]; + stage!.Put(new NodeRef(type, id), $"{type}:{id}", text, floats.AsSpan(i * (int)dim, (int)dim), model, out bool replaced); + if (replaced) throw new SnapshotRejected($"the key {type}:{id} appears twice"); + } + left -= (ulong)rows; + } + r.VerifyTrailer(); + return (stage, info); + } +} diff --git a/isongraph-vector-csharp/src/IsonGraph.Vector/VectorTable.cs b/isongraph-vector-csharp/src/IsonGraph.Vector/VectorTable.cs new file mode 100644 index 0000000..c195eee --- /dev/null +++ b/isongraph-vector-csharp/src/IsonGraph.Vector/VectorTable.cs @@ -0,0 +1,325 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using IsonGraph; + +namespace IsonGraph.Vector; + +/// +/// The whole store as parallel arrays over "slots": a record is a slot number, +/// and nothing else owns the floats (DESIGN.md section 1). +/// +/// The matrix is a list of chunks of rows, Dim +/// floats each, row-major. Chunking means growth never copies stored vectors +/// (only chunk 0, which starts small and doubles up to one full chunk, is ever +/// resized) and no single array reaches the large-object-heap sizes a +/// contiguous 1.5 GB matrix would need. A scan block never straddles chunks. +/// +/// Slots are in arrival order. An update overwrites its slot in place, an +/// insert takes slot n, a delete tombstones (the norm becomes NaN, so a +/// scan scores it NaN and the collector drops it without a liveness branch). +/// When more than a quarter of the slots are dead the store compacts, moving +/// live rows down in order. +/// +/// Not thread-safe: the owning store guards it (readers share, a writer is +/// alone). +/// +internal sealed class VectorTable +{ + public const int ChunkRows = 1024; + private const int ChunkShift = 10; + private const int ChunkMask = ChunkRows - 1; + private const int FirstChunkRows = 16; + + public readonly int Dim; + + private float[]?[] _chunks = new float[]?[4]; + private int _chunkCount; + private int _n; // slots in use, tombstones included + private int _dead; + + private float[] _norms = new float[16]; + private int[] _codes = new int[16]; + private NodeRef[] _refs = new NodeRef[16]; + private string?[] _keys = new string?[16]; // null = tombstone + private string[] _texts = new string[16]; + private string[] _models = new string[16]; + + private readonly Dictionary _index = new(); // key -> slot + private readonly Dictionary _codeOf = new(); // node type -> code + private readonly List _typeNames = new(); + private readonly Dictionary _modelNames = new(); // interned model names + private readonly Dictionary _modelCounts = new(); + + public VectorTable(int dim) + { + if (dim <= 0) throw new ArgumentOutOfRangeException(nameof(dim)); + if ((long)dim * ChunkRows > Array.MaxLength) + throw new EmbeddingException( + $"vectors of {dim} dimensions are too wide for a {ChunkRows}-row chunk"); + Dim = dim; + } + + public int Slots => _n; + public int Live => _n - _dead; + public int Dead => _dead; + public bool ShouldCompact => _dead > 0 && (long)_dead * 4 > _n; + public IReadOnlyDictionary ModelCounts => _modelCounts; + public IReadOnlyList TypeNames => _typeNames; + + public float[] Norms => _norms; + public int[] Codes => _codes; + public bool IsLive(int slot) => (uint)slot < (uint)_n && _keys[slot] is not null; + public NodeRef RefAt(int slot) => _refs[slot]; + public string KeyAt(int slot) => _keys[slot]!; + public string TextAt(int slot) => _texts[slot]; + public string ModelAt(int slot) => _models[slot]; + public string TypeOf(int slot) => _typeNames[_codes[slot]]; + + /// Code of a node type, or -1 when no row ever had that type. + public int CodeOf(string nodeType) => _codeOf.TryGetValue(nodeType, out int c) ? c : -1; + + public bool TryGetSlot(string key, out int slot) => _index.TryGetValue(key, out slot); + + /// Number of chunks holding at least one slot. + public int ChunkCount => (_n + ChunkRows - 1) >> ChunkShift; + + public float[] Chunk(int c) => _chunks[c]!; + + /// The vector of a slot (live or not), valid until the next write. + public ReadOnlySpan Row(int slot) => + new(_chunks[slot >> ChunkShift]!, (slot & ChunkMask) * Dim, Dim); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref float RowRef(int slot) => + ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(_chunks[slot >> ChunkShift]!), + (slot & ChunkMask) * Dim); + + private Span RowSpan(int slot) => + new(_chunks[slot >> ChunkShift]!, (slot & ChunkMask) * Dim, Dim); + + // -- writes ----------------------------------------------------------------- + + private string Intern(string model) + { + if (!_modelNames.TryGetValue(model, out var shared)) _modelNames[model] = shared = model; + return shared; + } + + private int CodeFor(string nodeType) + { + if (!_codeOf.TryGetValue(nodeType, out int code)) + { + code = _typeNames.Count; + _codeOf[nodeType] = code; + _typeNames.Add(nodeType); + } + return code; + } + + private void GrowMeta() + { + int cap = Math.Max(_norms.Length * 2, 16); + Array.Resize(ref _norms, cap); + Array.Resize(ref _codes, cap); + Array.Resize(ref _refs, cap); + Array.Resize(ref _keys, cap); + Array.Resize(ref _texts, cap); + Array.Resize(ref _models, cap); + } + + /// Make room for slot in the matrix. + private void EnsureRow(int slot) + { + int c = slot >> ChunkShift; + if (c >= _chunkCount) + { + if (c >= _chunks.Length) Array.Resize(ref _chunks, Math.Max(_chunks.Length * 2, c + 1)); + // chunk 0 grows like a vector; every later chunk is allocated whole + _chunks[c] = new float[(long)(c == 0 ? FirstChunkRows : ChunkRows) * Dim]; + _chunkCount = c + 1; + } + else if (c == 0) + { + var first = _chunks[0]!; + if ((slot + 1) * Dim > first.Length) + { + int rows = Math.Min(ChunkRows, Math.Max(first.Length / Dim * 2, slot + 1)); + var bigger = new float[(long)rows * Dim]; + Array.Copy(first, bigger, first.Length); + _chunks[0] = bigger; + } + } + else if (_chunks[c] is null) + { + _chunks[c] = new float[(long)ChunkRows * Dim]; + } + } + + private static float SumSq(ReadOnlySpan v) + { + float s = 0f; + for (int i = 0; i < v.Length; i++) s += v[i] * v[i]; + return s; + } + + /// + /// Store a vector under : overwrite the slot in + /// place if the key exists, else append. Returns the slot. + /// + public int Put(NodeRef nodeRef, string key, string text, ReadOnlySpan vector, string model, out bool replaced) + { + if (vector.Length != Dim) + throw new DimensionMismatchException($"vector has {vector.Length} dimensions, store holds {Dim}"); + model = Intern(model); + int code = CodeFor(nodeRef.Type); + var canonical = new NodeRef(_typeNames[code], nodeRef.Id); + + int slot; + if (_index.TryGetValue(key, out slot)) + { + replaced = true; + string oldModel = _models[slot]; + if (!ReferenceEquals(oldModel, model)) { DropModel(oldModel); AddModel(model); } + } + else + { + replaced = false; + if (_n >= _norms.Length) GrowMeta(); + slot = _n; + EnsureRow(slot); + _n++; + _index[key] = slot; + _keys[slot] = key; + AddModel(model); + } + vector.CopyTo(RowSpan(slot)); + _norms[slot] = MathF.Sqrt(SumSq(vector)); + _codes[slot] = code; + _refs[slot] = canonical; + _texts[slot] = text; + _models[slot] = model; + return slot; + } + + private void AddModel(string model) => + _modelCounts[model] = _modelCounts.GetValueOrDefault(model) + 1; + + private void DropModel(string model) + { + int c = _modelCounts[model] - 1; + if (c == 0) _modelCounts.Remove(model); else _modelCounts[model] = c; + } + + /// Tombstone a key's row. False if the key is not stored. + public bool Remove(string key, out int slot) + { + if (!_index.Remove(key, out slot)) return false; + DropModel(_models[slot]); + _keys[slot] = null; + _refs[slot] = default; + _texts[slot] = ""; + _models[slot] = ""; + _norms[slot] = float.NaN; + _dead++; + return true; + } + + public void Clear() + { + Array.Clear(_chunks); + _chunkCount = 0; + _n = 0; + _dead = 0; + _norms = new float[16]; + _codes = new int[16]; + _refs = new NodeRef[16]; + _keys = new string?[16]; + _texts = new string[16]; + _models = new string[16]; + _index.Clear(); + _modelCounts.Clear(); + // type and model name tables never shrink: they are tiny + } + + /// + /// Move live rows down over the tombstones, keeping their order. Returns + /// remap[old] = new slot (-1 for a dead one) for the indexes that + /// follow the table. + /// + public int[] Compact() + { + var remap = new int[_n]; + int kept = 0; + for (int i = 0; i < _n; i++) + { + if (_keys[i] is null) { remap[i] = -1; continue; } + remap[i] = kept; + if (kept != i) // the destination never overtakes the source + { + Row(i).CopyTo(RowSpan(kept)); + _norms[kept] = _norms[i]; + _codes[kept] = _codes[i]; + _refs[kept] = _refs[i]; + _keys[kept] = _keys[i]; + _texts[kept] = _texts[i]; + _models[kept] = _models[i]; + _index[_keys[kept]!] = kept; + } + kept++; + } + for (int i = kept; i < _n; i++) + { + _keys[i] = null; + _refs[i] = default; + _texts[i] = ""; + _models[i] = ""; + _norms[i] = float.NaN; + } + _n = kept; + _dead = 0; + int chunks = (kept + ChunkRows - 1) >> ChunkShift; + for (int c = chunks; c < _chunkCount; c++) _chunks[c] = null; // give the emptied tail chunks back + _chunkCount = chunks; + return remap; + } + + // -- checks ----------------------------------------------------------------- + + /// Every invariant of DESIGN.md section 7 that concerns the table; null when they hold. + public string? Verify() + { + if (_chunkCount < ChunkCount) return "fewer chunks than slots need"; + int dead = 0; + for (int i = 0; i < _n; i++) + { + if (_keys[i] is null) + { + dead++; + if (!float.IsNaN(_norms[i])) return $"dead slot {i} has a norm"; + continue; + } + if (!_index.TryGetValue(_keys[i]!, out int s) || s != i) return $"index disagrees about slot {i}"; + if (_keys[i] != $"{_refs[i].Type}:{_refs[i].Id}") return $"key of slot {i} is not type:id"; + if (_typeNames[_codes[i]] != _refs[i].Type) return $"type code of slot {i} is wrong"; + } + if (dead != _dead) return $"dead counter {_dead} but {dead} tombstones"; + if (_index.Count != _n - _dead) return "index size is not the live count"; + var counts = new Dictionary(); + for (int i = 0; i < _n; i++) + if (_keys[i] is not null) counts[_models[i]] = counts.GetValueOrDefault(_models[i]) + 1; + if (counts.Count != _modelCounts.Count || counts.Any(kv => _modelCounts.GetValueOrDefault(kv.Key) != kv.Value)) + return "model counts disagree with the rows"; + return null; + } + + /// Bytes held by the matrix chunks (what the vectors cost). + public long MatrixBytes + { + get + { + long total = 0; + for (int c = 0; c < _chunkCount; c++) total += (long)_chunks[c]!.Length * sizeof(float); + return total; + } + } +} diff --git a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/AnnTests.cs b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/AnnTests.cs index 585a056..0b7bf1e 100644 --- a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/AnnTests.cs +++ b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/AnnTests.cs @@ -195,7 +195,8 @@ public void Recall_floor_on_clustered_data() Assert.True(stats.Active); Assert.Equal(64, stats.Nlist); Assert.Equal(4000, stats.TrainedN); - Assert.True(stats.IndexBytes > 0 && stats.VectorBytes > 0); + Assert.True(stats.IndexBytes > 0); + Assert.Equal(0, stats.VectorBytes); // the index keeps no copy of any vector Assert.True(stats.Searches.Approximate >= 60); Assert.Equal(1, stats.Searches.Builds); } @@ -381,10 +382,11 @@ public void Pending_rows_fold_and_heavy_churn_retrains() long builds = store.GetAnnStats().Searches.Builds; var extra = gen.Draw(700); for (int i = 0; i < 700; i++) store.Add(Ref(i, "late"), "", extra[i]); - Assert.Equal(700, store.GetAnnStats().Pending); - store.SimilaritySearchVector(extra[0], 5); // >= 512 pending: folded + // every 256th write folds the pending rows into their cells (at write time, not at search time) + Assert.Equal(700 % IvfIndex.FoldBatch, store.GetAnnStats().Pending); + store.SimilaritySearchVector(extra[0], 5); var stats = store.GetAnnStats(); - Assert.Equal(0, stats.Pending); + Assert.True(stats.Pending < IvfIndex.FoldBatch); Assert.Equal(builds, stats.Searches.Builds); var more = gen.Draw(1200); diff --git a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/ReadOnlyGraphTests.cs b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/ReadOnlyGraphTests.cs index 2aeaa3f..bf387c0 100644 --- a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/ReadOnlyGraphTests.cs +++ b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/ReadOnlyGraphTests.cs @@ -64,7 +64,8 @@ public void ReturnedNodesAreDetachedCopies() Assert.Equal("Alice", g.Graph.GetNode("person", "1").Properties["name"]); Assert.Equal("Alice", g.GetEmbedding(P("1"))!.Text); - Assert.Same(before, g.GetEmbedding(P("1"))); + Assert.Equal(before.Vector, g.GetEmbedding(P("1"))!.Vector); // the store hands out copies, so equal content, not the same instance + Assert.Equal(before.Text, g.GetEmbedding(P("1"))!.Text); } [Fact] diff --git a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/SemanticGraphTests.cs b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/SemanticGraphTests.cs index df5ed70..ffc1e1d 100644 --- a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/SemanticGraphTests.cs +++ b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/SemanticGraphTests.cs @@ -724,6 +724,18 @@ public void ImportsAPayloadWrittenByThePythonPort() public class UpdateNodeTests { + // The store hands out copies of its one stored vector, so "left alone" is equal content, not one instance. + private static void AssertSameRecord(EmbeddingRecord? before, EmbeddingRecord? after) + { + Assert.NotNull(before); + Assert.NotNull(after); + Assert.Equal(before!.Id, after!.Id); + Assert.Equal(before.NodeRef, after.NodeRef); + Assert.Equal(before.Text, after.Text); + Assert.Equal(before.Model, after.Model); + Assert.Equal(before.Vector, after.Vector); + } + private static Dictionary Props(string key, object? value) => new() { [key] = value }; @@ -766,7 +778,7 @@ public void LeavesTheEmbeddingAloneWhenNoEmbedFieldChanged() graph.UpdateNode("person", "1", Props("age", 30L)); - Assert.Same(before, graph.GetEmbedding(new NodeRef("person", "1"))); + AssertSameRecord(before, graph.GetEmbedding(new NodeRef("person", "1"))); Assert.Equal(30L, graph.Graph.GetNode("person", "1").Properties["age"]); } @@ -779,7 +791,7 @@ public void LeavesTheEmbeddingAloneWhenAutoEmbedIsOff() graph.UpdateNode("person", "1", Props("name", "Zebra")); - Assert.Same(before, graph.GetEmbedding(new NodeRef("person", "1"))); + AssertSameRecord(before, graph.GetEmbedding(new NodeRef("person", "1"))); Assert.Equal("managed by the caller", before!.Text); } diff --git a/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/StageDTests.cs b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/StageDTests.cs new file mode 100644 index 0000000..61d7801 --- /dev/null +++ b/isongraph-vector-csharp/tests/IsonGraph.Vector.Tests/StageDTests.cs @@ -0,0 +1,834 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using IsonGraph; +using IsonGraph.Vector; +using Xunit; + +namespace IsonGraph.Vector.Tests; + +internal static class StageD +{ + public static float[] Vec(Random r, int dim) + { + var v = new float[dim]; + for (int i = 0; i < dim; i++) v[i] = (float)(r.NextDouble() * 2 - 1); + return v; + } + + public static (string Key, int Score, string Text)[] Shape(IReadOnlyList r) => + r.Select(x => ($"{x.NodeRef.Type}:{x.NodeRef.Id}", BitConverter.SingleToInt32Bits(x.Score), x.Text)).ToArray(); + + public static EmbeddingStore Filled(int n, int dim, int seed = 1, bool ties = true, AnnMode ann = AnnMode.None, int annMin = 0, int? nprobe = null) + { + var rng = new Random(seed); + var store = new EmbeddingStore(ann: ann, annMinSize: annMin, annNprobe: nprobe); + var keep = new List(); + for (int i = 0; i < n; i++) + { + float[] v; + if (ties && i % 11 == 3 && keep.Count > 0) v = (float[])keep[rng.Next(keep.Count)].Clone(); // exact duplicate + else if (ties && i % 29 == 7) v = new float[dim]; // zero vector + else v = Vec(rng, dim); + keep.Add(v); + store.Add(new NodeRef(i % 3 == 0 ? "a" : i % 3 == 1 ? "b" : "c", i.ToString()), "text " + i, v); + } + return store; + } + + public static string TempDir() + { + var dir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "ison-stage-d-" + Guid.NewGuid().ToString("N")[..10]); + Directory.CreateDirectory(dir); + return dir; + } + + public static string FixtureDir() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = System.IO.Path.Combine(dir.FullName, "tests", "fixtures", "snapshot"); + if (File.Exists(System.IO.Path.Combine(candidate, "golden.ison"))) return candidate; + dir = dir.Parent; + } + throw new DirectoryNotFoundException("tests/fixtures/snapshot (the golden snapshot fixture) not found above " + AppContext.BaseDirectory); + } +} + +public class OneCopyTests +{ + [Fact] + public void Invariants_hold_through_random_writes_updates_deletes_and_compaction() + { + foreach (var ann in new[] { AnnMode.None, AnnMode.Ivf }) + { + var store = new EmbeddingStore(ann: ann, annMinSize: 300, annSeed: 2); + var rng = new Random(5); + var live = new Dictionary(); + for (int step = 0; step < 6000; step++) + { + int id = rng.Next(1500); + var r = new NodeRef(id % 2 == 0 ? "x" : "y", id.ToString()); + if (rng.NextDouble() < 0.35) + { + bool removed = store.Remove(r); + Assert.Equal(live.Remove($"{r.Type}:{r.Id}"), removed); + } + else + { + var v = StageD.Vec(rng, 24); + store.Add(r, "t" + step, v); + live[$"{r.Type}:{r.Id}"] = v; + } + if (step % 250 == 0) + { + store.SimilaritySearchVector(StageD.Vec(rng, 24), 5); // builds / retrains the index when it qualifies + Assert.Null(store.VerifyInvariants()); + } + } + Assert.Null(store.VerifyInvariants()); + Assert.Equal(live.Count, store.Count); + if (ann == AnnMode.Ivf) Assert.True(store.GetAnnStats().Searches.Builds >= 1); // the index really was live through the churn + foreach (var (key, v) in live.Take(200)) + { + var parts = key.Split(':'); + Assert.Equal(v, store.Get(new NodeRef(parts[0], parts[1]))!.Vector); + } + store.Compact(); + Assert.Null(store.VerifyInvariants()); + Assert.Equal(live.Count, store.Count); + } + } + + [Fact] + public void A_store_emptied_by_deletes_compacts_and_takes_new_rows() + { + foreach (int n in new[] { 1, 5, 1500 }) + { + var store = StageD.Filled(n, 12); + for (int i = 0; i < n; i++) + Assert.True(store.Remove(new NodeRef(i % 3 == 0 ? "a" : i % 3 == 1 ? "b" : "c", i.ToString()))); + Assert.Equal(0, store.Count); + Assert.Empty(store.SimilaritySearchVector(new float[12], 5)); + Assert.Null(store.VerifyInvariants()); + store.Add(new NodeRef("a", "again"), "x", new float[12]); + Assert.Equal(1, store.Count); + Assert.Null(store.VerifyInvariants()); + Assert.Single(store.SimilaritySearchVector(new float[12], 5)); + } + } + + [Fact] + public void Get_returns_a_copy_the_store_does_not_alias() + { + var store = new EmbeddingStore(); + var original = new float[] { 1f, 2f, 3f }; + store.Add(new NodeRef("t", "1"), "x", original); + + original[0] = 99f; // the caller's array is not the store's + var record = store.Get(new NodeRef("t", "1"))!; + Assert.Equal(1f, record.Vector[0]); + + record.Vector[1] = 77f; // and neither is what Get hands back + Assert.Equal(2f, store.Get(new NodeRef("t", "1"))!.Vector[1]); + Assert.Equal(1.0, store.SimilaritySearchVector(new float[] { 1f, 2f, 3f }, 1)[0].Score, 5); + } + + [Fact] + public void The_store_holds_one_copy_of_every_vector_even_with_the_index_built() + { + const int n = 6000, dim = 64; + var store = StageD.Filled(n, dim, ann: AnnMode.Ivf, annMin: 0); + store.BuildAnnIndex(); + long raw = (long)n * dim * sizeof(float); + Assert.InRange(store.MatrixBytes, raw, raw + (long)VectorTable.ChunkRows * dim * sizeof(float)); // at most one partly empty chunk over + Assert.Equal(0, store.GetAnnStats().VectorBytes); + } + + [Fact] + public void Concurrent_readers_get_the_same_answers_as_one_reader() + { + var store = StageD.Filled(8000, 48, ann: AnnMode.Ivf, annMin: 0, nprobe: 4); + var rng = new Random(9); + var queries = Enumerable.Range(0, 64).Select(_ => StageD.Vec(rng, 48)).ToArray(); + var expected = queries.Select(q => StageD.Shape(store.SimilaritySearchVector(q, 10))).ToArray(); + var expectedExact = queries.Select(q => StageD.Shape(store.SimilaritySearchVector(q, 10, exact: true))).ToArray(); + + var errors = new List(); + var threads = Enumerable.Range(0, 8).Select(t => new Thread(() => + { + try + { + for (int round = 0; round < 5; round++) + for (int i = 0; i < queries.Length; i++) + { + Assert.Equal(expected[i], StageD.Shape(store.SimilaritySearchVector(queries[i], 10))); + Assert.Equal(expectedExact[i], StageD.Shape(store.SimilaritySearchVector(queries[i], 10, exact: true))); + } + } + catch (Exception e) { lock (errors) errors.Add(e); } + })).ToList(); + threads.ForEach(t => t.Start()); + threads.ForEach(t => t.Join()); + Assert.Empty(errors); + } + + [Fact] + public void Readers_and_a_writer_never_see_a_torn_store() + { + var store = StageD.Filled(3000, 16, ties: false); + // the writer replaces one row with a vector whose direction is exactly +x or exactly +y; a reader + // must see one or the other, never a mix, and never a result twice + var target = new NodeRef("w", "flip"); + store.Add(target, "flip", Enumerable.Range(0, 16).Select(i => i == 0 ? 1f : 0f).ToArray()); + var stop = false; + var errors = new List(); + var writer = new Thread(() => + { + try + { + for (int i = 0; !Volatile.Read(ref stop); i++) + { + var v = new float[16]; + v[i % 2] = 1f; + store.Add(target, "flip", v); + store.Add(new NodeRef("w", "churn" + (i % 50)), "c", StageD.Vec(new Random(i), 16)); + store.Remove(new NodeRef("w", "churn" + ((i + 25) % 50))); + } + } + catch (Exception e) { lock (errors) errors.Add(e); } + }); + var readers = Enumerable.Range(0, 4).Select(_ => new Thread(() => + { + try + { + var qx = Enumerable.Range(0, 16).Select(i => i == 0 ? 1f : 0f).ToArray(); + for (int i = 0; i < 400; i++) + { + var res = store.SimilaritySearchVector(qx, null, "w", -1f); + Assert.Equal(res.Count, res.Select(r => r.NodeRef).Distinct().Count()); + var flip = res.First(r => r.NodeRef == target); + Assert.True(flip.Score == 1f || flip.Score == 0f, $"torn score {flip.Score}"); + } + } + catch (Exception e) { lock (errors) errors.Add(e); } + })).ToList(); + writer.Start(); + readers.ForEach(t => t.Start()); + readers.ForEach(t => t.Join()); + stop = true; + writer.Join(); + Assert.Empty(errors); + Assert.Null(store.VerifyInvariants()); + } +} + +public class BatchSearchTests +{ + [Theory] + [InlineData(AnnMode.None, 0)] + [InlineData(AnnMode.None, 1)] + [InlineData(AnnMode.None, 3)] + [InlineData(AnnMode.Ivf, 0)] + [InlineData(AnnMode.Ivf, 2)] + public void A_batch_equals_the_single_query_answers_bit_for_bit(AnnMode ann, int degree) + { + var store = StageD.Filled(9000, 40, ann: ann, annMin: 0, nprobe: 5); + var rng = new Random(77); + var queries = Enumerable.Range(0, 50).Select(_ => StageD.Vec(rng, 40)).ToList(); + queries.Add(new float[40]); // a zero query + queries.Add(store.Get(new NodeRef("a", "3"))!.Vector); // a stored vector (tied with its duplicates) + + foreach (var (topK, type, threshold) in new (int?, string?, float)[] { (10, null, 0f), (1, "b", -1f), (200, null, 0.1f), (null, "c", 0.2f) }) + { + var batch = store.SimilaritySearchBatch(queries, topK, type, threshold, maxDegreeOfParallelism: degree); + Assert.Equal(queries.Count, batch.Count); + for (int i = 0; i < queries.Count; i++) + Assert.Equal(StageD.Shape(store.SimilaritySearchVector(queries[i], topK, type, threshold)), StageD.Shape(batch[i])); + } + } + + [Fact] + public void Scan_thread_count_does_not_change_an_answer() + { + var store = StageD.Filled(30000, 64); + var q = StageD.Vec(new Random(3), 64); + store.ScanThreads = 1; + var one = StageD.Shape(store.SimilaritySearchVector(q, 25)); + foreach (int threads in new[] { 0, 2, 3, 8 }) + { + store.ScanThreads = threads; + Assert.Equal(one, StageD.Shape(store.SimilaritySearchVector(q, 25))); + } + } + + [Fact] + public void The_text_batch_encodes_once_and_matches_single_text_searches() + { + var graph = new SemanticGraph("g", new MockEncoder(32)); + for (int i = 0; i < 400; i++) graph.AddNode("n", i.ToString(), new() { ["name"] = "item number " + i }); + var texts = new[] { "item number 7", "number 12", "zebra", "item" }; + var batch = graph.EmbeddingStore.SimilaritySearchBatch(texts, 5); + for (int i = 0; i < texts.Length; i++) + Assert.Equal(StageD.Shape(graph.EmbeddingStore.SimilaritySearch(texts[i], 5)), StageD.Shape(batch[i])); + } + + [Fact] + public void A_batch_validates_every_query_before_running_any() + { + var store = StageD.Filled(100, 8); + var bad = new List { new float[8], new float[3] }; + Assert.Throws(() => store.SimilaritySearchBatch(bad)); + Assert.Empty(store.SimilaritySearchBatch(new List())); + } +} + +public class SidecarStreamingTests +{ + [Fact] + public void Streamed_sidecar_bytes_equal_the_serializer_bytes() + { + var store = new EmbeddingStore(new MockEncoder(10) { Version = "v-2" }); + var rng = new Random(4); + string[] ids = { "1", "00007", "a:b", "é", "😀", "quote\"back\\slash", "ctrl\u0001\n\t", "<&>+" }; + foreach (var (id, i) in ids.Select((x, i) => (x, i))) + store.Add(new NodeRef(i % 2 == 0 ? "t" : "Ünï", id), id + " \u2028 text", StageD.Vec(rng, 10)); + store.Add(new NodeRef("t", "zero"), "", new float[10]); + store.Add(new NodeRef("t", "tiny"), "", Enumerable.Repeat(1e-7f, 10).ToArray()); + + foreach (string? digest in new string?[] { null, "sha256:abc" }) + { + var expected = JsonSerializer.Serialize(store.ExportEmbeddings(digest)); + using var ms = new MemoryStream(); + int count = store.WriteSidecar(ms, digest); + Assert.Equal(store.Count, count); + Assert.Equal(expected, Encoding.UTF8.GetString(ms.ToArray())); + } + + var empty = new EmbeddingStore(); + using var ms2 = new MemoryStream(); + empty.WriteSidecar(ms2, null); + Assert.Equal(JsonSerializer.Serialize(empty.ExportEmbeddings()), Encoding.UTF8.GetString(ms2.ToArray())); + } +} + +public class SnapshotTests +{ + private static SemanticGraph MakeGraph(int n = 300, int dim = 16, int seed = 3) + { + var graph = new SemanticGraph("snap", new MockEncoder(dim) { Version = "enc-1" }); + var rng = new Random(seed); + for (int i = 0; i < n; i++) + { + var id = i % 7 == 0 ? $"id_{i}" : i.ToString("D5"); + graph.AddNode(i % 2 == 0 ? "doc" : "note", id, new() { ["name"] = "n" + i }, embedText: i % 5 == 0 ? "" : $"unicode \U0001F600 é中 \"q\" {i}\nline"); + if (i % 5 != 0 && i % 13 == 0) + graph.EmbeddingStore.Add(new NodeRef(i % 2 == 0 ? "doc" : "note", id), "dup", i == 13 ? new float[dim] : StageD.Vec(rng, dim)); + } + // keys that are not graph nodes, with colons and unicode in type and id (the key is "type:id") + graph.EmbeddingStore.Add(new NodeRef("a:b", "id:with:colons"), "colons", StageD.Vec(rng, dim)); + graph.EmbeddingStore.Add(new NodeRef("ünï", "ключ"), "unicode", StageD.Vec(rng, dim)); + graph.SetEncoder(new MockEncoder(dim) { ModelName = "other-model" }); + graph.AddNode("late", "1", new() { ["name"] = "late" }); + return graph; + } + + private static void AssertSameStore(EmbeddingStore a, EmbeddingStore b, bool compareMeta = true) + { + Assert.Equal(a.Count, b.Count); + Assert.Equal(a.Dimension, b.Dimension); + Assert.Equal(a.Models().OrderBy(x => x.Key).ToList(), b.Models().OrderBy(x => x.Key).ToList()); + var pa = a.ExportEmbeddings("d"); + var pb = b.ExportEmbeddings("d"); + Assert.Equal(JsonSerializer.Serialize(pa.Embeddings), JsonSerializer.Serialize(pb.Embeddings)); // same keys, texts, models, vector bits + foreach (var e in pa.Embeddings) + { + var r = new NodeRef(e.Type, e.IdAsString()); + var x = a.Get(r)!; var y = b.Get(r)!; + Assert.Equal(x.Text, y.Text); + Assert.Equal(x.Model, y.Model); + Assert.Equal(x.Vector.Select(BitConverter.SingleToInt32Bits), y.Vector.Select(BitConverter.SingleToInt32Bits)); + } + var rng = new Random(21); + int dim = a.Dimension ?? 0; + for (int i = 0; i < 20 && dim > 0; i++) + { + var q = StageD.Vec(rng, dim); + foreach (var (k, type) in new (int?, string?)[] { (5, null), (null, null), (3, "doc") }) + Assert.Equal(StageD.Shape(a.SimilaritySearchVector(q, k, type, -1f)), StageD.Shape(b.SimilaritySearchVector(q, k, type, -1f))); + } + if (compareMeta) + Assert.Equal(a.LastImportMeta.OrderBy(x => x.Key).ToList(), b.LastImportMeta.OrderBy(x => x.Key).ToList()); + } + + private static (string Path, SemanticGraph Graph) SaveWithSnapshot(string dir, SemanticGraph? graph = null) + { + graph ??= MakeGraph(); + var path = System.IO.Path.Combine(dir, "g.ison"); + graph.Save(path, snapshot: true); + return (path, graph); + } + + private static string Snap(string path) => SemanticGraph.SnapshotPathFor(path); + + private static void Seal(string snapPath, byte[] body) => + File.WriteAllBytes(snapPath, body.Concat(SHA256.HashData(body)).ToArray()); + + // -- round trip --------------------------------------------------------------------- + + [Fact] + public void Own_save_then_load_uses_the_snapshot_and_equals_a_json_load() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + Assert.True(File.Exists(Snap(path))); + + var (viaSnap, report) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }); + Assert.Equal(LoadSource.Snapshot, report.Source); + Assert.Null(report.SnapshotSkipped); + Assert.Equal(graph.EmbeddingStore.Count, report.Embeddings); + + File.Delete(Snap(path)); + var (viaJson, jsonReport) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }); + Assert.Equal(LoadSource.Json, jsonReport.Source); + AssertSameStore(viaSnap.EmbeddingStore, viaJson.EmbeddingStore); + AssertSameStore(graph.EmbeddingStore, viaSnap.EmbeddingStore, compareMeta: false); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Snapshot_save_does_not_change_the_graph_or_the_json_sidecar_bytes() + { + var dir = StageD.TempDir(); + try + { + var graph = MakeGraph(); + var a = System.IO.Path.Combine(dir, "plain.ison"); + var b = System.IO.Path.Combine(dir, "snap.ison"); + graph.Save(a); + graph.Save(b, snapshot: true); + Assert.Equal(File.ReadAllBytes(a), File.ReadAllBytes(b)); + Assert.Equal(File.ReadAllBytes(SemanticGraph.EmbeddingsPathFor(a)), File.ReadAllBytes(SemanticGraph.EmbeddingsPathFor(b))); + Assert.False(File.Exists(Snap(a))); + Assert.True(File.Exists(Snap(b))); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void A_save_without_snapshot_removes_an_old_one_and_an_empty_store_round_trips() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + graph.Save(path); // JSON only now + Assert.False(File.Exists(Snap(path))); + + var empty = new SemanticGraph("e", new MockEncoder(8)); + var p2 = System.IO.Path.Combine(dir, "e.ison"); + empty.Save(p2, snapshot: true); + var (loaded, report) = SemanticGraph.LoadWithReport(p2, new MockEncoder(8)); + Assert.Equal(LoadSource.Snapshot, report.Source); + Assert.Equal(0, loaded.EmbeddingStore.Count); + Assert.Null(loaded.EmbeddingStore.Dimension); + loaded.EmbeddingStore.Add(new NodeRef("a", "1"), "x", new float[5]); // the dimension is not pinned by an empty snapshot + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Round_trip_reloads_through_ann_index_and_exact_search_the_same() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + var (loaded, report) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }, ann: AnnMode.Ivf, annMinSize: 0, annNprobe: 3); + Assert.Equal(LoadSource.Snapshot, report.Source); + var q = StageD.Vec(new Random(1), 16); + var hits = loaded.EmbeddingStore.SimilaritySearchVector(q, 5); + Assert.Equal(5, hits.Count); + Assert.Equal(StageD.Shape(graph.EmbeddingStore.SimilaritySearchVector(q, 5)), + StageD.Shape(loaded.EmbeddingStore.SimilaritySearchVector(q, 5, exact: true))); + } + finally { Directory.Delete(dir, true); } + } + + // -- the byte layout (DESIGN.md 11.2) ------------------------------------------------ + + [Fact] + public void The_smallest_example_of_the_spec_is_byte_for_byte_what_we_write() + { + var store = new EmbeddingStore(); // no encoder: model "provided" + store.Add(new NodeRef("t", "a"), "x", new[] { 1.0f, -2.0f }); + var dir = StageD.TempDir(); + try + { + var snap = System.IO.Path.Combine(dir, "x.snap"); + var info = store.SaveSnapshot(snap); + var bytes = File.ReadAllBytes(snap); + + var body = new List(); + body.AddRange("ISONVSNP"u8.ToArray()); + body.AddRange(new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); // version 1, flags 0 + body.AddRange(new byte[] { 2, 0, 0, 0, 0, 4, 0, 0 }); // dim 2, block_rows 1024 + body.AddRange(new byte[] { 1, 0, 0, 0, 0, 0, 0, 0 }); // count 1 + for (int i = 0; i < 16; i++) body.Add(0xff); // four absent strings + body.AddRange(new byte[] { 1, 0, 0, 0, 8, 0, 0, 0 }); + body.AddRange("provided"u8.ToArray()); + body.AddRange(new byte[] { 1, 0, 0, 0, (byte)'t', 1, 0, 0, 0, (byte)'a', 1, 0, 0, 0, (byte)'x', 0, 0, 0, 0 }); + body.AddRange(new byte[] { 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0 }); + Assert.Equal(91, body.Count); + Assert.Equal(123, bytes.Length); + Assert.Equal(123, info.Bytes); + Assert.Equal(body.ToArray(), bytes[..91]); + Assert.Equal(SHA256.HashData(body.ToArray()), bytes[91..]); + + var loaded = new EmbeddingStore(); + var read = loaded.LoadSnapshot(snap); + Assert.Equal(1, read.Count); + Assert.Equal(new[] { 1.0f, -2.0f }, loaded.Get(new NodeRef("t", "a"))!.Vector); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void The_sidecar_fingerprint_follows_the_spec_for_short_and_long_files() + { + var dir = StageD.TempDir(); + try + { + foreach (int len in new[] { 0, 1, 5000, 70_000, 400_000 }) + { + var path = System.IO.Path.Combine(dir, $"f{len}"); + var data = new byte[len]; + new Random(len).NextBytes(data); + File.WriteAllBytes(path, data); + + var h = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + h.AppendData(BitConverter.GetBytes((long)len)); // little endian on every host we run on + void Take(long at, int n) + { + n = (int)Math.Min(n, Math.Max(0, len - at)); + h.AppendData(data, (int)at, n); + } + Take(0, 65536); + Take(Math.Max(0, len - 65536), 65536); + for (int i = 1; i <= 64; i++) Take((long)len * i / 65, 4096); + var id = SidecarId.OfFile(path); + Assert.Equal(len, id.Length); + Assert.Equal(h.GetHashAndReset(), id.Fingerprint); + } + } + finally { Directory.Delete(dir, true); } + } + + // -- golden fixture written by the Rust crate ------------------------------------------- + + private static string CopyGolden(string dir, bool withSnap) + { + var src = StageD.FixtureDir(); + foreach (var name in new[] { "golden.ison", "golden.ison.embeddings.json", "golden.ison.embeddings.snap" }) + if (withSnap || !name.EndsWith(".snap")) File.Copy(System.IO.Path.Combine(src, name), System.IO.Path.Combine(dir, name)); + return System.IO.Path.Combine(dir, "golden.ison"); + } + + [Fact] + public void The_rust_golden_snapshot_is_used_and_equals_the_json_load() + { + var dir = StageD.TempDir(); + try + { + var golden = CopyGolden(dir, withSnap: true); + var (viaSnap, report) = SemanticGraph.LoadWithReport(golden, new MockEncoder(12)); + Assert.True(report.Source == LoadSource.Snapshot, $"snapshot was not used: {report.Source}, skipped: {report.SnapshotSkipped}"); + Assert.Equal(36, report.Embeddings); + + var dir2 = StageD.TempDir(); + try + { + var plain = CopyGolden(dir2, withSnap: false); + var (viaJson, jsonReport) = SemanticGraph.LoadWithReport(plain, new MockEncoder(12)); + Assert.Equal(LoadSource.Json, jsonReport.Source); + Assert.Equal(36, viaJson.EmbeddingStore.Count); + AssertSameStore(viaSnap.EmbeddingStore, viaJson.EmbeddingStore); + Assert.Contains("second-model", viaSnap.EmbeddingStore.Models().Keys); + Assert.Equal(2, viaSnap.EmbeddingStore.Models().Count); + } + finally { Directory.Delete(dir2, true); } + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Our_writer_reproduces_the_rust_golden_snapshot_byte_for_byte() + { + var dir = StageD.TempDir(); + try + { + var golden = CopyGolden(dir, withSnap: false); + var goldenSnap = File.ReadAllBytes(System.IO.Path.Combine(StageD.FixtureDir(), "golden.ison.embeddings.snap")); + var encoder = new MockEncoder(12) { ModelName = "second-model" }; + var (graph, report) = SemanticGraph.LoadWithReport(golden, encoder); + Assert.Equal(LoadSource.Json, report.Source); + var digest = graph.EmbeddingStore.LastImportMeta["graph_digest"]; + + var mine = System.IO.Path.Combine(dir, "mine.snap"); + graph.EmbeddingStore.SaveSnapshot(mine, new SnapshotIdentity(digest, SidecarId.OfFile(SemanticGraph.EmbeddingsPathFor(golden)))); + Assert.Equal(goldenSnap, File.ReadAllBytes(mine)); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void The_rust_golden_snapshot_rejections_are_the_same_as_for_our_own() + { + var dir = StageD.TempDir(); + try + { + var golden = CopyGolden(dir, withSnap: true); + var snap = Snap(golden); + var good = File.ReadAllBytes(snap); + var bad = (byte[])good.Clone(); + bad[bad.Length / 2] ^= 0x40; + File.WriteAllBytes(snap, bad); + var (_, report) = SemanticGraph.LoadWithReport(golden, new MockEncoder(12)); + Assert.Equal(LoadSource.Json, report.Source); + Assert.NotNull(report.SnapshotSkipped); + } + finally { Directory.Delete(dir, true); } + } + + // -- every rejection falls back to JSON without touching the store --------------------------- + + private static void AssertFallsBack(string path, string reasonPart) + { + var (graph, report) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }); + Assert.Equal(LoadSource.Json, report.Source); + Assert.NotNull(report.SnapshotSkipped); + Assert.Contains(reasonPart, report.SnapshotSkipped!, StringComparison.OrdinalIgnoreCase); + Assert.True(graph.EmbeddingStore.Count > 0); // the JSON sidecar was read + } + + [Fact] + public void Corrupt_truncated_and_stale_snapshots_fall_back_to_json() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + var snap = Snap(path); + var good = File.ReadAllBytes(snap); + + // a flipped bit in the body: checksum + var flipped = (byte[])good.Clone(); + flipped[good.Length - 100] ^= 0x01; + File.WriteAllBytes(snap, flipped); + AssertFallsBack(path, "checksum"); + + // a flipped bit in the trailer itself + flipped = (byte[])good.Clone(); + flipped[^1] ^= 0x80; + File.WriteAllBytes(snap, flipped); + AssertFallsBack(path, "checksum"); + + // truncation, everywhere that matters + foreach (int len in new[] { 0, 7, 39, 40, 100, good.Length / 2, good.Length - 33, good.Length - 32, good.Length - 1 }) + { + File.WriteAllBytes(snap, good[..len]); + var (g, r) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }); + Assert.Equal(LoadSource.Json, r.Source); + Assert.NotNull(r.SnapshotSkipped); + Assert.Equal(graph.EmbeddingStore.Count, g.EmbeddingStore.Count); + } + + // bytes appended after the trailer move it: the checksum is read from the wrong place + File.WriteAllBytes(snap, good.Concat(new byte[] { 0 }).ToArray()); + AssertFallsBack(path, ""); + + // not a snapshot at all + File.WriteAllText(snap, new string('x', 200)); + AssertFallsBack(path, "magic"); + + // stale: the sidecar was rewritten (same graph digest, different bytes) + File.WriteAllBytes(snap, good); + var (_, ok) = SemanticGraph.LoadWithReport(path, new MockEncoder(16) { Version = "enc-1" }); + Assert.Equal(LoadSource.Snapshot, ok.Source); + var sidecar = SemanticGraph.EmbeddingsPathFor(path); + var text = File.ReadAllText(sidecar); + File.WriteAllText(sidecar, text.Replace("\"text\":\"dup\"", "\"text\":\"dux\"")); // same length, sampled bytes differ + AssertFallsBack(path, "stale"); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void A_snapshot_for_another_graph_is_refused_by_digest_and_by_a_missing_sidecar() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + var sidecar = SemanticGraph.EmbeddingsPathFor(path); + + // the graph file is rewritten after the save (a node added): the digest in the header no longer matches + graph.AddNode("extra", "1", new() { ["name"] = "x" }, embedText: ""); + graph.Save(path, embeddings: false); + Assert.True(File.Exists(Snap(path))); + Assert.Throws(() => SemanticGraph.LoadWithReport(path, new MockEncoder(16))); + var (g, report) = SemanticGraph.LoadWithReport(path, new MockEncoder(16), onMismatch: GraphMismatchMode.Warn, onWarning: _ => { }); + Assert.Equal(LoadSource.Json, report.Source); + Assert.Contains("different graph", report.SnapshotSkipped); + Assert.True(g.EmbeddingStore.Count > 0); + + // no sidecar: never use a snapshot alone + File.Delete(sidecar); + var (loaded, none) = SemanticGraph.LoadWithReport(path, new MockEncoder(16)); + Assert.Equal(LoadSource.NoEmbeddings, none.Source); + Assert.Equal(0, loaded.EmbeddingStore.Count); + } + finally { Directory.Delete(dir, true); } + } + + private static byte[] Body(string snapPath) => File.ReadAllBytes(snapPath)[..^32]; + + private static void ExpectRejected(string path, byte[] body, string reason) + { + Seal(Snap(path), body); + AssertFallsBack(path, reason); + var store = StoreWithRow(); + var before = JsonSerializer.Serialize(store.ExportEmbeddings()); + var ex = Assert.Throws(() => store.LoadSnapshot(Snap(path))); // a failed load leaves the store as it was + Assert.Contains(reason, ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(before, JsonSerializer.Serialize(store.ExportEmbeddings())); + Assert.Equal(1, store.Count); + } + + private static EmbeddingStore StoreWithRow() + { + var s = new EmbeddingStore(); + s.Add(new NodeRef("keep", "me"), "kept", new float[16]); + return s; + } + + [Fact] + public void Every_structural_rejection_rule_of_the_spec_is_enforced() + { + var dir = StageD.TempDir(); + try + { + var (path, _) = SaveWithSnapshot(dir); + var good = Body(Snap(path)); + + byte[] With(Action edit) { var c = (byte[])good.Clone(); edit(c); return c; } + + ExpectRejected(path, With(b => b[0] = (byte)'X'), "magic"); + ExpectRejected(path, With(b => b[8] = 2), "version"); + ExpectRejected(path, With(b => b[12] = 2), "flags"); // an unknown flag bit + ExpectRejected(path, With(b => { b[20] = 0; b[21] = 0; b[22] = 0; b[23] = 0; }), "block size"); // block_rows 0 + ExpectRejected(path, With(b => { b[22] = 0x20; }), "block size"); // block_rows > 2^20 + ExpectRejected(path, With(b => { b[16] = 0; b[17] = 0; b[18] = 0; b[19] = 0; }), "shape"); // dim 0 with rows + ExpectRejected(path, With(b => b[27] = 0x80), "shape"); // count past 2^32-1... and rows + ExpectRejected(path, With(b => b[24] += 1), ""); // count one too many: runs out of rows + ExpectRejected(path, With(b => b[24] -= 1), "left over"); // count one too few: bytes left over + ExpectRejected(path, good.Concat(new byte[] { 7, 7 }).ToArray(), "left over"); // bytes between the last block and the trailer + ExpectRejected(path, good[..^1], ""); // short by one byte (sealed, so only structure can object) + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void A_repeated_key_a_bad_model_index_a_bad_utf8_string_and_an_absent_required_string_are_rejected() + { + var dir = StageD.TempDir(); + try + { + // hand-built single-block snapshots, sealed with a correct checksum, so only the rule under test can object + byte[] Build(Action> rows, int count = 2, int modelCount = 1) + { + var b = new List(); + b.AddRange("ISONVSNP"u8.ToArray()); + b.AddRange(new byte[] { 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0 }); + b.AddRange(BitConverter.GetBytes((long)count)); + for (int i = 0; i < 16; i++) b.Add(0xff); + b.AddRange(BitConverter.GetBytes((uint)modelCount)); + for (int m = 0; m < modelCount; m++) { b.AddRange(BitConverter.GetBytes(1u)); b.Add((byte)'m'); } + rows(b); + return b.ToArray(); + } + void Str(List b, string s) { var raw = Encoding.UTF8.GetBytes(s); b.AddRange(BitConverter.GetBytes((uint)raw.Length)); b.AddRange(raw); } + void Row(List b, string type, string id, uint model) { Str(b, type); Str(b, id); Str(b, ""); b.AddRange(BitConverter.GetBytes(model)); } + void Floats(List b, int n) { for (int i = 0; i < n; i++) b.AddRange(BitConverter.GetBytes(1.5f)); } + + var path = System.IO.Path.Combine(dir, "h.ison"); + var graph = MakeGraph(20, 2); + graph.Save(path, snapshot: true); + + ExpectRejectedHand(path, Build(b => { Row(b, "t", "1", 0); Row(b, "t", "1", 0); Floats(b, 4); }), "twice"); + ExpectRejectedHand(path, Build(b => { Row(b, "t", "1", 0); Row(b, "t", "2", 1); Floats(b, 4); }), "model"); + ExpectRejectedHand(path, Build(b => { Row(b, "t", "1", 0); b.AddRange(BitConverter.GetBytes(1u)); b.Add(0xC3); Str(b, ""); Str(b, ""); b.AddRange(BitConverter.GetBytes(0u)); Floats(b, 4); }), "utf-8"); + ExpectRejectedHand(path, Build(b => { Row(b, "t", "1", 0); b.AddRange(BitConverter.GetBytes(uint.MaxValue)); }), "absent"); + ExpectRejectedHand(path, Build(b => { Row(b, "t", "1", 0); Row(b, "t", "2", 0); Floats(b, 3); }), ""); // one float short + ExpectRejectedHand(path, Build(b => { }, modelCount: 0, count: 0).Take(40).ToArray(), ""); // cut inside the header + + // and the same two rows with distinct keys are fine + Seal(Snap(path), Build(b => { Row(b, "t", "1", 0); Row(b, "t", "2", 0); Floats(b, 4); })); + var store = new EmbeddingStore(); + Assert.Equal(2, store.LoadSnapshot(Snap(path)).Count); + } + finally { Directory.Delete(dir, true); } + } + + private static void ExpectRejectedHand(string path, byte[] body, string reason) + { + Seal(Snap(path), body); + var store = StoreWithRow(); + var ex = Assert.Throws(() => store.LoadSnapshot(Snap(path))); + Assert.Contains(reason, ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(1, store.Count); // untouched + Assert.Equal(new float[16], store.Get(new NodeRef("keep", "me"))!.Vector); + } + + [Fact] + public void A_failed_snapshot_write_keeps_the_previous_file_and_leaves_no_temp() + { + var dir = StageD.TempDir(); + try + { + var (path, graph) = SaveWithSnapshot(dir); + var snap = Snap(path); + var before = File.ReadAllBytes(snap); + + // a read-only target directory entry cannot be forced portably; make the write itself fail instead + Assert.Throws(() => AtomicFile.Write(snap, stream => + { + stream.Write(new byte[] { 1, 2, 3 }); + throw new ArgumentException("injected"); + })); + Assert.Equal(before, File.ReadAllBytes(snap)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + Assert.Empty(Directory.GetFiles(dir, ".*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void A_snapshot_made_of_several_blocks_round_trips() + { + var dir = StageD.TempDir(); + try + { + var graph = MakeGraph(n: 2600, dim: 8); // three blocks of 1024 rows + var (path, _) = SaveWithSnapshot(dir, graph); + var (viaSnap, report) = SemanticGraph.LoadWithReport(path, new MockEncoder(8) { Version = "enc-1" }); + Assert.Equal(LoadSource.Snapshot, report.Source); + AssertSameStore(graph.EmbeddingStore, viaSnap.EmbeddingStore, compareMeta: false); + } + finally { Directory.Delete(dir, true); } + } +} From ae93567f373a2d489198b262a4fb3b05db1403ad Mon Sep 17 00:00:00 2001 From: Mahesh Vaikri Date: Tue, 17 Nov 2026 08:44:22 -0600 Subject: [PATCH 3/7] TypeScript/JavaScript: one float32 copy of every vector, optional binary snapshot VectorTable replaces the record map plus ScanCache: float32 rows in 1024-row chunks, norms/type codes/models in parallel columns, tombstones compacted at 25%, IVF reads rows from the table. The exact scan scores four rows at a time with independent per-row sums, so scores are unchanged and the scan is faster. Optional .embeddings.snap per Rust DESIGN.md section 11 (reproduces the Rust golden snapshot byte for byte; the only way to persist 100k x 384 in JS, since the JSON export exceeds V8's string limit). New browser-safe snapshot module with injected SHA-256; file I/O in node.ts / node.js. Breaking: vectors are stored as float32 (values widen to double on read), get()/getEmbedding() return a copy, AnnStats.cacheBytes is always 0. Not done: write-time IVF fold (kept the search-time fold; a fold in a write would block it for seconds), batch search (single-threaded). Co-Authored-By: Claude Sonnet 5.5 --- isongraph-vector-js/bench/ann.mjs | 12 +- isongraph-vector-js/package.json | 6 +- isongraph-vector-js/src/async-encoder.test.js | 4 +- isongraph-vector-js/src/consistency.test.js | 2 +- isongraph-vector-js/src/index.js | 675 ++++++++++++----- isongraph-vector-js/src/node.js | 369 +++++++-- isongraph-vector-js/src/snapshot.js | 523 +++++++++++++ isongraph-vector-js/src/snapshot.test.js | 576 +++++++++++++++ isongraph-vector-ts/package.json | 6 +- isongraph-vector-ts/src/async-encoder.test.ts | 4 +- isongraph-vector-ts/src/index.ts | 699 +++++++++++++----- isongraph-vector-ts/src/node.ts | 415 +++++++++-- isongraph-vector-ts/src/snapshot.test.ts | 594 +++++++++++++++ isongraph-vector-ts/src/snapshot.ts | 638 ++++++++++++++++ 14 files changed, 4021 insertions(+), 502 deletions(-) create mode 100644 isongraph-vector-js/src/snapshot.js create mode 100644 isongraph-vector-js/src/snapshot.test.js create mode 100644 isongraph-vector-ts/src/snapshot.test.ts create mode 100644 isongraph-vector-ts/src/snapshot.ts diff --git a/isongraph-vector-js/bench/ann.mjs b/isongraph-vector-js/bench/ann.mjs index 95e7355..a72d899 100644 --- a/isongraph-vector-js/bench/ann.mjs +++ b/isongraph-vector-js/bench/ann.mjs @@ -14,10 +14,12 @@ * somewhere else - run store.recallCheck() on your own queries before relying * on a setting. * - * Stored vectors are plain JS arrays of doubles (8 bytes per component) plus - * the index's Float32Array copy (4 bytes), so 1M x 384 needs about 12 GB of - * heap: run it with --max-old-space-size. A size that will not fit is skipped - * with a message instead of crashing. + * The store keeps one float32 copy of each vector (4 bytes per component, in + * Float32Array chunks outside the JS heap; the IVF index reads those rows and + * adds about 8 bytes per vector), so 1M x 384 needs a little over 1.5 GB. The + * vectors this script generates are plain JS arrays of doubles first, which is + * what dominates its heap: run it with --max-old-space-size. A size that will + * not fit is skipped with a message instead of crashing. * * To measure the TypeScript build instead, pass --port ts (needs `npm run * build` in isongraph-vector-ts first); --entry imports any other build. @@ -147,7 +149,7 @@ console.log(`# exact vs IVF (${opts.port}), dim ${opts.dim}, k ${opts.k}, ${opts console.log(`node ${process.version}, ${cpus().length} logical CPUs, ${(totalmem() / 2 ** 30).toFixed(1)} GB RAM, heap limit ${(heapLimit / 2 ** 30).toFixed(1)} GB\n`); for (const n of opts.sizes) { - const need = n * opts.dim * 13 * 1.2; // doubles + float32 copy + slack + const need = n * 400 * 1.2; // per-row JS objects (key, ref, text); the float32 rows live off the JS heap if (need > heapLimit * 0.9) { console.log(`## N = ${n.toLocaleString('en-US')}: skipped (needs about ${(need / 2 ** 30).toFixed(1)} GB of heap, limit ${(heapLimit / 2 ** 30).toFixed(1)} GB; rerun with --max-old-space-size)\n`); continue; diff --git a/isongraph-vector-js/package.json b/isongraph-vector-js/package.json index dbcdc87..1a4d128 100644 --- a/isongraph-vector-js/package.json +++ b/isongraph-vector-js/package.json @@ -6,19 +6,21 @@ "exports": { ".": "./src/index.js", "./sqlite": "./src/sqlite-store.js", - "./node": "./src/node.js" + "./node": "./src/node.js", + "./snapshot": "./src/snapshot.js" }, "type": "module", "files": [ "src/index.js", "src/sqlite-store.js", "src/node.js", + "src/snapshot.js", "README.md", "LICENSE" ], "scripts": { "test": "vitest run", - "test:core": "vitest run src/index.test.js src/http-encoder.test.js src/async-encoder.test.js src/ann.test.js", + "test:core": "vitest run src/index.test.js src/http-encoder.test.js src/async-encoder.test.js src/ann.test.js src/snapshot.test.js", "bench:ann": "node --max-old-space-size=16384 bench/ann.mjs" }, "keywords": [ diff --git a/isongraph-vector-js/src/async-encoder.test.js b/isongraph-vector-js/src/async-encoder.test.js index 47a30ef..89f1dfd 100644 --- a/isongraph-vector-js/src/async-encoder.test.js +++ b/isongraph-vector-js/src/async-encoder.test.js @@ -166,7 +166,7 @@ describe('SemanticGraph auto-embed over HTTP (openai shape)', () => { expect(seen[0].body.input).toEqual(['Alice an engineer']); expect(seen[0].headers.authorization).toBe('Bearer sk-test-key'); const record = graph.getEmbedding(['person', 1]); - expect(record.vector).toEqual(vectorOf('Alice an engineer')); + expect(record.vector).toEqual(vectorOf('Alice an engineer').map(Math.fround)); // stored as float32 expect(record.model).toBe('remote-4'); expect(graph.embeddingStore.dimension).toBe(4); @@ -211,7 +211,7 @@ describe('SemanticGraph auto-embed over HTTP (openai shape)', () => { await graph.updateNodeAsync('person', 1, { name: 'Alice the manager' }); expect(seen).toHaveLength(2); expect(graph.getEmbedding(['person', 1]).text).toBe('Alice the manager'); - expect(graph.getEmbedding(['person', 1]).vector).toEqual(vectorOf('manager')); + expect(graph.getEmbedding(['person', 1]).vector).toEqual(vectorOf('manager').map(Math.fround)); // stored as float32 await graph.updateNodeAsync('person', 1, { name: null }); // nothing left to embed expect(graph.getEmbedding(['person', 1])).toBeNull(); diff --git a/isongraph-vector-js/src/consistency.test.js b/isongraph-vector-js/src/consistency.test.js index 113905d..6d26008 100644 --- a/isongraph-vector-js/src/consistency.test.js +++ b/isongraph-vector-js/src/consistency.test.js @@ -41,7 +41,7 @@ describe('updateNode keeps embeddings in step', () => { g.addNode('person', 1, { name: 'Alice' }); const before = g.getEmbedding(['person', 1]); g.updateNode('person', 1, { age: 31 }); - expect(g.getEmbedding(['person', 1])).toBe(before); + expect(g.getEmbedding(['person', 1])).toEqual(before); // get() hands out a copy, so compare by value }); it('leaves vectors alone when autoEmbed is off', () => { diff --git a/isongraph-vector-js/src/index.js b/isongraph-vector-js/src/index.js index 2941e52..eb83d52 100644 --- a/isongraph-vector-js/src/index.js +++ b/isongraph-vector-js/src/index.js @@ -591,6 +591,8 @@ export function selectTopK(results, k) { // ============================================================================= // Approximate (IVF) search // ============================================================================= +/** An exact search for at most this many results keeps them in a small sorted buffer; a larger k collects every hit first. */ +const EXACT_TOPK_BUFFER = 256; /** Stores below this many vectors answer exactly even with `ann` on. */ const ANN_MIN_SIZE = 50000; /** New or updated rows are scored exactly until this many (or this fraction of the trained size) are pending. */ @@ -814,107 +816,198 @@ class CentroidSearch { return best; } } +/** Rows per matrix chunk (= one scan block). Chunks never move once allocated, so growth never copies stored rows. */ +const CHUNK_ROWS = 1024; +const CHUNK_SHIFT = 10; +const CHUNK_MASK = CHUNK_ROWS - 1; +/** Rows the first chunk starts with; it doubles until it holds CHUNK_ROWS, later chunks are allocated whole. */ +const FIRST_CHUNK_ROWS = 16; +/** True when this host stores a Float32Array in little-endian byte order (the snapshot's byte order). */ +export function hostIsLittleEndian() { + return new Uint8Array(new Uint32Array([1]).buffer)[0] === 1; +} +const HOST_LITTLE_ENDIAN = hostIsLittleEndian(); /** - * Float32Array snapshot of the store's vectors, kept in step with its writes. + * The store's single copy of every vector, plus the per-row facts around it. + * + * A record *is a slot number*; nothing else owns the floats (see the Rust + * crate's DESIGN.md, section 1). The rows live in `Float32Array` chunks of + * {@link CHUNK_ROWS} rows each, `dim` floats per row, row-major; `norms`, + * `codes` and `modelIx` are parallel columns, and `keys`/`refs`/`texts` hold the + * strings. Slots are in arrival order, which is the order the store used to + * iterate its records in, so ties still break the way they always did: an update + * overwrites its slot in place, an insert takes `n`, a delete tombstones (key + * `null`, norm `NaN`). Trailing tombstones are popped; once a quarter of the + * slots are dead the live rows are moved down, keeping their order. * - * One contiguous matrix with spare row capacity, the vector norms, a type code - * per row and a liveness flag. A write patches it in place: an insert appends - * a row, an update overwrites one, a delete tombstones one (squeezed out once - * a quarter of the rows are dead). Rows stay in insertion order, which is the - * order the store's Map iterates, so ties break exactly as the exact scan's do. - * Built lazily by the first approximate search, never persisted. + * Both exact search and the IVF index read the rows from here, so the index adds + * no second copy of the vectors. Not part of the stable API: it is exported so + * the snapshot module can read and fill it. */ -class ScanCache { - constructor(dim, capacity) { - /** Rows in use, tombstones included, and how many of those are tombstones. */ - this.n = 0; - this.dead = 0; +export class VectorTable { + constructor(dim) { + this.chunks = []; + /** Euclidean norm of each row, computed from the float32 values; NaN = tombstone. */ + this.norms = new Float64Array(0); + /** Node type of each row, as an index into `typeNames`. */ + this.codes = new Int32Array(0); + /** Model of each row, as an index into `modelNames`. */ + this.modelIx = new Int32Array(0); + /** `"type:id"` of each row, or null for a tombstone. */ this.keys = []; - this.pos = new Map(); + this.refs = []; + this.texts = []; + this.modelNames = []; + this.typeNames = []; this.codeOf = new Map(); + /** key -> slot: the only lookup structure. */ + this.index = new Map(); + this._modelOf = new Map(); + /** Slots in use, tombstones included, and how many of those are tombstones. */ + this.n = 0; + this.dead = 0; this.ivf = null; this.dim = dim; - this.matrix = new Float32Array(capacity * dim); - this.norms = new Float64Array(capacity); - this.codes = new Int32Array(capacity); - this.alive = new Uint8Array(capacity); this.query = new Float32Array(dim); } - static build(records, count, dim) { - const cache = new ScanCache(dim, count + Math.max(1024, count >> 4)); - for (const record of records) - cache.put(record.id, record); - return cache; - } get capacity() { return this.norms.length; } - _grow() { - const capacity = this.capacity + Math.max(1024, this.capacity >> 2); - const matrix = new Float32Array(capacity * this.dim); - matrix.set(this.matrix.subarray(0, this.n * this.dim)); + /** Live rows. */ + get live() { + return this.index.size; + } + _modelId(name) { + let id = this._modelOf.get(name); + if (id === undefined) { + id = this.modelNames.length; + this.modelNames.push(name); + this._modelOf.set(name, id); + } + return id; + } + _growColumns() { + const capacity = this.capacity + Math.max(1024, this.capacity >> 1); const norms = new Float64Array(capacity); norms.set(this.norms.subarray(0, this.n)); const codes = new Int32Array(capacity); codes.set(this.codes.subarray(0, this.n)); - const alive = new Uint8Array(capacity); - alive.set(this.alive.subarray(0, this.n)); - this.matrix = matrix; + const modelIx = new Int32Array(capacity); + modelIx.set(this.modelIx.subarray(0, this.n)); this.norms = norms; this.codes = codes; - this.alive = alive; + this.modelIx = modelIx; } - /** Mirror one upserted record. */ - put(key, record) { + /** Make sure the chunk holding `slot` exists and is big enough. */ + _ensureRow(slot) { + const c = slot >>> CHUNK_SHIFT; const dim = this.dim; - let i = this.pos.get(key); - if (i === undefined) { + if (c === this.chunks.length) { + this.chunks.push(new Float32Array((c === 0 ? FIRST_CHUNK_ROWS : CHUNK_ROWS) * dim)); + } + else if (c === 0) { + const first = this.chunks[0]; + const need = ((slot & CHUNK_MASK) + 1) * dim; + if (first.length < need) { + const grown = new Float32Array(Math.min(CHUNK_ROWS * dim, Math.max(need, first.length * 2))); + grown.set(first); + this.chunks[0] = grown; + } + } + } + /** Find or create the slot of `key` and set its metadata; the caller writes the row and the norm. */ + _claim(key, ref, text, model) { + const type = ref[0]; + let code = this.codeOf.get(type); + if (code === undefined) { + code = this.typeNames.length; + this.typeNames.push(type); + this.codeOf.set(type, code); + } + let slot = this.index.get(key); + if (slot === undefined) { if (this.n >= this.capacity) - this._grow(); - i = this.n++; - this.keys[i] = key; - this.pos.set(key, i); - const type = record.nodeRef[0]; - let code = this.codeOf.get(type); - if (code === undefined) { - code = this.codeOf.size; - this.codeOf.set(type, code); - } - this.codes[i] = code; - this.alive[i] = 1; - } - const vector = record.vector; - const base = i * dim; + this._growColumns(); + slot = this.n++; + this._ensureRow(slot); + this.keys[slot] = key; + this.index.set(key, slot); + } + this.refs[slot] = ref; + this.texts[slot] = text; + this.codes[slot] = code; + this.modelIx[slot] = this._modelId(model); + return slot; + } + /** Insert or overwrite the row of `key`; returns its slot. The values are stored as float32. */ + put(key, ref, text, model, vector) { + const slot = this._claim(key, ref, text, model); + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; let sum = 0; for (let j = 0; j < dim; j++) { const f = Math.fround(vector[j]); - this.matrix[base + j] = f; + chunk[base + j] = f; sum += f * f; } - this.norms[i] = Math.sqrt(sum); + this.norms[slot] = Math.sqrt(sum); if (this.ivf !== null) - this.ivf.notePut(i, this.capacity); + this.ivf.notePut(slot, this.capacity); + return slot; } - /** Mirror one deleted record. */ - drop(key) { - const i = this.pos.get(key); - if (i === undefined) - return; - this.pos.delete(key); - this.matrix.fill(0, i * this.dim, (i + 1) * this.dim); - this.norms[i] = 0; - this.alive[i] = 0; - this.keys[i] = null; + /** + * Like {@link put}, from `dim * 4` bytes of little-endian IEEE-754 float32 + * (the snapshot's row encoding). On a little-endian host the bytes are copied + * as they are; otherwise (or with `portable`) each float is read through a + * DataView. + */ + putLittleEndian(key, ref, text, model, bytes, portable = false) { + const slot = this._claim(key, ref, text, model); + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + if (HOST_LITTLE_ENDIAN && !portable) { + new Uint8Array(chunk.buffer, chunk.byteOffset + base * 4, dim * 4).set(bytes); + } + else { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + for (let j = 0; j < dim; j++) + chunk[base + j] = view.getFloat32(4 * j, true); + } + let sum = 0; + for (let j = 0; j < dim; j++) { + const f = chunk[base + j]; + sum += f * f; + } + this.norms[slot] = Math.sqrt(sum); + if (this.ivf !== null) + this.ivf.notePut(slot, this.capacity); + return slot; + } + /** Tombstone the row of `key`; false when there is none. */ + remove(key) { + const slot = this.index.get(key); + if (slot === undefined) + return false; + this.index.delete(key); + this.keys[slot] = null; + this.refs[slot] = null; + this.texts[slot] = ''; + this.norms[slot] = NaN; this.dead++; if (this.ivf !== null) - this.ivf.noteDrop(i); - while (this.n > 0 && this.alive[this.n - 1] === 0) { // trailing tombstones + this.ivf.noteDrop(slot); + while (this.n > 0 && this.keys[this.n - 1] === null) { // trailing tombstones this.n--; this.keys.pop(); + this.refs.pop(); + this.texts.pop(); this.dead--; } if (this.dead > 32 && this.dead * 4 > this.n) this._compact(); + return true; } /** Squeeze the tombstones out, keeping the live rows in their order. */ _compact() { @@ -922,33 +1015,58 @@ class ScanCache { const live = new Int32Array(this.n - this.dead); let kept = 0; for (let r = 0; r < this.n; r++) - if (this.alive[r] === 1) + if (this.keys[r] !== null) live[kept++] = r; if (this.ivf !== null) this.ivf.compact(live); for (let w = 0; w < kept; w++) { const r = live[w]; - if (r !== w) { - this.matrix.copyWithin(w * dim, r * dim, (r + 1) * dim); - this.norms[w] = this.norms[r]; - this.codes[w] = this.codes[r]; - const key = this.keys[r]; - this.keys[w] = key; - this.pos.set(key, w); - } + if (r === w) + continue; + const src = this.chunks[r >>> CHUNK_SHIFT]; + const dst = this.chunks[w >>> CHUNK_SHIFT]; + const so = (r & CHUNK_MASK) * dim; + const d = (w & CHUNK_MASK) * dim; + if (src === dst) + src.copyWithin(d, so, so + dim); + else + dst.set(src.subarray(so, so + dim), d); + this.norms[w] = this.norms[r]; + this.codes[w] = this.codes[r]; + this.modelIx[w] = this.modelIx[r]; + const key = this.keys[r]; + this.keys[w] = key; + this.refs[w] = this.refs[r]; + this.texts[w] = this.texts[r]; + this.index.set(key, w); } - this.alive.fill(1, 0, kept); - this.alive.fill(0, kept, this.n); this.keys.length = kept; + this.refs.length = kept; + this.texts.length = kept; + this.chunks.length = Math.ceil(kept / CHUNK_ROWS); this.n = kept; this.dead = 0; } + /** The vector of `slot` as a fresh array of numbers (the float32 values, widened). */ + vectorAt(slot) { + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + const out = new Array(dim); + for (let j = 0; j < dim; j++) + out[j] = chunk[base + j]; + return out; + } + /** Bytes of the vectors and the numeric columns (not the strings). */ bytes() { - return this.matrix.byteLength + this.norms.byteLength + this.codes.byteLength + this.alive.byteLength; + let total = this.norms.byteLength + this.codes.byteLength + this.modelIx.byteLength; + for (const chunk of this.chunks) + total += chunk.byteLength; + return total; } } /** - * Inverted-file ("IVF-flat") index over a {@link ScanCache}. + * Inverted-file ("IVF-flat") index over a {@link VectorTable}. * * Spherical k-means partitions the rows into `nlist` cells by direction, which * is what cosine similarity compares. A query scores the `nprobe` cells whose @@ -957,7 +1075,7 @@ class ScanCache { * cell number per row (`assign`) plus the rows grouped by cell * (`order`/`offsets`), about 8 bytes per vector plus the centroids. * - * It follows its cache instead of being rebuilt on every write: + * It follows its table instead of being rebuilt on every write: * - a **new** or **updated** row is *pending* (cell -1): every query scores * the pending rows exactly, so a write is visible at once and costs the * index nothing; @@ -965,7 +1083,7 @@ class ScanCache { * are enough of them (see `EmbeddingStore._annIndex`); * - a **delete** clears the row's cell; stale entries in `order` are skipped * at query time; - * - the cache's compaction renumbers rows, and `compact` carries the cells + * - the table's compaction renumbers rows, and `compact` carries the cells * across without retraining; * - `churn` counts writes and deletes since training; the store retrains when * it passes a fraction of the trained size. @@ -992,11 +1110,11 @@ class IvfIndex { /** Train centroids on a sample, then give every live row its cell. */ static build(cache, nlistWanted, seed) { const started = nowMs(); - const { n, dim, matrix, norms, alive } = cache; + const { n, dim, chunks, norms, keys } = cache; const live = new Int32Array(n - cache.dead); let total = 0; for (let r = 0; r < n; r++) - if (alive[r] === 1) + if (keys[r] !== null) live[total++] = r; const nlist = Math.max(1, Math.min(Math.floor(nlistWanted), total)); const rng = new Rng(seed); @@ -1021,10 +1139,11 @@ class IvfIndex { for (let i = 0; i < sampleN; i++) { const r = sample[i]; const inv = norms[r] > 0 ? 1 / norms[r] : 0; - const src = r * dim; + const chunk = chunks[r >>> CHUNK_SHIFT]; + const src = (r & CHUNK_MASK) * dim; const dst = i * dim; for (let j = 0; j < dim; j++) - x[dst + j] = matrix[src + j] * inv; + x[dst + j] = chunk[src + j] * inv; } // Initial centroids: distinct sample points. const centroids = new Float32Array(nlist * dim); @@ -1076,7 +1195,7 @@ class IvfIndex { index.trainedN = total; for (let t = 0; t < total; t++) { const r = live[t]; - index.assign[r] = index._search.nearest(matrix, r * dim); + index.assign[r] = index._search.nearest(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim); } index.relist(n); index.buildSeconds = (nowMs() - started) / 1000; @@ -1109,8 +1228,9 @@ class IvfIndex { /** Give every pending row its nearest cell and regroup. */ fold(cache) { if (this.pending.size > 0) { + const chunks = cache.chunks; for (const r of this.pending) { - this.assign[r] = this._search.nearest(cache.matrix, r * this.dim); + this.assign[r] = this._search.nearest(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * this.dim); } this.pending.clear(); } @@ -1132,7 +1252,7 @@ class IvfIndex { this.pending.delete(i); this.churn++; } - /** Follow the cache's compaction (`live` = the old rows kept, in order). */ + /** Follow the table's compaction (`live` = the old rows kept, in order). */ compact(live) { const kept = live.length; const moved = new Int32Array(kept); @@ -1195,7 +1315,7 @@ class IvfIndex { } const candRows = this.candRows; const candScores = this.candScores; - const { matrix, norms, codes } = cache; + const { chunks, norms, codes } = cache; const assign = this.assign; let m = 0; for (let k = 0; k < nChosen; k++) { @@ -1207,7 +1327,7 @@ class IvfIndex { if (code >= 0 && codes[r] !== code) continue; const denom = norms[r] * qnorm; - const s = denom > 0 ? dot2(matrix, r * dim, q, 0, dim) / denom : 0; + const s = denom > 0 ? dot2(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim, q, 0, dim) / denom : 0; if (s < floor) continue; candRows[m] = r; @@ -1219,7 +1339,7 @@ class IvfIndex { if (code >= 0 && codes[r] !== code) continue; const denom = norms[r] * qnorm; - const s = denom > 0 ? dot2(matrix, r * dim, q, 0, dim) / denom : 0; + const s = denom > 0 ? dot2(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim, q, 0, dim) / denom : 0; if (s < floor) continue; candRows[m] = r; @@ -1311,6 +1431,29 @@ function lowerBound(values, target) { } return lo; } +/** + * Dot products of one query with four rows, written to `out[0..3]`. Each sum + * runs over the dimensions in order, exactly as a loop over one row would, so + * the results are bit-identical to four separate loops; only the interleaving + * differs. + */ +function dotRows4(q, a, ao, b, bo, c, co, d, dof, dim, out) { + let s0 = 0; + let s1 = 0; + let s2 = 0; + let s3 = 0; + for (let j = 0; j < dim; j++) { + const x = q[j]; + s0 += x * a[ao + j]; + s1 += x * b[bo + j]; + s2 += x * c[co + j]; + s3 += x * d[dof + j]; + } + out[0] = s0; + out[1] = s1; + out[2] = s2; + out[3] = s3; +} /** * In-memory embedding storage with brute-force cosine similarity search. * @@ -1321,7 +1464,7 @@ function lowerBound(values, target) { */ export class EmbeddingStore { constructor(encoder, options = {}) { - this._embeddings = new Map(); + this._table = null; this._dimension = null; this._onModelMismatch = 'warn'; this._warnedMismatch = new Set(); @@ -1331,7 +1474,6 @@ export class EmbeddingStore { this._annMinSize = ANN_MIN_SIZE; this._annSeed = 0; this._annCounts = { approximate: 0, exact: 0, widened: 0, builds: 0 }; - this._scan = null; /** Metadata of the most recent imported sidecar (empty before any). */ this.lastImportMeta = {}; this._encoder = adoptEncoder(encoder); @@ -1413,20 +1555,29 @@ export class EmbeddingStore { throw new DimensionMismatchError(`${what} has ${vector.length} dimensions, store holds ${this._dimension}`); } } - _cosineSimilarity(v1, v2) { - let dot = 0; - let norm1 = 0; - let norm2 = 0; - for (let i = 0; i < v1.length; i++) { - dot += v1[i] * v2[i]; - norm1 += v1[i] * v1[i]; - norm2 += v2[i] * v2[i]; - } - norm1 = Math.sqrt(norm1); - norm2 = Math.sqrt(norm2); - if (norm1 === 0 || norm2 === 0) + /** Euclidean norm, summed in index order (the order the stored norms are summed in). */ + static _norm(v) { + let sum = 0; + for (let i = 0; i < v.length; i++) + sum += v[i] * v[i]; + return Math.sqrt(sum); + } + /** + * Cosine of a query against the row at `slot`: float64, the dot product summed + * in dimension order, 0 when either norm is 0. The one definition every exact + * path shares. + */ + static _rowScore(table, q, qnorm, slot) { + const rn = table.norms[slot]; + if (qnorm === 0 || rn === 0) return 0; - return dot / (norm1 * norm2); + const dim = table.dim; + const chunk = table.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + let dot = 0; + for (let j = 0; j < dim; j++) + dot += q[j] * chunk[base + j]; + return dot / (qnorm * rn); } _resolveQuery(query) { let vector; @@ -1470,8 +1621,14 @@ export class EmbeddingStore { */ models() { const counts = new Map(); - for (const record of this._embeddings.values()) { - counts.set(record.model, (counts.get(record.model) ?? 0) + 1); + const table = this._table; + if (table !== null) { + for (let slot = 0; slot < table.n; slot++) { + if (table.keys[slot] === null) + continue; + const model = table.modelNames[table.modelIx[slot]]; + counts.set(model, (counts.get(model) ?? 0) + 1); + } } return Object.fromEntries(counts); } @@ -1524,13 +1681,19 @@ export class EmbeddingStore { } const includeUnknown = options.includeUnknown ?? false; const out = []; - for (const record of this._embeddings.values()) { + const table = this._table; + if (table === null) + return out; + const byDim = dimension !== null && table.dim !== dimension; + for (let slot = 0; slot < table.n; slot++) { + if (table.keys[slot] === null) + continue; + const rowModel = table.modelNames[table.modelIx[slot]]; const byModel = model !== null && - record.model !== model && - (includeUnknown || !UNKNOWN_MODELS.has(record.model)); - const byDim = dimension !== null && record.vector.length !== dimension; + rowModel !== model && + (includeUnknown || !UNKNOWN_MODELS.has(rowModel)); if (byModel || byDim) - out.push(record.nodeRef); + out.push(table.refs[slot]); } return out; } @@ -1543,14 +1706,18 @@ export class EmbeddingStore { } this._checkDimension(vector, "vector"); const key = this._nodeRefToKey(nodeRef); - this._set(key, { id: key, nodeRef, vector, text, model: this._modelName() }); + this._put(key, nodeRef, text, this._modelName(), vector); return key; } - /** Every write goes through here, so the approximate index's cache stays in step. */ - _set(key, record) { - this._embeddings.set(key, record); - if (this._scan !== null) - this._scan.put(key, record); + /** + * Every write goes through here: the row is copied into the table (the one + * copy of the vector), and an existing approximate index is told about it. + */ + _put(key, ref, text, model, vector) { + let table = this._table; + if (table === null) + table = this._table = new VectorTable(this._dimension); + table.put(key, ref, text, model, vector); } /** Add many embeddings, encoding their texts in one batched call. */ addBatch(entries) { @@ -1640,16 +1807,35 @@ export class EmbeddingStore { this._checkModelCompat(); return this._checkQuery((await this.encodeTextsAsync([query]))[0]); } + /** + * The stored record of a node, or null. The record is a copy: `vector` is a + * fresh array of the stored float32 values, so changing it changes nothing in + * the store (use {@link add} to replace a vector). + */ get(nodeRef) { - const key = this._nodeRefToKey(nodeRef); - return this._embeddings.get(key) || null; + const table = this._table; + if (table === null) + return null; + const slot = table.index.get(this._nodeRefToKey(nodeRef)); + if (slot === undefined) + return null; + return { + id: table.keys[slot], + nodeRef: table.refs[slot], + vector: table.vectorAt(slot), + text: table.texts[slot], + model: table.modelNames[table.modelIx[slot]], + }; + } + /** Whether a node has an embedding (no vector is copied). */ + has(nodeRef) { + return this._table !== null && this._table.index.has(this._nodeRefToKey(nodeRef)); } remove(nodeRef) { - const key = this._nodeRefToKey(nodeRef); - const removed = this._embeddings.delete(key); - if (removed && this._scan !== null) - this._scan.drop(key); - return removed; + const table = this._table; + if (table === null) + return false; + return table.remove(this._nodeRefToKey(nodeRef)); } /** * Find the most similar nodes by embedding. @@ -1675,20 +1861,108 @@ export class EmbeddingStore { } return this._scanExact(queryVector, topK, nodeType, threshold); } - /** The brute-force scan every search used before the approximate index existed. */ + /** + * The brute-force scan every search used before the approximate index existed. + * + * Reads the rows straight from the table, in slot (= arrival) order: float64 + * dot products over the float32 values, summed in dimension order, so a score + * is the same number whatever else is in the store. A bounded `topK` is kept + * in a small sorted buffer (a hit is only allocated as a result if it makes + * the cut); ties keep arrival order, exactly as {@link selectTopK} does. + */ _scanExact(queryVector, topK, nodeType, threshold) { - const results = []; - for (const record of this._embeddings.values()) { - if (nodeType && record.nodeRef[0] !== nodeType) + const table = this._table; + if (table === null || table.live === 0) + return []; + if (topK !== null && topK <= 0) + return []; + let code = -1; + if (nodeType) { + code = table.codeOf.get(nodeType) ?? -2; + if (code === -2) + return []; // no node of that type exists + } + const bounded = topK !== null && Number.isInteger(topK) && topK <= EXACT_TOPK_BUFFER; + const k = bounded ? topK : 0; + const topRows = new Int32Array(k); + const topScores = new Float64Array(k); + let count = 0; + const all = []; + const { dim, chunks, norms, codes, keys, refs, texts } = table; + const qnorm = EmbeddingStore._norm(queryVector); + const consider = (slot, score) => { + if (!(score >= threshold)) + return; + if (bounded) { + if (count === k && score <= topScores[k - 1]) + return; + let i = count < k ? count++ : k - 1; + while (i > 0 && topScores[i - 1] < score) { + topScores[i] = topScores[i - 1]; + topRows[i] = topRows[i - 1]; + i--; + } + topScores[i] = score; + topRows[i] = slot; + } + else { + all.push({ nodeRef: refs[slot], score, text: texts[slot] }); + } + }; + // Four rows at a time: four independent running sums hide the latency of the + // adds. Each is still summed in dimension order, so a score is the same + // number the one-row loop would give. Rows are handed to `consider` in slot + // order, so ties break as they always did. + const pend = new Int32Array(4); + const dots = new Float64Array(4); + let np = 0; + const flush = () => { + if (np === 4) { + const s0 = pend[0], s1 = pend[1], s2 = pend[2], s3 = pend[3]; + dotRows4(queryVector, chunks[s0 >>> CHUNK_SHIFT], (s0 & CHUNK_MASK) * dim, chunks[s1 >>> CHUNK_SHIFT], (s1 & CHUNK_MASK) * dim, chunks[s2 >>> CHUNK_SHIFT], (s2 & CHUNK_MASK) * dim, chunks[s3 >>> CHUNK_SHIFT], (s3 & CHUNK_MASK) * dim, dim, dots); + } + else { + for (let i = 0; i < np; i++) { + const slot = pend[i]; + const chunk = chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + let dot = 0; + for (let j = 0; j < dim; j++) + dot += queryVector[j] * chunk[base + j]; + dots[i] = dot; + } + } + for (let i = 0; i < np; i++) { + const rn = norms[pend[i]]; + consider(pend[i], rn === 0 ? 0 : dots[i] / (qnorm * rn)); + } + np = 0; + }; + for (let slot = 0; slot < table.n; slot++) { + if (keys[slot] === null) + continue; + if (code >= 0 && codes[slot] !== code) continue; - const score = this._cosineSimilarity(queryVector, record.vector); - if (score >= threshold) { - results.push({ nodeRef: record.nodeRef, score, text: record.text }); + if (qnorm === 0) { + consider(slot, 0); + continue; + } + pend[np++] = slot; + if (np === 4) + flush(); + } + if (np > 0) + flush(); + if (bounded) { + const out = new Array(count); + for (let i = 0; i < count; i++) { + out[i] = { nodeRef: refs[topRows[i]], score: topScores[i], text: texts[topRows[i]] }; } + return out; } if (topK === null) - return results.sort((a, b) => b.score - a.score); - return selectTopK(results, topK); + return all.sort((a, b) => b.score - a.score); + return selectTopK(all, topK); } /** {@link similaritySearch} with a text query encoded by awaiting the encoder. */ async similaritySearchAsync(query, topK = 10, nodeType, threshold = 0.0, exact = false) { @@ -1718,32 +1992,32 @@ export class EmbeddingStore { * the index is retrained from the current data, with an automatic `nlist` * re-chosen for the new size; * - a cleared store, a dimension change, or a store that shrinks below - * `annMinSize` drops the index with its cache. + * `annMinSize` drops the index. + * + * The index reads the store's own rows; there is no second copy of the vectors. */ _annIndex() { if (this._ann === null || this._dimension === null) return null; - const live = this._embeddings.size; - if (live < Math.max(2, this._annMinSize)) { - this._scan = null; + const table = this._table; + if (table === null || table.live < Math.max(2, this._annMinSize)) { + if (table !== null) + table.ivf = null; return null; } - let scan = this._scan; - if (scan === null) { - scan = this._scan = ScanCache.build(this._embeddings.values(), live, this._dimension); - } - let ivf = scan.ivf; + const live = table.live; + let ivf = table.ivf; if (ivf !== null && ivf.churn > ANN_RETRAIN_FRACTION * ivf.trainedN) - ivf = scan.ivf = null; + ivf = table.ivf = null; if (ivf === null) { const nlist = this._annNlist ?? EmbeddingStore._annAutoNlist(live); - scan.ivf = IvfIndex.build(scan, nlist, this._annSeed); + table.ivf = IvfIndex.build(table, nlist, this._annSeed); this._annCounts.builds++; } else if (ivf.pending.size >= Math.max(ANN_FOLD_MIN, ANN_FOLD_FRACTION * ivf.trainedN)) { - ivf.fold(scan); + ivf.fold(table); } - return scan; + return table; } _annProbe(ivf) { const wanted = this._annNprobe ?? Math.max(1, roundHalfEven(0.05 * ivf.nlist)); @@ -1762,7 +2036,7 @@ export class EmbeddingStore { return lists >= ivf.nlist ? null : lists; return lists * 2 >= ivf.nlist ? null : lists; } - /** Load the query into the cache's float32 buffer; null when it has no direction (all zeros). */ + /** Load the query into the table's float32 buffer; null when it has no direction (all zeros). */ _annQuery(scan, queryVector, nodeType) { const q = scan.query; let sum = 0; @@ -1808,8 +2082,8 @@ export class EmbeddingStore { const best = ivf.best(m, topK); const results = []; for (let i = 0; i < best.length; i++) { - const record = this._embeddings.get(scan.keys[ivf.candRows[best[i]]]); - results.push({ nodeRef: record.nodeRef, score: ivf.candScores[best[i]], text: record.text }); + const row = ivf.candRows[best[i]]; + results.push({ nodeRef: scan.refs[row], score: ivf.candScores[best[i]], text: scan.texts[row] }); } this._annCounts.approximate++; if (scale > 1) @@ -1824,11 +2098,15 @@ export class EmbeddingStore { */ annPolish(results, queryVector, threshold) { const out = []; + const table = this._table; + if (table === null) + return out; + const qnorm = EmbeddingStore._norm(queryVector); for (const result of results) { - const record = this._embeddings.get(this._nodeRefToKey(result.nodeRef)); - if (!record) + const slot = table.index.get(this._nodeRefToKey(result.nodeRef)); + if (slot === undefined) continue; - const score = this._cosineSimilarity(queryVector, record.vector); + const score = EmbeddingStore._rowScore(table, queryVector, qnorm, slot); if (score >= threshold) out.push({ nodeRef: result.nodeRef, score, text: result.text }); } @@ -1861,12 +2139,12 @@ export class EmbeddingStore { for (let i = 0; i < m; i++) slots[i] = i; slots.sort((a, b) => scores[b] - scores[a] || rows[a] - rows[b]); - const keys = scan.keys; - const embeddings = this._embeddings; + const refs = scan.refs; + const texts = scan.texts; const ranked = (function* () { for (let i = 0; i < m; i++) { - const record = embeddings.get(keys[rows[slots[i]]]); - yield { nodeRef: record.nodeRef, score: scores[slots[i]], text: record.text }; + const row = rows[slots[i]]; + yield { nodeRef: refs[row], score: scores[slots[i]], text: texts[row] }; } })(); return { ranked, approximate: true }; @@ -1885,8 +2163,8 @@ export class EmbeddingStore { if (this._ann === null) { throw new EmbeddingError("this store has no approximate index; create it with ann: 'ivf'"); } - if (force && this._scan !== null) - this._scan.ivf = null; + if (force && this._table !== null) + this._table.ivf = null; const scan = this._annIndex(); if (scan !== null && scan.ivf !== null && scan.ivf.pending.size > 0) scan.ivf.fold(scan); @@ -1894,7 +2172,7 @@ export class EmbeddingStore { } /** What the approximate index looks like right now. Never builds anything. */ annStats() { - const scan = this._scan; + const scan = this._table; const ivf = scan === null ? null : scan.ivf; return { enabled: this._ann !== null, @@ -1907,7 +2185,8 @@ export class EmbeddingStore { churn: ivf === null ? 0 : ivf.churn, buildSeconds: ivf === null ? null : ivf.buildSeconds, indexBytes: ivf === null ? 0 : ivf.bytes(), - cacheBytes: scan === null ? 0 : scan.bytes(), + cacheBytes: 0, + vectorBytes: scan === null ? 0 : scan.bytes(), minSize: this._annMinSize, searches: { ...this._annCounts }, }; @@ -1976,7 +2255,7 @@ export class EmbeddingStore { const mid = ordered.length >> 1; return ordered.length % 2 ? ordered[mid] : 0.5 * (ordered[mid - 1] + ordered[mid]); }; - const ivf = this._scan === null ? null : this._scan.ivf; + const ivf = this._table === null ? null : this._table.ivf; return { recall: recalls.length ? recalls.reduce((a, b) => a + b, 0) / recalls.length : 1, minRecall: recalls.length ? Math.min(...recalls) : 1, @@ -1998,15 +2277,28 @@ export class EmbeddingStore { scoreMap(query, nodeType) { const queryVector = this._resolveQuery(query); const scores = new Map(); - for (const [key, record] of this._embeddings) { - if (nodeType && record.nodeRef[0] !== nodeType) + const table = this._table; + if (table === null) + return scores; + let code = -1; + if (nodeType) { + code = table.codeOf.get(nodeType) ?? -2; + if (code === -2) + return scores; + } + const qnorm = EmbeddingStore._norm(queryVector); + for (let slot = 0; slot < table.n; slot++) { + const key = table.keys[slot]; + if (key === null) + continue; + if (code >= 0 && table.codes[slot] !== code) continue; - scores.set(key, this._cosineSimilarity(queryVector, record.vector)); + scores.set(key, EmbeddingStore._rowScore(table, queryVector, qnorm, slot)); } return scores; } count() { - return this._embeddings.size; + return this._table === null ? 0 : this._table.live; } /** * Clear all embeddings. @@ -2017,8 +2309,7 @@ export class EmbeddingStore { * wrong length even when empty. */ clear(resetDimension = false) { - this._embeddings.clear(); - this._scan = null; + this._table = null; if (resetDimension) this._dimension = null; } @@ -2032,14 +2323,18 @@ export class EmbeddingStore { */ exportEmbeddings(graphDigest) { const embeddings = []; - for (const record of this._embeddings.values()) { + const table = this._table; + for (let slot = 0; table !== null && slot < table.n; slot++) { + const ref = table.refs[slot]; + if (ref === null) + continue; embeddings.push({ - type: String(record.nodeRef[0]), - id: record.nodeRef[1], - id_type: typeof record.nodeRef[1] === 'number' ? 'int' : 'str', - text: record.text, - model: record.model, - vector: record.vector, + type: String(ref[0]), + id: ref[1], + id_type: typeof ref[1] === 'number' ? 'int' : 'str', + text: table.texts[slot], + model: table.modelNames[table.modelIx[slot]], + vector: table.vectorAt(slot), }); } const payload = { @@ -2086,17 +2381,29 @@ export class EmbeddingStore { const ref = [item.type, id]; this._checkDimension(item.vector, `embedding for :${item.type}:${id}`); const key = this._nodeRefToKey(ref); - this._set(key, { - id: key, - nodeRef: ref, - vector: item.vector, - text: item.text ?? '', - model: item.model ?? 'unknown', - }); + this._put(key, ref, item.text ?? '', item.model ?? 'unknown', item.vector); written++; } return written; } + /** + * The table, the encoder's model and the version, for the snapshot writer + * (see `snapshot.ts`). Low-level: not part of the stable API. + */ + snapshotState() { + return { table: this._table, encoderModel: this._currentModel(), encoderVersion: this.version }; + } + /** + * Replace every embedding with the rows of a verified, fully built table + * (see `snapshot.ts`), as `importEmbeddings(payload, true)` would, and record + * the snapshot's metadata as the last import's. Low-level: not part of the + * stable API. Any approximate index is rebuilt lazily. + */ + adoptTable(table, meta) { + this._table = table.live === 0 ? null : table; + this._dimension = table.live === 0 ? null : table.dim; + this.lastImportMeta = meta; + } } // ============================================================================= // Graph digest @@ -3068,7 +3375,7 @@ export class SemanticGraph extends ISONGraph { const fn = textFn || defaultTextFn; const batch = []; for (const node of this.nodes()) { - if (skipExisting && this._embeddingStore.get(node.ref)) + if (skipExisting && this._embeddingStore.has(node.ref)) continue; const text = fn(node); if (text) @@ -3081,7 +3388,7 @@ export class SemanticGraph extends ISONGraph { const fn = textFn || ((node) => this.embedTextFor(node.properties) || `${node.type}:${node.id}`); const batch = []; for (const node of this.nodes()) { - if (skipExisting && this._embeddingStore.get(node.ref)) + if (skipExisting && this._embeddingStore.has(node.ref)) continue; const text = fn(node); if (text) diff --git a/isongraph-vector-js/src/node.js b/isongraph-vector-js/src/node.js index e4367fa..520d516 100644 --- a/isongraph-vector-js/src/node.js +++ b/isongraph-vector-js/src/node.js @@ -11,15 +11,19 @@ */ import { - readFileSync, writeFileSync, existsSync, openSync, fsyncSync, closeSync, renameSync, unlinkSync, + readFileSync, writeFileSync, existsSync, statSync, openSync, fsyncSync, closeSync, renameSync, + unlinkSync, fstatSync, readSync, writeSync, } from 'node:fs'; -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { basename, dirname, extname, join } from 'node:path'; import { ISONGraph } from 'ison-graph-js'; import { SemanticGraph, EmbeddingError, GraphEmbeddingMismatchError, GRAPH_DIGEST_BASIS, EMBEDDING_SIDECAR_SUFFIX, graphDigest, } from './index.js'; +import { + SNAPSHOT_SUFFIX, SnapshotError, writeSnapshot, readSnapshot, applySnapshot, findUnrestorableId, +} from './snapshot.js'; /** Suffix appended to a graph path to locate its embeddings. */ export { EMBEDDING_SIDECAR_SUFFIX }; @@ -95,6 +99,162 @@ export function loadEmbeddings(graph, path, replace = false) { return (graph.embeddingStore ?? graph).importEmbeddings(payload, replace); } +// ============================================================================= +// Binary snapshot (optional, beside the JSON sidecar) +// ============================================================================= +export { SNAPSHOT_SUFFIX, SnapshotError }; +/** `.embeddings.snap`. */ +export function snapshotPathFor(path) { + return path + SNAPSHOT_SUFFIX; +} +/** SHA-256 over `node:crypto`, in the shape the browser-safe snapshot module asks for. */ +function sha256() { + const hash = createHash('sha256'); + return { + update: bytes => { + hash.update(bytes); + }, + digest: () => new Uint8Array(hash.digest()), + }; +} +/** Bytes sampled from the head and tail of a sidecar, and per probe (see DESIGN.md 11.1). */ +const FP_EDGE = 64 * 1024; +const FP_PROBE = 4 * 1024; +const FP_PROBES = 64; +/** + * Identity of a sidecar file: its length and a SHA-256 over the length (u64 LE), + * its first and last 64 KiB and 64 probes of 4 KiB at `floor(len * i / 65)`. + * Any serializer rewriting the sidecar changes it; an in-place edit that keeps + * the length and touches no sampled byte does not. + */ +export function sidecarIdentity(path) { + const fd = openSync(path, 'r'); + try { + const len = fstatSync(fd).size; + const hash = createHash('sha256'); + const head = Buffer.alloc(8); + head.writeBigUInt64LE(BigInt(len)); + hash.update(head); + const take = (at, want) => { + const n = Math.min(want, Math.max(0, len - at)); + const buf = Buffer.alloc(n); + let got = 0; + while (got < n) { + const read = readSync(fd, buf, got, n - got, at + got); + if (read === 0) + throw new EmbeddingError(`${path} changed while its identity was being read`); + got += read; + } + hash.update(buf); + }; + take(0, FP_EDGE); + take(Math.max(0, len - FP_EDGE), FP_EDGE); + for (let i = 1; i <= FP_PROBES; i++) + take(Number((BigInt(len) * BigInt(i)) / BigInt(FP_PROBES + 1)), FP_PROBE); + return { len, fingerprint: new Uint8Array(hash.digest()) }; + } + finally { + closeSync(fd); + } +} +/** + * Write whatever `fill` streams to `write` into `path`, so a reader sees the old + * file or the new one: a temp file in the same directory, flushed, renamed over + * `path`. A failure removes the temp file and leaves `path` as it was. + */ +function writeStreamAtomic(path, fill) { + const tmp = join(dirname(path), `.${basename(path)}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`); + let fd; + try { + fd = openSync(tmp, 'w'); + const handle = fd; + const value = fill(bytes => { + let at = 0; + while (at < bytes.length) + at += writeSync(handle, bytes, at, bytes.length - at); + }); + try { + fsyncSync(fd); + } + catch { + // fsync unsupported on this filesystem; the rename still gives atomicity. + } + closeSync(fd); + fd = undefined; + renameSync(tmp, path); + return value; + } + catch (error) { + if (fd !== undefined) { + try { + closeSync(fd); + } + catch { /* already failing */ } + } + try { + unlinkSync(tmp); + } + catch { /* temp file never created */ } + throw error; + } +} +/** + * Write a binary snapshot of every embedding in `store` to `path`, atomically (a + * temp file, flushed, renamed over `path`: a failure or crash leaves the + * previous file untouched). `identity` ties it to the graph and sidecar it was + * saved with; a load that finds a different identity refuses it. Prefer + * `saveGraph(graph, path, { snapshot: true })`, which does all of that. + */ +export function saveSnapshot(store, path, identity = {}, options = {}) { + return writeStreamAtomic(path, write => writeSnapshot(store, identity, sha256(), write, options)); +} +function readSnapshotFile(path, options) { + const fd = openSync(path, 'r'); + try { + const size = fstatSync(fd).size; + return readSnapshot(into => readSync(fd, into, 0, into.length, null), size, sha256(), options); + } + finally { + closeSync(fd); + } +} +/** + * Replace the store's embeddings with those of a snapshot file. Nothing changes + * unless the whole file verifies (structure and checksum); throws SnapshotError + * otherwise. Does not check identity (pass `accept` to), and ids come back as + * strings unless `resolveId` says otherwise: use `loadGraph`, which does both. + */ +export function loadSnapshot(store, path, options = {}) { + const result = readSnapshotFile(path, options); + applySnapshot(store, result); + return result.info; +} +/** + * How a snapshot's id strings map back to node ids, given the graph they were + * saved with: an id that names a node of that type as a string stays a string; + * a canonical integer string that names a node as a number becomes the number; + * anything else stays a string. (The snapshot format carries no id type; the + * JSON sidecar's `id_type` is what this stands in for.) + */ +function idResolverFor(graph) { + return (type, id) => { + if (graph.hasNode(type, id)) + return id; + const n = Number(id); + if (id !== '' && String(n) === id && graph.hasNode(type, n)) + return n; + return id; + }; +} +function removeQuietly(path) { + try { + unlinkSync(path); + } + catch { + // no such file, or it cannot be removed: either way there is nothing to keep stale + } +} + /** * Save a graph to a file, and its embeddings alongside it. * @@ -114,16 +274,49 @@ export function loadEmbeddings(graph, path, replace = false) { * @param options.format 'ison', 'isonl' or 'auto' (by extension; default) * @param options.embeddings write the sidecar (default true; false for * graph-only saves) + * @param options.snapshot also write a binary snapshot of the embeddings at + * `.embeddings.snap` (default false), after and + * in addition to the JSON sidecar, which stays the + * portable copy. loadGraph uses it, when it provably + * belongs to the graph and sidecar on disk, instead + * of parsing the JSON. A save without it deletes an + * existing snapshot, which the new sidecar would make + * stale. If the snapshot cannot be written the graph + * and sidecar are saved anyway, a stale snapshot is + * removed, and the error is thrown (a SnapshotError + * when the store holds something the snapshot cannot + * carry, such as an integer id for a node the graph + * does not have). */ export function saveGraph(graph, path, options = {}) { - const { format = 'auto', embeddings = true } = options; + const { format = 'auto', embeddings = true, snapshot = false } = options; const kind = resolveFormat(path, format); const text = kind === 'isonl' ? graph.toIsonl() : graph.toIson(); writeFileAtomic(path, text); if (embeddings && graph.embeddingStore) { const stem = basename(String(path), extname(String(path))); const reread = kind === 'isonl' ? ISONGraph.fromIsonl(text, stem) : ISONGraph.fromIson(text, stem); - saveEmbeddings(graph, embeddingsPathFor(path), graphDigest(reread)); + const digest = graphDigest(reread); + const sidecar = embeddingsPathFor(path); + const snap = snapshotPathFor(path); + if (!snapshot) + removeQuietly(snap); // the sidecar is about to change: a snapshot of the old one is stale + saveEmbeddings(graph, sidecar, digest); + if (snapshot) { + try { + const problem = findUnrestorableId(graph.embeddingStore, idResolverFor(reread)); + if (problem !== null) + throw new SnapshotError(`not written: ${problem}`); + saveSnapshot(graph.embeddingStore, snap, { + graphDigest: digest, + sidecar: sidecarIdentity(sidecar), + }); + } + catch (error) { + removeQuietly(snap); + throw error; + } + } } } @@ -136,6 +329,17 @@ function hasInexactNumberLiteral(text) { return /(?:^|[\s|])[-+]?(?:\d+\.0+|\d{16,})(?=$|[\s|])/m.test(text); } +/** Warn when the sidecar's (or snapshot's) encoder version is not the one in use. */ +function warnOnVersionSkew(graph) { + const storedVersion = graph.embeddingStore.lastImportMeta.encoder_version; + const currentVersion = graph.embeddingStore.version; + if (storedVersion !== undefined && currentVersion !== null && + String(storedVersion) !== String(currentVersion)) { + console.warn(`embeddings were made by encoder version '${storedVersion}', ` + + `but the encoder in use is version '${currentVersion}'`); + } +} + /** * Load a SemanticGraph from a file, restoring embeddings from its sidecar. * @@ -165,8 +369,27 @@ function hasInexactNumberLiteral(text) { * the graph file: 'error' (default) throws * GraphEmbeddingMismatchError, 'warn' loads the vectors anyway, * 'reembed' discards them and re-embeds every node + * + * A binary snapshot beside the sidecar (saveGraph with `snapshot: true`, or the + * Rust crate's `save_with_snapshot`) is used instead of the JSON when its header + * names this graph's digest and the sidecar file as it is now and its checksum + * verifies; the result is the same store the JSON gives. Any other outcome + * falls back to the JSON, exactly as without a snapshot. Use loadGraphWithReport + * to see which one was used. */ export function loadGraph(path, options = {}) { + return loadGraphWithReport(path, options).graph; +} + +/** + * loadGraph, also saying where the embeddings came from: + * `{ graph, report: { source, snapshotSkipped, embeddings, seconds } }`, where + * `source` is 'snapshot' (the binary file), 'json' (the sidecar) or 'none' + * (embeddings off, or no sidecar) and `snapshotSkipped` is why a snapshot that + * exists was not used (null when it was used or there is none). + */ +export function loadGraphWithReport(path, options = {}) { + const started = typeof performance !== 'undefined' ? performance.now() : Date.now(); const { format = 'auto', embeddings = true, encoder, autoEmbed, embedFields, version, onModelMismatch, onMismatch = 'error', @@ -205,57 +428,97 @@ export function loadGraph(path, options = {}) { } } - if (embeddings) { - const sidecar = embeddingsPathFor(path); - if (existsSync(sidecar)) { - let payload; - try { - payload = JSON.parse(readFileSync(sidecar, 'utf8')); - } - catch (error) { - throw new EmbeddingError(`unreadable embeddings sidecar ${sidecar}: ${error.message}`); - } + const report = (source, snapshotSkipped, count) => ({ + graph, + report: { + source, + snapshotSkipped, + embeddings: count, + seconds: ((typeof performance !== 'undefined' ? performance.now() : Date.now()) - started) / 1000, + }, + }); + if (!embeddings) + return report('none', null, 0); + const sidecar = embeddingsPathFor(path); + if (!(existsSync(sidecar) && statSync(sidecar).isFile())) + return report('none', null, 0); - let problem = null; - let unverifiable = false; - const recorded = payload && typeof payload === 'object' ? payload.graph_digest : undefined; - if (recorded !== undefined && recorded !== null && - (payload.graph_digest_basis ?? GRAPH_DIGEST_BASIS) === GRAPH_DIGEST_BASIS) { - const actual = graphDigest(base); - if (recorded !== actual) { - unverifiable = hasInexactNumberLiteral(text); - problem = - `embeddings sidecar ${sidecar} does not belong to graph ${path}: it was written ` + - `for graph digest ${recorded}, but the graph reads as ${actual} (the sidecar is ` + - `from a different save, or the graph was edited since). ` + - (unverifiable - ? `The graph file holds number literals (integral floats such as 1.0, or ` + - `integers beyond 2^53) that this port reads differently from Python, so the ` + - `digests may differ for an unedited file; using the sidecar's vectors anyway.` - : `Pass onMismatch: 'reembed' to rebuild the vectors or onMismatch: 'warn' to ` + - `use them anyway.`); - } - } + let actualDigest = null; + const digestNow = () => (actualDigest ??= graphDigest(base)); - if (problem !== null && onMismatch === 'error' && !unverifiable) { - throw new GraphEmbeddingMismatchError(problem); - } - if (problem !== null && onMismatch === 'reembed' && !unverifiable) { - graph.embedAllNodes(); - } - else { - if (problem !== null) - console.warn(problem); - graph.embeddingStore.importEmbeddings(payload); - const storedVersion = graph.embeddingStore.lastImportMeta.encoder_version; - const currentVersion = graph.embeddingStore.version; - if (storedVersion !== undefined && currentVersion !== null && - String(storedVersion) !== String(currentVersion)) { - console.warn(`embeddings were made by encoder version '${storedVersion}', ` + - `but the encoder in use is version '${currentVersion}'`); - } - } + // The snapshot first: a graph digest equal to this graph's own (no leniency + // for number literals read differently from Python), the sidecar as it is + // now, a valid body and checksum. Anything else and the JSON is read. + let snapshotSkipped = null; + const snap = snapshotPathFor(path); + if (existsSync(snap) && statSync(snap).isFile()) { + try { + const current = sidecarIdentity(sidecar); + const digest = digestNow(); + const result = readSnapshotFile(snap, { + resolveId: idResolverFor(base), + accept: info => { + if (info.graphDigest !== digest || info.graphDigestBasis !== GRAPH_DIGEST_BASIS) { + return 'it was saved for a different graph (digest differs)'; + } + if (info.sidecar === null) + return 'it records no sidecar identity'; + if (info.sidecar.len !== current.len || + !info.sidecar.fingerprint.every((b, i) => b === current.fingerprint[i])) { + return 'the JSON sidecar changed since the snapshot was written (stale snapshot)'; + } + return null; + }, + }); + applySnapshot(graph.embeddingStore, result); + warnOnVersionSkew(graph); + return report('snapshot', null, graph.embeddingStore.count()); + } + catch (error) { + snapshotSkipped = error instanceof SnapshotError ? error.reason : error.message; } } - return graph; + + let payload; + try { + payload = JSON.parse(readFileSync(sidecar, 'utf8')); + } + catch (error) { + throw new EmbeddingError(`unreadable embeddings sidecar ${sidecar}: ${error.message}`); + } + + let problem = null; + let unverifiable = false; + const recorded = payload && typeof payload === 'object' ? payload.graph_digest : undefined; + if (recorded !== undefined && recorded !== null && + (payload.graph_digest_basis ?? GRAPH_DIGEST_BASIS) === GRAPH_DIGEST_BASIS) { + const actual = digestNow(); + if (recorded !== actual) { + unverifiable = hasInexactNumberLiteral(text); + problem = + `embeddings sidecar ${sidecar} does not belong to graph ${path}: it was written ` + + `for graph digest ${recorded}, but the graph reads as ${actual} (the sidecar is ` + + `from a different save, or the graph was edited since). ` + + (unverifiable + ? `The graph file holds number literals (integral floats such as 1.0, or ` + + `integers beyond 2^53) that this port reads differently from Python, so the ` + + `digests may differ for an unedited file; using the sidecar's vectors anyway.` + : `Pass onMismatch: 'reembed' to rebuild the vectors or onMismatch: 'warn' to ` + + `use them anyway.`); + } + } + + if (problem !== null && onMismatch === 'error' && !unverifiable) { + throw new GraphEmbeddingMismatchError(problem); + } + if (problem !== null && onMismatch === 'reembed' && !unverifiable) { + graph.embedAllNodes(); + } + else { + if (problem !== null) + console.warn(problem); + graph.embeddingStore.importEmbeddings(payload); + warnOnVersionSkew(graph); + } + return report('json', snapshotSkipped, graph.embeddingStore.count()); } diff --git a/isongraph-vector-js/src/snapshot.js b/isongraph-vector-js/src/snapshot.js new file mode 100644 index 0000000..66d61a9 --- /dev/null +++ b/isongraph-vector-js/src/snapshot.js @@ -0,0 +1,523 @@ +/** + * Binary snapshot of an EmbeddingStore (optional, beside the JSON sidecar). + * + * The byte layout is version 1 of the snapshot format in the Rust crate's + * DESIGN.md, section 11.2; every integer is little endian and every float an + * IEEE-754 binary32 (read and written through a DataView with an explicit byte + * order, with a plain byte copy as the fast path on little-endian hosts). The + * JSON sidecar stays the portable, authoritative format; a snapshot is a cache + * of it, taken only when it provably belongs to the graph and sidecar on disk. + * + * This module is browser-safe: it touches no file and no `node:` API, and it + * does not hash anything itself. The SHA-256 trailer is computed by a + * {@link Sha256Like} the caller supplies (`node:crypto` in `./node`, which also + * does the file handling and the sidecar fingerprint). + * + * Differences from the Rust crate that are forced by this port: + * - rows are written in the order of this store's JSON sidecar (arrival order, + * live rows only), so that a snapshot load and a JSON load give the same + * slots; the Rust crate's sidecar happens to be sorted, ours is not; + * - the format has no `id_type`, so an integer node id comes back as a string + * unless the caller's `resolveId` says otherwise (`./node` asks the graph). + */ +import { EmbeddingError, VectorTable, GRAPH_DIGEST_BASIS, hostIsLittleEndian, } from './index.js'; +/** Appended to the graph file's name: `.embeddings.snap`. */ +export const SNAPSHOT_SUFFIX = '.embeddings.snap'; +/** The only layout version this build reads and writes. */ +export const SNAPSHOT_VERSION = 1; +const MAGIC = 'ISONVSNP'; +/** Rows per block (a writer's choice, recorded in the header; readers honour it). */ +const BLOCK_ROWS = 1024; +/** Length value of an absent optional string. */ +const ABSENT = 0xffffffff; +/** Header flag: the header carries the sidecar's identity. */ +const FLAG_SIDECAR = 1; +const KNOWN_FLAGS = FLAG_SIDECAR; +const TRAILER = 32; +const TWO_32 = 4294967296; +const DEFAULT_WINDOW = 4 << 20; +/** The snapshot is not usable (malformed, truncated, stale, or not the one asked for). */ +export class SnapshotError extends EmbeddingError { + constructor(reason) { + super(`invalid snapshot: ${reason}`); + this.name = 'SnapshotError'; + this.reason = reason; + } +} +const encoder = new TextEncoder(); +const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }); +// ============================================================================= +// Writing +// ============================================================================= +/** Appends little-endian values to a growing buffer. */ +class ByteWriter { + constructor(capacity) { + this.len = 0; + this.buf = new Uint8Array(capacity); + this.dv = new DataView(this.buf.buffer); + } + reset() { + this.len = 0; + } + /** Make room for `n` more bytes. */ + reserve(n) { + if (this.len + n <= this.buf.length) + return; + const grown = new Uint8Array(Math.max(this.len + n, this.buf.length * 2)); + grown.set(this.buf.subarray(0, this.len)); + this.buf = grown; + this.dv = new DataView(grown.buffer); + } + u32(value) { + this.reserve(4); + this.dv.setUint32(this.len, value, true); + this.len += 4; + } + u64(value) { + this.reserve(8); + this.dv.setUint32(this.len, value % TWO_32, true); + this.dv.setUint32(this.len + 4, Math.floor(value / TWO_32), true); + this.len += 8; + } + bytes(data) { + this.reserve(data.length); + this.buf.set(data, this.len); + this.len += data.length; + } + /** A length-prefixed string, already encoded. */ + str(data) { + this.u32(data.length); + this.bytes(data); + } + /** An optional string: absent is a length of 0xFFFFFFFF. */ + opt(value) { + if (value === null || value === undefined) + this.u32(ABSENT); + else + this.str(encoder.encode(value)); + } + view() { + return this.buf.subarray(0, this.len); + } +} +/** A string the format can carry: UTF-8 cannot hold a lone surrogate. */ +function wellFormed(s) { + const check = s.isWellFormed; + if (typeof check === 'function') + return check.call(s); + return !/[\ud800-\udbff](?![\udc00-\udfff])|(? 1 << 20) { + throw new EmbeddingError(`blockRows must be an integer in 1..${1 << 20}, got ${blockRows}`); + } + const littleEndian = hostIsLittleEndian() && !options.portable; + const { table, encoderModel, encoderVersion } = store.snapshotState(); + const slots = []; + if (table !== null) + for (let s = 0; s < table.n; s++) + if (table.keys[s] !== null) + slots.push(s); + const count = slots.length; + const dim = table === null || count === 0 ? 0 : table.dim; + // models, in order of first appearance + const modelIndex = new Map(); + const models = []; + if (table !== null) { + for (const s of slots) { + const name = table.modelNames[table.modelIx[s]]; + if (!modelIndex.has(name)) { + modelIndex.set(name, models.length); + models.push(name); + } + } + } + const digest = identity.graphDigest ?? null; + const basis = digest === null ? null : GRAPH_DIGEST_BASIS; + const sidecar = identity.sidecar ?? null; + if (sidecar !== null && sidecar.fingerprint.length !== 32) { + throw new EmbeddingError(`sidecar fingerprint must be 32 bytes, got ${sidecar.fingerprint.length}`); + } + const out = new ByteWriter(4096); + out.bytes(encoder.encode(MAGIC)); + out.u32(SNAPSHOT_VERSION); + out.u32(sidecar !== null ? FLAG_SIDECAR : 0); + out.u32(dim); + out.u32(blockRows); + out.u64(count); + out.opt(digest); + out.opt(basis); + out.opt(encoderModel); + out.opt(encoderVersion); + if (sidecar !== null) { + out.u64(sidecar.len); + out.bytes(sidecar.fingerprint); + } + out.u32(models.length); + for (const name of models) + out.str(encodeText(name, 'a model name')); + let written = 0; + const emit = () => { + const bytes = out.view(); + hasher.update(bytes); + sink(bytes); + written += bytes.length; + }; + emit(); + for (let start = 0; start < count; start += blockRows) { + const rows = Math.min(blockRows, count - start); + out.reset(); + for (let i = 0; i < rows; i++) { + const s = slots[start + i]; + const ref = table.refs[s]; + out.str(encodeText(String(ref[0]), 'a node type')); + out.str(encodeText(String(ref[1]), 'a node id')); + out.str(encodeText(table.texts[s], 'an embedding text')); + out.u32(modelIndex.get(table.modelNames[table.modelIx[s]])); + } + out.reserve(rows * dim * 4); + for (let i = 0; i < rows; i++) { + const s = slots[start + i]; + const chunk = table.chunks[s >>> 10]; + const base = (s & 1023) * dim; + if (littleEndian) { + out.buf.set(new Uint8Array(chunk.buffer, chunk.byteOffset + base * 4, dim * 4), out.len); + out.len += dim * 4; + } + else { + for (let j = 0; j < dim; j++) { + out.dv.setFloat32(out.len, chunk[base + j], true); + out.len += 4; + } + } + } + emit(); + } + const trailer = hasher.digest(); + if (trailer.length !== TRAILER) + throw new EmbeddingError('the hasher did not return 32 bytes'); + sink(trailer); + written += TRAILER; + return { + version: SNAPSHOT_VERSION, + dimension: dim, + count, + graphDigest: digest, + graphDigestBasis: basis, + encoderModel, + encoderVersion, + sidecar, + bytes: written, + }; +} +/** {@link writeSnapshot} into memory. */ +export function snapshotToBytes(store, identity, hasher, options = {}) { + const pieces = []; + const info = writeSnapshot(store, identity, hasher, piece => pieces.push(piece.slice()), options); + const bytes = new Uint8Array(info.bytes); + let at = 0; + for (const piece of pieces) { + bytes.set(piece, at); + at += piece.length; + } + return { bytes, info }; +} +// ============================================================================= +// Reading +// ============================================================================= +/** + * A read-ahead window over a stream of known size. It hands out the next `n` + * bytes of the body (everything before the trailer) as a view that is valid until + * the next call, hashes each byte of the body once as it enters the window, and + * refuses to hand out more than the body holds. + */ +class Window { + constructor(_fill, size, _hasher, capacity) { + this._fill = _fill; + this._hasher = _hasher; + this.pos = 0; + this.end = 0; + this.consumed = 0; + this._pulled = 0; + this._body = size - TRAILER; + this.buf = new Uint8Array(capacity); + this.dv = new DataView(this.buf.buffer); + } + remaining() { + return this._body - this.consumed; + } + /** Make `n` bytes available at `pos`. */ + _ensure(n) { + if (this.end - this.pos >= n) + return; + if (this.pos > 0) { + this.buf.copyWithin(0, this.pos, this.end); + this.end -= this.pos; + this.pos = 0; + } + if (this.buf.length < n) { + const grown = new Uint8Array(Math.max(n, this.buf.length * 2)); + grown.set(this.buf.subarray(0, this.end)); + this.buf = grown; + this.dv = new DataView(grown.buffer); + } + while (this.end < n) { + const got = this._fill(this.buf.subarray(this.end)); + if (!(got > 0)) + throw new SnapshotError('truncated'); + const hashable = Math.max(0, Math.min(got, this._body - this._pulled)); + if (hashable > 0) + this._hasher.update(this.buf.subarray(this.end, this.end + hashable)); + this._pulled += got; + this.end += got; + } + } + take(n) { + if (n > this.remaining()) + throw new SnapshotError('truncated'); + this._ensure(n); + const view = this.buf.subarray(this.pos, this.pos + n); + this.pos += n; + this.consumed += n; + return view; + } + u32() { + if (4 > this.remaining()) + throw new SnapshotError('truncated'); + this._ensure(4); + const value = this.dv.getUint32(this.pos, true); + this.pos += 4; + this.consumed += 4; + return value; + } + u64() { + const low = this.u32(); + const high = this.u32(); + return high * TWO_32 + low; + } + /** A length-prefixed string; `optional` lets the length be ABSENT. */ + string(optional) { + const len = this.u32(); + if (len === ABSENT) { + if (optional) + return null; + throw new SnapshotError('a required string is marked absent'); + } + if (len > this.remaining()) + throw new SnapshotError('a string is longer than the file'); + const bytes = this.take(len); + try { + return decoder.decode(bytes); + } + catch { + throw new SnapshotError('a string is not UTF-8'); + } + } + /** The trailer and a check that nothing follows it. Call after the body is consumed. */ + trailer() { + this._ensureRaw(TRAILER); + const view = this.buf.slice(this.pos, this.pos + TRAILER); + this.pos += TRAILER; + return view; + } + _ensureRaw(n) { + if (this.end - this.pos >= n) + return; + if (this.pos > 0) { + this.buf.copyWithin(0, this.pos, this.end); + this.end -= this.pos; + this.pos = 0; + } + while (this.end < n) { + const got = this._fill(this.buf.subarray(this.end)); + if (!(got > 0)) + throw new SnapshotError('truncated'); + this._pulled += got; + this.end += got; + } + } + /** True when the stream has more bytes than the size it was declared with. */ + hasMore() { + if (this.end > this.pos) + return true; + const probe = new Uint8Array(1); + return this._fill(probe) > 0; + } +} +function sameBytes(a, b) { + if (a.length !== b.length) + return false; + for (let i = 0; i < a.length; i++) + if (a[i] !== b[i]) + return false; + return true; +} +/** + * Read and fully verify a snapshot of `size` bytes whose bytes `fill` supplies, + * applying every rejection rule of the format: bad magic, version or flags, + * implausible shape, a length past the end, a model index out of range, a + * repeated key, bytes between the last block and the trailer or after it, a + * checksum mismatch. The rows go into a fresh table; nothing else is touched, so + * a failure (a SnapshotError, or whatever `fill` throws) leaves every store as + * it was. Install the result with {@link applySnapshot}. + */ +export function readSnapshot(fill, size, hasher, options = {}) { + if (size < TRAILER + MAGIC.length) + throw new SnapshotError('file too short'); + const portable = options.portable ?? false; + const resolveId = options.resolveId ?? ((_type, id) => id); + const w = new Window(fill, size, hasher, Math.max(64, options.windowBytes ?? DEFAULT_WINDOW)); + if (!sameBytes(w.take(MAGIC.length), encoder.encode(MAGIC))) + throw new SnapshotError('not a snapshot (bad magic)'); + const version = w.u32(); + if (version !== SNAPSHOT_VERSION) { + throw new SnapshotError(`unsupported snapshot version ${version} (this build reads ${SNAPSHOT_VERSION})`); + } + const flags = w.u32(); + if ((flags & ~KNOWN_FLAGS) !== 0) + throw new SnapshotError(`unknown flags 0x${flags.toString(16)}`); + const dim = w.u32(); + const blockRows = w.u32(); + const count = w.u64(); + if (blockRows === 0 || blockRows > 1 << 20) + throw new SnapshotError(`implausible block size ${blockRows}`); + if (count > 0xffffffff || (count > 0 && dim === 0)) + throw new SnapshotError(`implausible shape ${count} x ${dim}`); + if (count * dim * 4 > w.remaining()) + throw new SnapshotError('fewer bytes than the rows need'); + const graphDigest = w.string(true); + const graphDigestBasis = w.string(true); + const encoderModel = w.string(true); + const encoderVersion = w.string(true); + let sidecar = null; + if ((flags & FLAG_SIDECAR) !== 0) { + const len = w.u64(); + sidecar = { len, fingerprint: w.take(32).slice() }; + } + const modelCount = w.u32(); + if (modelCount > w.remaining() / 4) + throw new SnapshotError('more models than the file can hold'); + const models = []; + for (let i = 0; i < modelCount; i++) + models.push(w.string(false)); + const info = { + version, + dimension: dim, + count, + graphDigest, + graphDigestBasis, + encoderModel, + encoderVersion, + sidecar, + bytes: size, + }; + const refusal = options.accept ? options.accept(info) : null; + if (refusal !== null) + throw new SnapshotError(refusal); + const table = count > 0 ? new VectorTable(dim) : null; + let left = count; + while (left > 0) { + const rows = Math.min(left, blockRows); + const types = new Array(rows); + const ids = new Array(rows); + const texts = new Array(rows); + const modelOf = new Int32Array(rows); + for (let i = 0; i < rows; i++) { + types[i] = w.string(false); + ids[i] = w.string(false); + texts[i] = w.string(false); + const m = w.u32(); + if (m >= models.length) + throw new SnapshotError('a row names a model that is not listed'); + modelOf[i] = m; + } + const need = rows * dim * 4; + if (need > w.remaining()) + throw new SnapshotError('truncated'); + const floats = w.take(need); + for (let i = 0; i < rows; i++) { + const key = `${types[i]}:${ids[i]}`; + if (table.index.has(key)) + throw new SnapshotError(`the key ${key} appears twice`); + const ref = [types[i], resolveId(types[i], ids[i])]; + table.putLittleEndian(key, ref, texts[i], models[modelOf[i]], floats.subarray(i * dim * 4, (i + 1) * dim * 4), portable); + } + left -= rows; + } + if (w.remaining() !== 0) + throw new SnapshotError('bytes left over after the last block'); + const expected = hasher.digest(); + const stored = w.trailer(); + if (!sameBytes(stored, expected)) + throw new SnapshotError('checksum mismatch'); + if (w.hasMore()) + throw new SnapshotError('bytes after the checksum'); + return { table, info }; +} +/** {@link readSnapshot} from a byte array. */ +export function readSnapshotBytes(bytes, hasher, options = {}) { + let at = 0; + const fill = into => { + const n = Math.min(into.length, bytes.length - at); + if (n <= 0) + return 0; + into.set(bytes.subarray(at, at + n)); + at += n; + return n; + }; + return readSnapshot(fill, bytes.length, hasher, options); +} +/** + * Replace the store's embeddings with a verified snapshot's rows (and remember + * its header as the store's `lastImportMeta`, as a JSON import does). The + * approximate-index settings stay; the index is rebuilt lazily. + */ +export function applySnapshot(store, result) { + const info = result.info; + const meta = {}; + if (info.encoderModel !== null) + meta.encoder_model = info.encoderModel; + if (info.encoderVersion !== null) + meta.encoder_version = info.encoderVersion; + if (info.graphDigest !== null) + meta.graph_digest = info.graphDigest; + if (info.graphDigestBasis !== null) + meta.graph_digest_basis = info.graphDigestBasis; + store.adoptTable(result.table ?? new VectorTable(1), meta); +} diff --git a/isongraph-vector-js/src/snapshot.test.js b/isongraph-vector-js/src/snapshot.test.js new file mode 100644 index 0000000..c45d6fc --- /dev/null +++ b/isongraph-vector-js/src/snapshot.test.js @@ -0,0 +1,576 @@ +/** + * The optional binary snapshot (DESIGN.md section 11 of the Rust crate): the + * golden files the Rust crate wrote, a save -> load round trip of our own, + * every rejection rule with its fall-back to the JSON sidecar, the save + * protocol, and the single-copy table against a plain reference model. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import { mkdtempSync, rmSync, readFileSync, writeFileSync, copyFileSync, existsSync, readdirSync, } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { EmbeddingStore, SemanticGraph, MockEncoder, graphDigest, selectTopK, } from './index.js'; +import { SnapshotError, snapshotToBytes, readSnapshotBytes, applySnapshot, } from './snapshot.js'; +import { saveGraph, loadGraphWithReport, loadSnapshot, saveSnapshot, snapshotPathFor, sidecarIdentity, embeddingsPathFor, } from './node.js'; +const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'tests', 'fixtures', 'snapshot'); +const dirs = []; +function tmp() { + const dir = mkdtempSync(join(tmpdir(), 'igv-snap-')); + dirs.push(dir); + return dir; +} +afterEach(() => { + while (dirs.length) + rmSync(dirs.pop(), { recursive: true, force: true }); +}); +function sha() { + const h = createHash('sha256'); + return { update: b => { h.update(b); }, digest: () => new Uint8Array(h.digest()) }; +} +/** Recompute the trailer so that only the structural rule under test can reject the file. */ +function reseal(bytes) { + const body = bytes.subarray(0, bytes.length - 32); + const out = new Uint8Array(bytes.length); + out.set(body); + out.set(createHash('sha256').update(body).digest(), body.length); + return out; +} +function rng(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} +/** Everything observable about a store's contents: order, keys, ids, texts, models and the vector bits. */ +function dump(store) { + return store.exportEmbeddings().embeddings.map(e => [ + e.type, e.id, e.id_type, e.text, e.model, + Array.from(new Uint32Array(new Float32Array(e.vector).buffer)), + ]); +} +/** A graph with numeric and string ids, two models, an update, a delete and a duplicate vector. */ +function sampleGraph() { + const graph = new SemanticGraph({ name: 'snap', encoder: new MockEncoder(8, 'v1'), autoEmbed: false }); + const next = rng(11); + const vec = () => Array.from({ length: 8 }, () => Math.fround(next() * 2 - 1)); + for (let i = 1; i <= 30; i++) { + graph.addNode('person', i, { name: `Person ${i}` }); + if (i === 12) + graph.embeddingStore.add(['person', i], 'Person 12'); // from the encoder + else + graph.embeddingStore.add(['person', i], `Person ${i} \u{1F600} "quoted"\nnewline`, vec()); + } + graph.embeddingStore.add(['doc', 'a:b'], 'id with a colon', vec()); // an embedding without a graph node (an id may hold a colon here) + graph.embeddingStore.add(['person', 3], 'updated', vec()); // update in place + graph.embeddingStore.remove(['person', 7]); // tombstone + const dup = vec(); + graph.embeddingStore.add(['doc', 'x'], 'dup 1', dup); // an exact tie + graph.addNode('doc', 'x', { name: 'x' }); + graph.embeddingStore.add(['doc', 'y'], 'dup 2', dup); + graph.addNode('doc', 'y', { name: 'y' }); + graph.embeddingStore.add(['doc', 'z'], '', new Array(8).fill(0)); // a zero vector + graph.addNode('doc', 'z', { name: 'z' }); + return graph; +} +function sameAnswers(a, b, dim, seed = 5) { + const next = rng(seed); + for (let i = 0; i < 12; i++) { + const q = Array.from({ length: dim }, () => next() * 2 - 1); + for (const [k, type, th] of [[10, undefined, -1], [3, 'doc', 0], [null, undefined, 0.2]]) { + expect(JSON.stringify(a.similaritySearch(q, k, type, th))).toBe(JSON.stringify(b.similaritySearch(q, k, type, th))); + } + } +} +describe('snapshot written by the Rust crate (golden fixture)', () => { + function copyFixture(dir, withSnap) { + const path = join(dir, 'golden.ison'); + for (const name of ['golden.ison', 'golden.ison.embeddings.json', ...(withSnap ? ['golden.ison.embeddings.snap'] : [])]) { + copyFileSync(join(FIXTURES, name), join(dir, name)); + } + return path; + } + it('is read through the snapshot, and equals a JSON-only load of the same graph and sidecar', () => { + const withSnap = loadGraphWithReport(copyFixture(tmp(), true)); + const jsonOnly = loadGraphWithReport(copyFixture(tmp(), false)); + expect(withSnap.report.source).toBe('snapshot'); + expect(withSnap.report.snapshotSkipped).toBeNull(); + expect(withSnap.report.embeddings).toBe(36); + expect(jsonOnly.report.source).toBe('json'); + const a = withSnap.graph.embeddingStore; + const b = jsonOnly.graph.embeddingStore; + expect(a.count()).toBe(36); + expect(a.dimension).toBe(12); + expect(dump(a)).toEqual(dump(b)); // keys, texts, models, vector bits, order + expect(a.models()).toEqual(b.models()); + expect(a.lastImportMeta).toEqual(b.lastImportMeta); + expect(a.lastImportMeta.encoder_model).toBe('second-model'); + sameAnswers(withSnap.graph, jsonOnly.graph, 12); + }); + it('is reproduced byte for byte by our writer from the loaded store', () => { + const dir = tmp(); + const path = copyFixture(dir, true); + const encoder = new MockEncoder(12); + encoder.modelName = 'second-model'; // the header records the encoder's model + const { graph } = loadGraphWithReport(path, { encoder }); + const meta = graph.embeddingStore.lastImportMeta; + const out = join(dir, 'again.snap'); + saveSnapshot(graph.embeddingStore, out, { + graphDigest: meta.graph_digest, + sidecar: sidecarIdentity(embeddingsPathFor(path)), + }); + expect(readFileSync(out)).toEqual(readFileSync(snapshotPathFor(path))); + }); + it('keeps unicode, colons, empty and multi-line texts, a zero vector and a tie', () => { + const { graph } = loadGraphWithReport(copyFixture(tmp(), true)); + const store = graph.embeddingStore; + const rows = dump(store); + expect(rows.some(r => r[3].includes('\u{1F600}'))).toBe(true); + expect(rows.some(r => r[1].includes('id:with:colons'))).toBe(true); + expect(rows.some(r => r[3] === '')).toBe(true); + expect(rows.some(r => r[3].includes('\n'))).toBe(true); + expect(rows.some(r => r[5].every(bits => bits === 0))).toBe(true); + expect(new Set(rows.map(r => r[4])).size).toBe(2); + }); + it('falls back to the JSON, untouched, when a copy of the snapshot is corrupt or truncated', () => { + const jsonOnly = loadGraphWithReport(copyFixture(tmp(), false)).graph.embeddingStore; + const original = new Uint8Array(readFileSync(join(FIXTURES, 'golden.ison.embeddings.snap'))); + const damaged = [ + ['a flipped float byte', (() => { const b = original.slice(); b[b.length - 100] ^= 0x40; return b; })()], + ['a flipped trailer byte', (() => { const b = original.slice(); b[b.length - 1] ^= 1; return b; })()], + ['half of the file', original.slice(0, original.length >> 1)], + ['one byte short', original.slice(0, original.length - 1)], + ['only the magic', original.slice(0, 8)], + ['an empty file', new Uint8Array(0)], + ]; + for (const [what, bytes] of damaged) { + const dir = tmp(); + const path = copyFixture(dir, false); + writeFileSync(snapshotPathFor(path), bytes); + const { graph, report } = loadGraphWithReport(path); + expect(report.source, what).toBe('json'); + expect(report.snapshotSkipped, what).not.toBeNull(); + expect(dump(graph.embeddingStore), what).toEqual(dump(jsonOnly)); + } + }); +}); +describe('snapshot round trip of our own save', () => { + it('saveGraph({ snapshot: true }) then loadGraph reads the snapshot and gives the same store', () => { + const dir = tmp(); + const path = join(dir, 'g.ison'); + const graph = sampleGraph(); + saveGraph(graph, path, { snapshot: true }); + expect(existsSync(snapshotPathFor(path))).toBe(true); + const viaSnap = loadGraphWithReport(path, { encoder: new MockEncoder(8, 'v1') }); + expect(viaSnap.report.source).toBe('snapshot'); + expect(viaSnap.report.embeddings).toBe(graph.embeddingStore.count()); + const noSnap = tmp(); + copyFileSync(path, join(noSnap, 'g.ison')); + copyFileSync(embeddingsPathFor(path), join(noSnap, 'g.ison.embeddings.json')); + const viaJson = loadGraphWithReport(join(noSnap, 'g.ison'), { encoder: new MockEncoder(8, 'v1') }); + expect(viaJson.report.source).toBe('json'); + expect(dump(viaSnap.graph.embeddingStore)).toEqual(dump(viaJson.graph.embeddingStore)); + expect(dump(viaSnap.graph.embeddingStore)).toEqual(dump(graph.embeddingStore)); // numeric ids stayed numbers + expect(viaSnap.graph.getEmbedding(['person', 3]).nodeRef).toEqual(['person', 3]); + expect(typeof viaSnap.graph.getEmbedding(['person', 3]).nodeRef[1]).toBe('number'); + expect(viaSnap.graph.getEmbedding(['person', 7])).toBeNull(); + sameAnswers(viaSnap.graph, viaJson.graph, 8); + // text search works against the restored store too + expect(viaSnap.graph.similaritySearch('Person 12', 1)[0].nodeRef).toEqual(['person', 12]); + }); + it('writes the graph file and the JSON sidecar exactly as it does without a snapshot', () => { + const graph = sampleGraph(); + const a = tmp(); + const b = tmp(); + saveGraph(graph, join(a, 'g.ison')); + saveGraph(graph, join(b, 'g.ison'), { snapshot: true }); + expect(readFileSync(join(b, 'g.ison'))).toEqual(readFileSync(join(a, 'g.ison'))); + expect(readFileSync(embeddingsPathFor(join(b, 'g.ison')))).toEqual(readFileSync(embeddingsPathFor(join(a, 'g.ison')))); + }); + it('spans several blocks, and the DataView path writes and reads the same bytes as the fast path', () => { + const store = new EmbeddingStore(); + const next = rng(3); + for (let i = 0; i < 2600; i++) { + store.add(['n', `id${i}`], `text ${i}`, Array.from({ length: 6 }, () => Math.fround(next() - 0.5))); + } + const fast = snapshotToBytes(store, {}, sha()); + const portable = snapshotToBytes(store, {}, sha(), { portable: true }); + expect(portable.bytes).toEqual(fast.bytes); + expect(fast.info.count).toBe(2600); + for (const options of [{}, { portable: true }, { windowBytes: 64 }, { windowBytes: 1000, portable: true }]) { + const copy = new EmbeddingStore(); + applySnapshot(copy, readSnapshotBytes(fast.bytes, sha(), options)); + expect(dump(copy)).toEqual(dump(store)); + } + const small = snapshotToBytes(store, {}, sha(), { blockRows: 7 }); // the reader honours the header's block size + const copy = new EmbeddingStore(); + applySnapshot(copy, readSnapshotBytes(small.bytes, sha())); + expect(dump(copy)).toEqual(dump(store)); + }); + it('round-trips an empty store', () => { + const store = new EmbeddingStore(); + const { bytes } = snapshotToBytes(store, {}, sha()); + const again = new EmbeddingStore(); + applySnapshot(again, readSnapshotBytes(bytes, sha())); + expect(again.count()).toBe(0); + expect(again.dimension).toBeNull(); + }); + it('matches the byte layout of the specification example (123 bytes)', () => { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'x', [1.0, -2.0]); + store.version = null; + const { bytes } = snapshotToBytes(store, {}, sha()); + const u32 = (n) => [n & 255, (n >> 8) & 255, (n >> 16) & 255, (n >>> 24) & 255]; + const ascii = (s) => Array.from(s).map(c => c.charCodeAt(0)); + const expected = [ + ...ascii('ISONVSNP'), ...u32(1), ...u32(0), + ...u32(2), ...u32(1024), ...u32(1), ...u32(0), + ...u32(0xffffffff), ...u32(0xffffffff), ...u32(0xffffffff), ...u32(0xffffffff), + ...u32(1), ...u32(8), ...ascii('provided'), + ...u32(1), ...ascii('t'), ...u32(1), ...ascii('a'), ...u32(1), ...ascii('x'), ...u32(0), + 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0, + ]; + expect(expected).toHaveLength(91); + expect(bytes).toHaveLength(123); + expect(Array.from(bytes.subarray(0, 91))).toEqual(expected); + expect(Buffer.from(bytes.subarray(91)).toString('hex')).toBe(createHash('sha256').update(Uint8Array.from(expected)).digest('hex')); + }); +}); +describe('snapshot rejection rules', () => { + /** A two-row snapshot ('t','a','x') and ('t','b','y'), the store it came from, and its bytes. */ + function twoRows() { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'x', [1, 0, 0]); + store.add(['t', 'b'], 'y', [0, 1, 0]); + return { store, bytes: snapshotToBytes(store, {}, sha()).bytes }; + } + const dv = (b) => new DataView(b.buffer, b.byteOffset, b.byteLength); + function find(bytes, needle, from = 0) { + outer: for (let i = from; i <= bytes.length - needle.length; i++) { + for (let j = 0; j < needle.length; j++) + if (bytes[i + j] !== needle[j]) + continue outer; + return i; + } + throw new Error('pattern not found'); + } + const reject = (bytes, reason, options = {}) => { + const target = new EmbeddingStore(); + target.add(['keep', 1], 'kept', [9, 9, 9]); + const before = dump(target); + expect(() => applySnapshot(target, readSnapshotBytes(bytes, sha(), options))).toThrow(SnapshotError); + try { + readSnapshotBytes(bytes, sha(), options); + } + catch (error) { + expect(error.reason).toMatch(reason); + } + expect(dump(target)).toEqual(before); // nothing was installed + expect(target.dimension).toBe(3); + }; + it('rejects bad magic, an unknown version, unknown flags and an implausible shape', () => { + const { bytes } = twoRows(); + let b = bytes.slice(); + b[0] = 0x58; + reject(reseal(b), /bad magic/); + b = bytes.slice(); + dv(b).setUint32(8, 2, true); + reject(reseal(b), /version 2/); + b = bytes.slice(); + dv(b).setUint32(12, 2, true); + reject(reseal(b), /unknown flags/); + b = bytes.slice(); + dv(b).setUint32(20, 0, true); + reject(reseal(b), /block size/); + b = bytes.slice(); + dv(b).setUint32(20, (1 << 20) + 1, true); + reject(reseal(b), /block size/); + b = bytes.slice(); + dv(b).setUint32(16, 0, true); + reject(reseal(b), /implausible shape/); + b = bytes.slice(); + dv(b).setUint32(24, 0xffffffff, true); + dv(b).setUint32(28, 1, true); + reject(reseal(b), /implausible shape/); + b = bytes.slice(); + dv(b).setUint32(24, 1000, true); + reject(reseal(b), /fewer bytes than the rows need/); + }); + it('rejects a repeated key, a model index out of range and a string past the end', () => { + const { bytes } = twoRows(); + const dup = bytes.slice(); + const idB = find(dup, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x62]); // type 't', id 'b' + dup[idB + 9] = 0x61; // ... now 'a': the key t:a twice + reject(reseal(dup), /appears twice/); + const model = bytes.slice(); + const rowA = find(model, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61, 1, 0, 0, 0, 0x78]); + dv(model).setUint32(rowA + 15, 5, true); // model_index 5 of 1 + reject(reseal(model), /model that is not listed/); + const long = bytes.slice(); + dv(long).setUint32(find(long, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61]), 0x7ffffff0, true); + reject(reseal(long), /longer than the file/); + const absent = bytes.slice(); + dv(absent).setUint32(find(absent, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61]), 0xffffffff, true); + reject(reseal(absent), /marked absent/); + }); + it('rejects bytes between the last block and the trailer, bytes after it, and a wrong checksum', () => { + const { bytes } = twoRows(); + const between = new Uint8Array(bytes.length + 1); + between.set(bytes.subarray(0, bytes.length - 32)); + between.set(bytes.subarray(bytes.length - 32), bytes.length - 31); + reject(reseal(between), /left over/); + const after = new Uint8Array(bytes.length + 1); + after.set(bytes); + reject(after, /./); + const bad = bytes.slice(); + bad[bytes.length - 40] ^= 1; // inside the last row's floats + reject(bad, /checksum/); + }); + it('rejects a truncation at every length', () => { + const { bytes } = twoRows(); + for (let n = 0; n < bytes.length; n++) { + expect(() => readSnapshotBytes(bytes.slice(0, n), sha()), `length ${n}`).toThrow(SnapshotError); + } + }); + it('rejects a string that is not UTF-8', () => { + const { bytes } = twoRows(); + const b = bytes.slice(); + b[find(b, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61, 1, 0, 0, 0, 0x78]) + 14] = 0xff; // the text of row a + reject(reseal(b), /not UTF-8/); + }); + it('refuses what `accept` refuses before reading any row', () => { + const { bytes } = twoRows(); + expect(() => readSnapshotBytes(bytes, sha(), { accept: () => 'not for you' })).toThrow(/not for you/); + }); + it('refuses to write a lone surrogate (UTF-8 cannot carry it)', () => { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'bad \ud800 text', [1]); + expect(() => snapshotToBytes(store, {}, sha())).toThrow(SnapshotError); + }); +}); +describe('snapshot protocol: which file wins, and why a snapshot is skipped', () => { + function saved() { + const dir = tmp(); + const path = join(dir, 'g.ison'); + const graph = sampleGraph(); + saveGraph(graph, path, { snapshot: true }); + return { dir, path, graph }; + } + it('skips a snapshot whose sidecar changed since (stale) and loads the new JSON', () => { + const { path, graph } = saved(); + const sidecar = embeddingsPathFor(path); + const payload = JSON.parse(readFileSync(sidecar, 'utf8')); + payload.embeddings[0].text = 'edited after the snapshot'; + writeFileSync(sidecar, JSON.stringify(payload)); + const { graph: loaded, report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/stale/); + expect(loaded.embeddingStore.get(['person', 1]).text).toBe('edited after the snapshot'); + expect(graph.embeddingStore.get(['person', 1]).text).not.toBe('edited after the snapshot'); + }); + it('skips a snapshot saved for a different graph digest', () => { + const { path, graph } = saved(); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { + graphDigest: 'sha256:' + '0'.repeat(64), + sidecar: sidecarIdentity(embeddingsPathFor(path)), + }); + const { report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/different graph/); + }); + it('skips a snapshot that records no digest or no sidecar identity', () => { + const { path, graph } = saved(); + const digest = graphDigest(graph); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { sidecar: sidecarIdentity(embeddingsPathFor(path)) }); + expect(loadGraphWithReport(path).report.snapshotSkipped).toMatch(/different graph/); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { graphDigest: digest }); + expect(loadGraphWithReport(path).report.snapshotSkipped).toMatch(/different graph|no sidecar identity/); + }); + it('skips a snapshot when the graph file was edited (with onMismatch: warn for the JSON)', () => { + const { path } = saved(); + writeFileSync(path, readFileSync(path, 'utf8').replace('Person 5', 'Person five')); + const quiet = () => { }; + const warn = console.warn; + console.warn = quiet; + try { + const { report } = loadGraphWithReport(path, { onMismatch: 'warn' }); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/different graph/); + } + finally { + console.warn = warn; + } + }); + it('never uses a snapshot when the sidecar is missing', () => { + const { path } = saved(); + rmSync(embeddingsPathFor(path)); + const { graph, report } = loadGraphWithReport(path); + expect(report.source).toBe('none'); + expect(graph.embeddingStore.count()).toBe(0); + }); + it('embeddings: false ignores the snapshot too', () => { + const { path } = saved(); + expect(loadGraphWithReport(path, { embeddings: false }).report.source).toBe('none'); + }); + it('a save without a snapshot deletes the old one, and a plain load then reads the JSON', () => { + const { path, graph } = saved(); + graph.embeddingStore.add(['person', 1], 'changed', new Array(8).fill(0.25)); + saveGraph(graph, path); + expect(existsSync(snapshotPathFor(path))).toBe(false); + const { graph: loaded, report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toBeNull(); + expect(loaded.embeddingStore.get(['person', 1]).text).toBe('changed'); + }); + it('a snapshot that cannot be written still saves the pair, removes a stale snapshot and throws', () => { + const { path, graph } = saved(); + graph.embeddingStore.add(['orphan', 5], 'an integer id for a node the graph does not have', new Array(8).fill(0.5)); + expect(() => saveGraph(graph, path, { snapshot: true })).toThrow(SnapshotError); + expect(existsSync(snapshotPathFor(path))).toBe(false); + expect(readdirSync(dirname(path)).filter(f => f.endsWith('.tmp'))).toEqual([]); + const { graph: loaded, report } = loadGraphWithReport(path); // the pair was saved: JSON reads fine + expect(report.source).toBe('json'); + expect(loaded.embeddingStore.get(['orphan', 5]).text).toMatch(/orphan|integer id/); + expect(typeof loaded.embeddingStore.get(['orphan', 5]).nodeRef[1]).toBe('number'); + }); + it('a failing write leaves the previous snapshot untouched and no temp file behind', () => { + const { dir, path, graph } = saved(); + const before = readFileSync(snapshotPathFor(path)); + const bad = new EmbeddingStore(); + bad.add(['t', 'a'], 'lone \ud800 surrogate', [1, 2]); + expect(() => saveSnapshot(bad, snapshotPathFor(path), {})).toThrow(SnapshotError); + expect(readFileSync(snapshotPathFor(path))).toEqual(before); + expect(readdirSync(dir).filter(f => f.endsWith('.tmp'))).toEqual([]); + expect(loadGraphWithReport(path).report.source).toBe('snapshot'); + expect(graph.embeddingStore.count()).toBeGreaterThan(0); + }); + it('loadSnapshot replaces the store only when the whole file verifies', () => { + const { path, graph } = saved(); + const snap = snapshotPathFor(path); + const store = new EmbeddingStore(); + store.add(['keep', 1], 'kept', new Array(8).fill(1)); + const bytes = new Uint8Array(readFileSync(snap)); + const broken = bytes.slice(); + broken[bytes.length - 50] ^= 8; + writeFileSync(snap, broken); + expect(() => loadSnapshot(store, snap)).toThrow(SnapshotError); + expect(store.count()).toBe(1); + expect(store.get(['keep', 1]).text).toBe('kept'); + writeFileSync(snap, bytes); + const info = loadSnapshot(store, snap); + expect(info.count).toBe(graph.embeddingStore.count()); + expect(store.get(['keep', 1])).toBeNull(); + expect(store.count()).toBe(info.count); + expect(store.get(['person', '1']).nodeRef).toEqual(['person', '1']); // no id type: a bare load reads ids as text + }); + it('is deterministic: saving the same store twice gives the same bytes', () => { + const { path, graph } = saved(); + const first = readFileSync(snapshotPathFor(path)); + saveGraph(graph, path, { snapshot: true }); + expect(readFileSync(snapshotPathFor(path))).toEqual(first); + }); +}); +describe('the single-copy table against a plain reference', () => { + /** What the store computed before the rework: float64 cosine over number[] copies, in arrival order. */ + class Reference { + constructor() { + this.rows = new Map(); + } + add(ref, text, vector) { + this.rows.set(`${ref[0]}:${ref[1]}`, { ref, text, vector: vector.map(Math.fround) }); + } + remove(ref) { + this.rows.delete(`${ref[0]}:${ref[1]}`); + } + static cosine(q, v) { + let dot = 0, n1 = 0, n2 = 0; + for (let i = 0; i < q.length; i++) { + dot += q[i] * v[i]; + n1 += q[i] * q[i]; + n2 += v[i] * v[i]; + } + n1 = Math.sqrt(n1); + n2 = Math.sqrt(n2); + return n1 === 0 || n2 === 0 ? 0 : dot / (n1 * n2); + } + search(q, k, type, threshold) { + const out = []; + for (const row of this.rows.values()) { + if (type && row.ref[0] !== type) + continue; + const score = Reference.cosine(q, row.vector); + if (score >= threshold) + out.push({ nodeRef: row.ref, score, text: row.text }); + } + return k === null ? out.sort((a, b) => b.score - a.score) : selectTopK(out, k); + } + scoreMap(q) { + return [...this.rows].map(([key, row]) => [key, Reference.cosine(q, row.vector)]); + } + } + it('returns the same ids, scores (bit for bit) and tie order after updates, deletes and compaction', () => { + const next = rng(21); + const dim = 9; + const vec = () => Array.from({ length: dim }, () => Math.fround(next() * 2 - 1)); + const store = new EmbeddingStore(); + const ref = new Reference(); + const types = ['a', 'b', 'c:d']; + const refOf = (i) => [types[i % 3], i % 5 === 0 ? `s${i}` : i]; + const pool = []; + for (let i = 0; i < 2300; i++) { // crosses two chunk boundaries + const r = next(); + const v = r < 0.06 && pool.length ? pool[Math.floor(next() * pool.length)] : r < 0.09 ? new Array(dim).fill(0) : vec(); + pool.push(v); + store.add(refOf(i), `t${i}`, v); + ref.add(refOf(i), `t${i}`, v); + } + for (let i = 0; i < 2300; i += 4) { + const v = vec(); + store.add(refOf(i), `u${i}`, v); + ref.add(refOf(i), `u${i}`, v); + } + for (let i = 1; i < 2300; i += 3) { + store.remove(refOf(i)); + ref.remove(refOf(i)); + } // more than a quarter: compaction + for (let i = 0; i < 30; i++) { + const v = vec(); + store.add(refOf(i), `again${i}`, v); + ref.add(refOf(i), `again${i}`, v); + } + expect(store.count()).toBe(ref.rows.size); + for (let i = 0; i < 25; i++) { + const q = i === 0 ? new Array(dim).fill(0) : i < 3 ? pool[i].slice() : vec(); + for (const k of [1, 7, 10, 300, 400, null]) + for (const type of [undefined, 'b', 'nope']) + for (const th of [0, -1, 0.4]) { + expect(store.similaritySearch(q, k, type, th), `k=${k} type=${type} th=${th}`).toEqual(ref.search(q, k, type, th)); + } + expect([...store.scoreMap(q)]).toEqual(ref.scoreMap(q)); + } + }); + it('gives a get() copy: changing its vector changes nothing in the store', () => { + const store = new EmbeddingStore(); + store.add(['t', 1], 'x', [1, 2, 3]); + const record = store.get(['t', 1]); + record.vector[0] = 99; + expect(store.get(['t', 1]).vector).toEqual([1, 2, 3]); + expect(store.similaritySearch([1, 2, 3], 1)[0].score).toBeCloseTo(1, 12); + }); + it('keeps one copy: the index reports no cache and the vectors take about 4 bytes per component', () => { + const store = new EmbeddingStore(undefined, { ann: 'ivf', annMinSize: 0, annNlist: 16, annNprobe: 4 }); + const next = rng(2); + for (let i = 0; i < 3000; i++) + store.add(['d', i], '', Array.from({ length: 32 }, () => next() - 0.5)); + store.buildAnnIndex(); + const stats = store.annStats(); + expect(stats.active).toBe(true); + expect(stats.cacheBytes).toBe(0); + expect(stats.vectorBytes).toBeGreaterThanOrEqual(3000 * 32 * 4); + expect(stats.vectorBytes).toBeLessThan(3000 * 32 * 4 * 1.2); + }); +}); diff --git a/isongraph-vector-ts/package.json b/isongraph-vector-ts/package.json index 9ef7fdd..d4a040b 100644 --- a/isongraph-vector-ts/package.json +++ b/isongraph-vector-ts/package.json @@ -13,7 +13,7 @@ "scripts": { "build": "tsc -p tsconfig.build.json", "test": "vitest run", - "test:core": "vitest run src/index.test.ts src/node.test.ts src/features.test.ts src/http-encoder.test.ts src/async-encoder.test.ts src/ann.test.ts", + "test:core": "vitest run src/index.test.ts src/node.test.ts src/features.test.ts src/http-encoder.test.ts src/async-encoder.test.ts src/ann.test.ts src/snapshot.test.ts", "prepublishOnly": "npm run build", "typecheck": "tsc --noEmit" }, @@ -61,6 +61,10 @@ "./node": { "types": "./dist/node.d.ts", "default": "./dist/node.js" + }, + "./snapshot": { + "types": "./dist/snapshot.d.ts", + "default": "./dist/snapshot.js" } } } diff --git a/isongraph-vector-ts/src/async-encoder.test.ts b/isongraph-vector-ts/src/async-encoder.test.ts index 85c1eff..7e980bb 100644 --- a/isongraph-vector-ts/src/async-encoder.test.ts +++ b/isongraph-vector-ts/src/async-encoder.test.ts @@ -174,7 +174,7 @@ describe('SemanticGraph auto-embed over HTTP (openai shape)', () => { expect(seen[0].body.input).toEqual(['Alice an engineer']); expect(seen[0].headers.authorization).toBe('Bearer sk-test-key'); const record = graph.getEmbedding(['person', 1])!; - expect(record.vector).toEqual(vectorOf('Alice an engineer')); + expect(record.vector).toEqual(vectorOf('Alice an engineer').map(Math.fround)); // stored as float32 expect(record.model).toBe('remote-4'); expect(graph.embeddingStore.dimension).toBe(4); @@ -219,7 +219,7 @@ describe('SemanticGraph auto-embed over HTTP (openai shape)', () => { await graph.updateNodeAsync('person', 1, { name: 'Alice the manager' }); expect(seen).toHaveLength(2); expect(graph.getEmbedding(['person', 1])!.text).toBe('Alice the manager'); - expect(graph.getEmbedding(['person', 1])!.vector).toEqual(vectorOf('manager')); + expect(graph.getEmbedding(['person', 1])!.vector).toEqual(vectorOf('manager').map(Math.fround)); // stored as float32 await graph.updateNodeAsync('person', 1, { name: null }); // nothing left to embed expect(graph.getEmbedding(['person', 1])).toBeNull(); diff --git a/isongraph-vector-ts/src/index.ts b/isongraph-vector-ts/src/index.ts index 48c5b37..a3c6608 100644 --- a/isongraph-vector-ts/src/index.ts +++ b/isongraph-vector-ts/src/index.ts @@ -862,6 +862,9 @@ export function selectTopK(results: SimilarityResult[], k: number): SimilarityRe // Approximate (IVF) search // ============================================================================= +/** An exact search for at most this many results keeps them in a small sorted buffer; a larger k collects every hit first. */ +const EXACT_TOPK_BUFFER = 256; + /** Stores below this many vectors answer exactly even with `ann` on. */ const ANN_MIN_SIZE = 50_000; /** New or updated rows are scored exactly until this many (or this fraction of the trained size) are pending. */ @@ -908,8 +911,8 @@ export interface AnnOptions { * unprobed cell is missed - so measure it with `recallCheck()`. The index is * built lazily on the first search, never persisted (a saved graph and its * sidecar are byte-for-byte what they would be without it), and kept - * current through writes and deletes. It keeps a Float32Array copy of the - * vectors (4 bytes per component) next to the stored ones. + * current through writes and deletes. It keeps no copy of the vectors: it reads + * the store's own float32 rows. */ ann?: AnnMode | null; /** Cells in the index. Default: about `4 * sqrt(N)` (at least 16, at most 65536) when it is built. */ @@ -939,8 +942,13 @@ export interface AnnStats { buildSeconds: number | null; /** Memory of the index itself (centroids, cell numbers, cell lists). */ indexBytes: number; - /** Memory of the Float32Array copy of the vectors the index scores. */ + /** + * Always 0 now: the index scores the store's own rows and keeps no copy of + * them. Kept so existing code reading it still compiles. + */ cacheBytes: number; + /** Memory of the store's one copy of the vectors (float32 chunks) and its numeric columns. */ + vectorBytes: number; minSize: number; /** Counts since the store was created. */ searches: { approximate: number; exact: number; widened: number; builds: number }; @@ -1164,114 +1172,212 @@ class CentroidSearch { } } +/** Rows per matrix chunk (= one scan block). Chunks never move once allocated, so growth never copies stored rows. */ +const CHUNK_ROWS = 1024; +const CHUNK_SHIFT = 10; +const CHUNK_MASK = CHUNK_ROWS - 1; +/** Rows the first chunk starts with; it doubles until it holds CHUNK_ROWS, later chunks are allocated whole. */ +const FIRST_CHUNK_ROWS = 16; + +/** True when this host stores a Float32Array in little-endian byte order (the snapshot's byte order). */ +export function hostIsLittleEndian(): boolean { + return new Uint8Array(new Uint32Array([1]).buffer)[0] === 1; +} + +const HOST_LITTLE_ENDIAN = hostIsLittleEndian(); + /** - * Float32Array snapshot of the store's vectors, kept in step with its writes. + * The store's single copy of every vector, plus the per-row facts around it. * - * One contiguous matrix with spare row capacity, the vector norms, a type code - * per row and a liveness flag. A write patches it in place: an insert appends - * a row, an update overwrites one, a delete tombstones one (squeezed out once - * a quarter of the rows are dead). Rows stay in insertion order, which is the - * order the store's Map iterates, so ties break exactly as the exact scan's do. - * Built lazily by the first approximate search, never persisted. + * A record *is a slot number*; nothing else owns the floats (see the Rust + * crate's DESIGN.md, section 1). The rows live in `Float32Array` chunks of + * {@link CHUNK_ROWS} rows each, `dim` floats per row, row-major; `norms`, + * `codes` and `modelIx` are parallel columns, and `keys`/`refs`/`texts` hold the + * strings. Slots are in arrival order, which is the order the store used to + * iterate its records in, so ties still break the way they always did: an update + * overwrites its slot in place, an insert takes `n`, a delete tombstones (key + * `null`, norm `NaN`). Trailing tombstones are popped; once a quarter of the + * slots are dead the live rows are moved down, keeping their order. + * + * Both exact search and the IVF index read the rows from here, so the index adds + * no second copy of the vectors. Not part of the stable API: it is exported so + * the snapshot module can read and fill it. */ -class ScanCache { +export class VectorTable { readonly dim: number; - matrix: Float32Array; - norms: Float64Array; - codes: Int32Array; - alive: Uint8Array; - /** Rows in use, tombstones included, and how many of those are tombstones. */ + readonly chunks: Float32Array[] = []; + /** Euclidean norm of each row, computed from the float32 values; NaN = tombstone. */ + norms = new Float64Array(0); + /** Node type of each row, as an index into `typeNames`. */ + codes = new Int32Array(0); + /** Model of each row, as an index into `modelNames`. */ + modelIx = new Int32Array(0); + /** `"type:id"` of each row, or null for a tombstone. */ + readonly keys: Array = []; + readonly refs: Array = []; + readonly texts: string[] = []; + readonly modelNames: string[] = []; + readonly typeNames: string[] = []; + readonly codeOf: Map = new Map(); + /** key -> slot: the only lookup structure. */ + readonly index: Map = new Map(); + private readonly _modelOf: Map = new Map(); + /** Slots in use, tombstones included, and how many of those are tombstones. */ n = 0; dead = 0; - keys: Array = []; - pos: Map = new Map(); - codeOf: Map = new Map(); ivf: IvfIndex | null = null; - /** The query as float32, reused between searches. */ - query: Float32Array; + /** The query as float32, reused between approximate searches. */ + readonly query: Float32Array; - constructor(dim: number, capacity: number) { + constructor(dim: number) { this.dim = dim; - this.matrix = new Float32Array(capacity * dim); - this.norms = new Float64Array(capacity); - this.codes = new Int32Array(capacity); - this.alive = new Uint8Array(capacity); this.query = new Float32Array(dim); } - static build(records: Iterable, count: number, dim: number): ScanCache { - const cache = new ScanCache(dim, count + Math.max(1024, count >> 4)); - for (const record of records) cache.put(record.id, record); - return cache; - } - get capacity(): number { return this.norms.length; } - private _grow(): void { - const capacity = this.capacity + Math.max(1024, this.capacity >> 2); - const matrix = new Float32Array(capacity * this.dim); - matrix.set(this.matrix.subarray(0, this.n * this.dim)); + /** Live rows. */ + get live(): number { + return this.index.size; + } + + private _modelId(name: string): number { + let id = this._modelOf.get(name); + if (id === undefined) { + id = this.modelNames.length; + this.modelNames.push(name); + this._modelOf.set(name, id); + } + return id; + } + + private _growColumns(): void { + const capacity = this.capacity + Math.max(1024, this.capacity >> 1); const norms = new Float64Array(capacity); norms.set(this.norms.subarray(0, this.n)); const codes = new Int32Array(capacity); codes.set(this.codes.subarray(0, this.n)); - const alive = new Uint8Array(capacity); - alive.set(this.alive.subarray(0, this.n)); - this.matrix = matrix; + const modelIx = new Int32Array(capacity); + modelIx.set(this.modelIx.subarray(0, this.n)); this.norms = norms; this.codes = codes; - this.alive = alive; + this.modelIx = modelIx; } - /** Mirror one upserted record. */ - put(key: string, record: EmbeddingRecord): void { + /** Make sure the chunk holding `slot` exists and is big enough. */ + private _ensureRow(slot: number): void { + const c = slot >>> CHUNK_SHIFT; const dim = this.dim; - let i = this.pos.get(key); - if (i === undefined) { - if (this.n >= this.capacity) this._grow(); - i = this.n++; - this.keys[i] = key; - this.pos.set(key, i); - const type = record.nodeRef[0]; - let code = this.codeOf.get(type); - if (code === undefined) { - code = this.codeOf.size; - this.codeOf.set(type, code); + if (c === this.chunks.length) { + this.chunks.push(new Float32Array((c === 0 ? FIRST_CHUNK_ROWS : CHUNK_ROWS) * dim)); + } else if (c === 0) { + const first = this.chunks[0]; + const need = ((slot & CHUNK_MASK) + 1) * dim; + if (first.length < need) { + const grown = new Float32Array(Math.min(CHUNK_ROWS * dim, Math.max(need, first.length * 2))); + grown.set(first); + this.chunks[0] = grown; } - this.codes[i] = code; - this.alive[i] = 1; } - const vector = record.vector; - const base = i * dim; + } + + /** Find or create the slot of `key` and set its metadata; the caller writes the row and the norm. */ + private _claim(key: string, ref: NodeRef, text: string, model: string): number { + const type = ref[0]; + let code = this.codeOf.get(type); + if (code === undefined) { + code = this.typeNames.length; + this.typeNames.push(type); + this.codeOf.set(type, code); + } + let slot = this.index.get(key); + if (slot === undefined) { + if (this.n >= this.capacity) this._growColumns(); + slot = this.n++; + this._ensureRow(slot); + this.keys[slot] = key; + this.index.set(key, slot); + } + this.refs[slot] = ref; + this.texts[slot] = text; + this.codes[slot] = code; + this.modelIx[slot] = this._modelId(model); + return slot; + } + + /** Insert or overwrite the row of `key`; returns its slot. The values are stored as float32. */ + put(key: string, ref: NodeRef, text: string, model: string, vector: ArrayLike): number { + const slot = this._claim(key, ref, text, model); + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; let sum = 0; for (let j = 0; j < dim; j++) { const f = Math.fround(vector[j]); - this.matrix[base + j] = f; + chunk[base + j] = f; sum += f * f; } - this.norms[i] = Math.sqrt(sum); - if (this.ivf !== null) this.ivf.notePut(i, this.capacity); + this.norms[slot] = Math.sqrt(sum); + if (this.ivf !== null) this.ivf.notePut(slot, this.capacity); + return slot; } - /** Mirror one deleted record. */ - drop(key: string): void { - const i = this.pos.get(key); - if (i === undefined) return; - this.pos.delete(key); - this.matrix.fill(0, i * this.dim, (i + 1) * this.dim); - this.norms[i] = 0; - this.alive[i] = 0; - this.keys[i] = null; + /** + * Like {@link put}, from `dim * 4` bytes of little-endian IEEE-754 float32 + * (the snapshot's row encoding). On a little-endian host the bytes are copied + * as they are; otherwise (or with `portable`) each float is read through a + * DataView. + */ + putLittleEndian( + key: string, + ref: NodeRef, + text: string, + model: string, + bytes: Uint8Array, + portable: boolean = false + ): number { + const slot = this._claim(key, ref, text, model); + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + if (HOST_LITTLE_ENDIAN && !portable) { + new Uint8Array(chunk.buffer, chunk.byteOffset + base * 4, dim * 4).set(bytes); + } else { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + for (let j = 0; j < dim; j++) chunk[base + j] = view.getFloat32(4 * j, true); + } + let sum = 0; + for (let j = 0; j < dim; j++) { + const f = chunk[base + j]; + sum += f * f; + } + this.norms[slot] = Math.sqrt(sum); + if (this.ivf !== null) this.ivf.notePut(slot, this.capacity); + return slot; + } + + /** Tombstone the row of `key`; false when there is none. */ + remove(key: string): boolean { + const slot = this.index.get(key); + if (slot === undefined) return false; + this.index.delete(key); + this.keys[slot] = null; + this.refs[slot] = null; + this.texts[slot] = ''; + this.norms[slot] = NaN; this.dead++; - if (this.ivf !== null) this.ivf.noteDrop(i); - while (this.n > 0 && this.alive[this.n - 1] === 0) { // trailing tombstones + if (this.ivf !== null) this.ivf.noteDrop(slot); + while (this.n > 0 && this.keys[this.n - 1] === null) { // trailing tombstones this.n--; this.keys.pop(); + this.refs.pop(); + this.texts.pop(); this.dead--; } if (this.dead > 32 && this.dead * 4 > this.n) this._compact(); + return true; } /** Squeeze the tombstones out, keeping the live rows in their order. */ @@ -1279,33 +1385,54 @@ class ScanCache { const dim = this.dim; const live = new Int32Array(this.n - this.dead); let kept = 0; - for (let r = 0; r < this.n; r++) if (this.alive[r] === 1) live[kept++] = r; + for (let r = 0; r < this.n; r++) if (this.keys[r] !== null) live[kept++] = r; if (this.ivf !== null) this.ivf.compact(live); for (let w = 0; w < kept; w++) { const r = live[w]; - if (r !== w) { - this.matrix.copyWithin(w * dim, r * dim, (r + 1) * dim); - this.norms[w] = this.norms[r]; - this.codes[w] = this.codes[r]; - const key = this.keys[r]!; - this.keys[w] = key; - this.pos.set(key, w); - } + if (r === w) continue; + const src = this.chunks[r >>> CHUNK_SHIFT]; + const dst = this.chunks[w >>> CHUNK_SHIFT]; + const so = (r & CHUNK_MASK) * dim; + const d = (w & CHUNK_MASK) * dim; + if (src === dst) src.copyWithin(d, so, so + dim); + else dst.set(src.subarray(so, so + dim), d); + this.norms[w] = this.norms[r]; + this.codes[w] = this.codes[r]; + this.modelIx[w] = this.modelIx[r]; + const key = this.keys[r]!; + this.keys[w] = key; + this.refs[w] = this.refs[r]; + this.texts[w] = this.texts[r]; + this.index.set(key, w); } - this.alive.fill(1, 0, kept); - this.alive.fill(0, kept, this.n); this.keys.length = kept; + this.refs.length = kept; + this.texts.length = kept; + this.chunks.length = Math.ceil(kept / CHUNK_ROWS); this.n = kept; this.dead = 0; } + /** The vector of `slot` as a fresh array of numbers (the float32 values, widened). */ + vectorAt(slot: number): number[] { + const dim = this.dim; + const chunk = this.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + const out = new Array(dim); + for (let j = 0; j < dim; j++) out[j] = chunk[base + j]; + return out; + } + + /** Bytes of the vectors and the numeric columns (not the strings). */ bytes(): number { - return this.matrix.byteLength + this.norms.byteLength + this.codes.byteLength + this.alive.byteLength; + let total = this.norms.byteLength + this.codes.byteLength + this.modelIx.byteLength; + for (const chunk of this.chunks) total += chunk.byteLength; + return total; } } /** - * Inverted-file ("IVF-flat") index over a {@link ScanCache}. + * Inverted-file ("IVF-flat") index over a {@link VectorTable}. * * Spherical k-means partitions the rows into `nlist` cells by direction, which * is what cosine similarity compares. A query scores the `nprobe` cells whose @@ -1314,7 +1441,7 @@ class ScanCache { * cell number per row (`assign`) plus the rows grouped by cell * (`order`/`offsets`), about 8 bytes per vector plus the centroids. * - * It follows its cache instead of being rebuilt on every write: + * It follows its table instead of being rebuilt on every write: * - a **new** or **updated** row is *pending* (cell -1): every query scores * the pending rows exactly, so a write is visible at once and costs the * index nothing; @@ -1322,7 +1449,7 @@ class ScanCache { * are enough of them (see `EmbeddingStore._annIndex`); * - a **delete** clears the row's cell; stale entries in `order` are skipped * at query time; - * - the cache's compaction renumbers rows, and `compact` carries the cells + * - the table's compaction renumbers rows, and `compact` carries the cells * across without retraining; * - `churn` counts writes and deletes since training; the store retrains when * it passes a fraction of the trained size. @@ -1354,12 +1481,12 @@ class IvfIndex { } /** Train centroids on a sample, then give every live row its cell. */ - static build(cache: ScanCache, nlistWanted: number, seed: number): IvfIndex { + static build(cache: VectorTable, nlistWanted: number, seed: number): IvfIndex { const started = nowMs(); - const { n, dim, matrix, norms, alive } = cache; + const { n, dim, chunks, norms, keys } = cache; const live = new Int32Array(n - cache.dead); let total = 0; - for (let r = 0; r < n; r++) if (alive[r] === 1) live[total++] = r; + for (let r = 0; r < n; r++) if (keys[r] !== null) live[total++] = r; const nlist = Math.max(1, Math.min(Math.floor(nlistWanted), total)); const rng = new Rng(seed); @@ -1383,9 +1510,10 @@ class IvfIndex { for (let i = 0; i < sampleN; i++) { const r = sample[i]; const inv = norms[r] > 0 ? 1 / norms[r] : 0; - const src = r * dim; + const chunk = chunks[r >>> CHUNK_SHIFT]; + const src = (r & CHUNK_MASK) * dim; const dst = i * dim; - for (let j = 0; j < dim; j++) x[dst + j] = matrix[src + j] * inv; + for (let j = 0; j < dim; j++) x[dst + j] = chunk[src + j] * inv; } // Initial centroids: distinct sample points. @@ -1435,7 +1563,7 @@ class IvfIndex { index.trainedN = total; for (let t = 0; t < total; t++) { const r = live[t]; - index.assign[r] = index._search.nearest(matrix, r * dim); + index.assign[r] = index._search.nearest(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim); } index.relist(n); index.buildSeconds = (nowMs() - started) / 1000; @@ -1466,10 +1594,11 @@ class IvfIndex { } /** Give every pending row its nearest cell and regroup. */ - fold(cache: ScanCache): void { + fold(cache: VectorTable): void { if (this.pending.size > 0) { + const chunks = cache.chunks; for (const r of this.pending) { - this.assign[r] = this._search.nearest(cache.matrix, r * this.dim); + this.assign[r] = this._search.nearest(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * this.dim); } this.pending.clear(); } @@ -1493,7 +1622,7 @@ class IvfIndex { this.churn++; } - /** Follow the cache's compaction (`live` = the old rows kept, in order). */ + /** Follow the table's compaction (`live` = the old rows kept, in order). */ compact(live: Int32Array): void { const kept = live.length; const moved = new Int32Array(kept); @@ -1516,7 +1645,7 @@ class IvfIndex { * into `candRows`/`candScores`; returns how many were kept. `code` < 0 means * any node type; rows scoring under `floor` are left out. */ - gather(cache: ScanCache, q: Float32Array, qnorm: number, code: number, floor: number, lists: number): number { + gather(cache: VectorTable, q: Float32Array, qnorm: number, code: number, floor: number, lists: number): number { const { nlist, dim, centroids } = this; if (this._chosen.length < lists) { this._chosen = new Int32Array(lists); @@ -1554,7 +1683,7 @@ class IvfIndex { } const candRows = this.candRows; const candScores = this.candScores; - const { matrix, norms, codes } = cache; + const { chunks, norms, codes } = cache; const assign = this.assign; let m = 0; @@ -1565,7 +1694,7 @@ class IvfIndex { if (assign[r] < 0) continue; // deleted or updated since the last regroup if (code >= 0 && codes[r] !== code) continue; const denom = norms[r] * qnorm; - const s = denom > 0 ? dot2(matrix, r * dim, q, 0, dim) / denom : 0; + const s = denom > 0 ? dot2(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim, q, 0, dim) / denom : 0; if (s < floor) continue; candRows[m] = r; candScores[m] = s; @@ -1575,7 +1704,7 @@ class IvfIndex { for (const r of this.pending) { if (code >= 0 && codes[r] !== code) continue; const denom = norms[r] * qnorm; - const s = denom > 0 ? dot2(matrix, r * dim, q, 0, dim) / denom : 0; + const s = denom > 0 ? dot2(chunks[r >>> CHUNK_SHIFT], (r & CHUNK_MASK) * dim, q, 0, dim) / denom : 0; if (s < floor) continue; candRows[m] = r; candScores[m] = s; @@ -1660,6 +1789,38 @@ function lowerBound(values: Int32Array, target: number): number { return lo; } +/** + * Dot products of one query with four rows, written to `out[0..3]`. Each sum + * runs over the dimensions in order, exactly as a loop over one row would, so + * the results are bit-identical to four separate loops; only the interleaving + * differs. + */ +function dotRows4( + q: ArrayLike, + a: Float32Array, ao: number, + b: Float32Array, bo: number, + c: Float32Array, co: number, + d: Float32Array, dof: number, + dim: number, + out: Float64Array +): void { + let s0 = 0; + let s1 = 0; + let s2 = 0; + let s3 = 0; + for (let j = 0; j < dim; j++) { + const x = q[j]; + s0 += x * a[ao + j]; + s1 += x * b[bo + j]; + s2 += x * c[co + j]; + s3 += x * d[dof + j]; + } + out[0] = s0; + out[1] = s1; + out[2] = s2; + out[3] = s3; +} + /** * In-memory embedding storage with brute-force cosine similarity search. * @@ -1669,7 +1830,7 @@ function lowerBound(values: Int32Array, target: number): number { * vector-database replacement. */ export class EmbeddingStore { - private _embeddings: Map = new Map(); + private _table: VectorTable | null = null; private _encoder: EmbeddingEncoder | null; private _dimension: number | null = null; private _version: string | null; @@ -1681,7 +1842,6 @@ export class EmbeddingStore { private _annMinSize: number = ANN_MIN_SIZE; private _annSeed = 0; private _annCounts = { approximate: 0, exact: 0, widened: 0, builds: 0 }; - private _scan: ScanCache | null = null; /** Metadata of the most recent imported sidecar (empty before any). */ lastImportMeta: ImportMeta = {}; @@ -1781,22 +1941,27 @@ export class EmbeddingStore { } } - private _cosineSimilarity(v1: number[], v2: number[]): number { - let dot = 0; - let norm1 = 0; - let norm2 = 0; - - for (let i = 0; i < v1.length; i++) { - dot += v1[i] * v2[i]; - norm1 += v1[i] * v1[i]; - norm2 += v2[i] * v2[i]; - } - - norm1 = Math.sqrt(norm1); - norm2 = Math.sqrt(norm2); + /** Euclidean norm, summed in index order (the order the stored norms are summed in). */ + private static _norm(v: ArrayLike): number { + let sum = 0; + for (let i = 0; i < v.length; i++) sum += v[i] * v[i]; + return Math.sqrt(sum); + } - if (norm1 === 0 || norm2 === 0) return 0; - return dot / (norm1 * norm2); + /** + * Cosine of a query against the row at `slot`: float64, the dot product summed + * in dimension order, 0 when either norm is 0. The one definition every exact + * path shares. + */ + private static _rowScore(table: VectorTable, q: ArrayLike, qnorm: number, slot: number): number { + const rn = table.norms[slot]; + if (qnorm === 0 || rn === 0) return 0; + const dim = table.dim; + const chunk = table.chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + let dot = 0; + for (let j = 0; j < dim; j++) dot += q[j] * chunk[base + j]; + return dot / (qnorm * rn); } private _resolveQuery(query: string | number[]): number[] { @@ -1845,8 +2010,13 @@ export class EmbeddingStore { */ models(): Record { const counts = new Map(); - for (const record of this._embeddings.values()) { - counts.set(record.model, (counts.get(record.model) ?? 0) + 1); + const table = this._table; + if (table !== null) { + for (let slot = 0; slot < table.n; slot++) { + if (table.keys[slot] === null) continue; + const model = table.modelNames[table.modelIx[slot]]; + counts.set(model, (counts.get(model) ?? 0) + 1); + } } return Object.fromEntries(counts); } @@ -1906,13 +2076,17 @@ export class EmbeddingStore { const includeUnknown = options.includeUnknown ?? false; const out: NodeRef[] = []; - for (const record of this._embeddings.values()) { + const table = this._table; + if (table === null) return out; + const byDim = dimension !== null && table.dim !== dimension; + for (let slot = 0; slot < table.n; slot++) { + if (table.keys[slot] === null) continue; + const rowModel = table.modelNames[table.modelIx[slot]]; const byModel = model !== null && - record.model !== model && - (includeUnknown || !UNKNOWN_MODELS.has(record.model)); - const byDim = dimension !== null && record.vector.length !== dimension; - if (byModel || byDim) out.push(record.nodeRef); + rowModel !== model && + (includeUnknown || !UNKNOWN_MODELS.has(rowModel)); + if (byModel || byDim) out.push(table.refs[slot]!); } return out; } @@ -1927,14 +2101,18 @@ export class EmbeddingStore { this._checkDimension(vector, "vector"); const key = this._nodeRefToKey(nodeRef); - this._set(key, { id: key, nodeRef, vector, text, model: this._modelName() }); + this._put(key, nodeRef, text, this._modelName(), vector); return key; } - /** Every write goes through here, so the approximate index's cache stays in step. */ - private _set(key: string, record: EmbeddingRecord): void { - this._embeddings.set(key, record); - if (this._scan !== null) this._scan.put(key, record); + /** + * Every write goes through here: the row is copied into the table (the one + * copy of the vector), and an existing approximate index is told about it. + */ + private _put(key: string, ref: NodeRef, text: string, model: string, vector: ArrayLike): void { + let table = this._table; + if (table === null) table = this._table = new VectorTable(this._dimension!); + table.put(key, ref, text, model, vector); } /** Add many embeddings, encoding their texts in one batched call. */ @@ -2034,16 +2212,34 @@ export class EmbeddingStore { return this._checkQuery((await this.encodeTextsAsync([query]))[0]); } + /** + * The stored record of a node, or null. The record is a copy: `vector` is a + * fresh array of the stored float32 values, so changing it changes nothing in + * the store (use {@link add} to replace a vector). + */ get(nodeRef: NodeRef): EmbeddingRecord | null { - const key = this._nodeRefToKey(nodeRef); - return this._embeddings.get(key) || null; + const table = this._table; + if (table === null) return null; + const slot = table.index.get(this._nodeRefToKey(nodeRef)); + if (slot === undefined) return null; + return { + id: table.keys[slot]!, + nodeRef: table.refs[slot]!, + vector: table.vectorAt(slot), + text: table.texts[slot], + model: table.modelNames[table.modelIx[slot]], + }; + } + + /** Whether a node has an embedding (no vector is copied). */ + has(nodeRef: NodeRef): boolean { + return this._table !== null && this._table.index.has(this._nodeRefToKey(nodeRef)); } remove(nodeRef: NodeRef): boolean { - const key = this._nodeRefToKey(nodeRef); - const removed = this._embeddings.delete(key); - if (removed && this._scan !== null) this._scan.drop(key); - return removed; + const table = this._table; + if (table === null) return false; + return table.remove(this._nodeRefToKey(nodeRef)); } /** @@ -2075,26 +2271,112 @@ export class EmbeddingStore { return this._scanExact(queryVector, topK, nodeType, threshold); } - /** The brute-force scan every search used before the approximate index existed. */ + /** + * The brute-force scan every search used before the approximate index existed. + * + * Reads the rows straight from the table, in slot (= arrival) order: float64 + * dot products over the float32 values, summed in dimension order, so a score + * is the same number whatever else is in the store. A bounded `topK` is kept + * in a small sorted buffer (a hit is only allocated as a result if it makes + * the cut); ties keep arrival order, exactly as {@link selectTopK} does. + */ private _scanExact( queryVector: number[], topK: number | null, nodeType: string | undefined, threshold: number ): SimilarityResult[] { - const results: SimilarityResult[] = []; + const table = this._table; + if (table === null || table.live === 0) return []; + if (topK !== null && topK <= 0) return []; + let code = -1; + if (nodeType) { + code = table.codeOf.get(nodeType) ?? -2; + if (code === -2) return []; // no node of that type exists + } + const bounded = topK !== null && Number.isInteger(topK) && topK <= EXACT_TOPK_BUFFER; + const k = bounded ? (topK as number) : 0; + const topRows = new Int32Array(k); + const topScores = new Float64Array(k); + let count = 0; + const all: SimilarityResult[] = []; + + const { dim, chunks, norms, codes, keys, refs, texts } = table; + const qnorm = EmbeddingStore._norm(queryVector); + + const consider = (slot: number, score: number): void => { + if (!(score >= threshold)) return; + if (bounded) { + if (count === k && score <= topScores[k - 1]) return; + let i = count < k ? count++ : k - 1; + while (i > 0 && topScores[i - 1] < score) { + topScores[i] = topScores[i - 1]; + topRows[i] = topRows[i - 1]; + i--; + } + topScores[i] = score; + topRows[i] = slot; + } else { + all.push({ nodeRef: refs[slot]!, score, text: texts[slot] }); + } + }; - for (const record of this._embeddings.values()) { - if (nodeType && record.nodeRef[0] !== nodeType) continue; + // Four rows at a time: four independent running sums hide the latency of the + // adds. Each is still summed in dimension order, so a score is the same + // number the one-row loop would give. Rows are handed to `consider` in slot + // order, so ties break as they always did. + const pend = new Int32Array(4); + const dots = new Float64Array(4); + let np = 0; + const flush = (): void => { + if (np === 4) { + const s0 = pend[0], s1 = pend[1], s2 = pend[2], s3 = pend[3]; + dotRows4( + queryVector, + chunks[s0 >>> CHUNK_SHIFT], (s0 & CHUNK_MASK) * dim, + chunks[s1 >>> CHUNK_SHIFT], (s1 & CHUNK_MASK) * dim, + chunks[s2 >>> CHUNK_SHIFT], (s2 & CHUNK_MASK) * dim, + chunks[s3 >>> CHUNK_SHIFT], (s3 & CHUNK_MASK) * dim, + dim, dots + ); + } else { + for (let i = 0; i < np; i++) { + const slot = pend[i]; + const chunk = chunks[slot >>> CHUNK_SHIFT]; + const base = (slot & CHUNK_MASK) * dim; + let dot = 0; + for (let j = 0; j < dim; j++) dot += queryVector[j] * chunk[base + j]; + dots[i] = dot; + } + } + for (let i = 0; i < np; i++) { + const rn = norms[pend[i]]; + consider(pend[i], rn === 0 ? 0 : dots[i] / (qnorm * rn)); + } + np = 0; + }; - const score = this._cosineSimilarity(queryVector, record.vector); - if (score >= threshold) { - results.push({ nodeRef: record.nodeRef, score, text: record.text }); + for (let slot = 0; slot < table.n; slot++) { + if (keys[slot] === null) continue; + if (code >= 0 && codes[slot] !== code) continue; + if (qnorm === 0) { + consider(slot, 0); + continue; } + pend[np++] = slot; + if (np === 4) flush(); } + if (np > 0) flush(); - if (topK === null) return results.sort((a, b) => b.score - a.score); - return selectTopK(results, topK); + if (bounded) { + const out: SimilarityResult[] = new Array(count); + for (let i = 0; i < count; i++) { + out[i] = { nodeRef: refs[topRows[i]]!, score: topScores[i], text: texts[topRows[i]] }; + } + return out; + } + if (topK === null) return all.sort((a, b) => b.score - a.score); + return selectTopK(all, topK); } /** {@link similaritySearch} with a text query encoded by awaiting the encoder. */ @@ -2135,29 +2417,28 @@ export class EmbeddingStore { * the index is retrained from the current data, with an automatic `nlist` * re-chosen for the new size; * - a cleared store, a dimension change, or a store that shrinks below - * `annMinSize` drops the index with its cache. + * `annMinSize` drops the index. + * + * The index reads the store's own rows; there is no second copy of the vectors. */ - private _annIndex(): ScanCache | null { + private _annIndex(): VectorTable | null { if (this._ann === null || this._dimension === null) return null; - const live = this._embeddings.size; - if (live < Math.max(2, this._annMinSize)) { - this._scan = null; + const table = this._table; + if (table === null || table.live < Math.max(2, this._annMinSize)) { + if (table !== null) table.ivf = null; return null; } - let scan = this._scan; - if (scan === null) { - scan = this._scan = ScanCache.build(this._embeddings.values(), live, this._dimension); - } - let ivf = scan.ivf; - if (ivf !== null && ivf.churn > ANN_RETRAIN_FRACTION * ivf.trainedN) ivf = scan.ivf = null; + const live = table.live; + let ivf = table.ivf; + if (ivf !== null && ivf.churn > ANN_RETRAIN_FRACTION * ivf.trainedN) ivf = table.ivf = null; if (ivf === null) { const nlist = this._annNlist ?? EmbeddingStore._annAutoNlist(live); - scan.ivf = IvfIndex.build(scan, nlist, this._annSeed); + table.ivf = IvfIndex.build(table, nlist, this._annSeed); this._annCounts.builds++; } else if (ivf.pending.size >= Math.max(ANN_FOLD_MIN, ANN_FOLD_FRACTION * ivf.trainedN)) { - ivf.fold(scan); + ivf.fold(table); } - return scan; + return table; } private _annProbe(ivf: IvfIndex): number { @@ -2178,8 +2459,8 @@ export class EmbeddingStore { return lists * 2 >= ivf.nlist ? null : lists; } - /** Load the query into the cache's float32 buffer; null when it has no direction (all zeros). */ - private _annQuery(scan: ScanCache, queryVector: number[], nodeType: string | undefined): { qnorm: number; code: number } | null { + /** Load the query into the table's float32 buffer; null when it has no direction (all zeros). */ + private _annQuery(scan: VectorTable, queryVector: number[], nodeType: string | undefined): { qnorm: number; code: number } | null { const q = scan.query; let sum = 0; for (let j = 0; j < scan.dim; j++) { @@ -2223,8 +2504,8 @@ export class EmbeddingStore { const best = ivf.best(m, topK); const results: SimilarityResult[] = []; for (let i = 0; i < best.length; i++) { - const record = this._embeddings.get(scan.keys[ivf.candRows[best[i]]]!)!; - results.push({ nodeRef: record.nodeRef, score: ivf.candScores[best[i]], text: record.text }); + const row = ivf.candRows[best[i]]; + results.push({ nodeRef: scan.refs[row]!, score: ivf.candScores[best[i]], text: scan.texts[row] }); } this._annCounts.approximate++; if (scale > 1) this._annCounts.widened++; @@ -2239,10 +2520,13 @@ export class EmbeddingStore { */ annPolish(results: SimilarityResult[], queryVector: number[], threshold: number): SimilarityResult[] { const out: SimilarityResult[] = []; + const table = this._table; + if (table === null) return out; + const qnorm = EmbeddingStore._norm(queryVector); for (const result of results) { - const record = this._embeddings.get(this._nodeRefToKey(result.nodeRef)); - if (!record) continue; - const score = this._cosineSimilarity(queryVector, record.vector); + const slot = table.index.get(this._nodeRefToKey(result.nodeRef)); + if (slot === undefined) continue; + const score = EmbeddingStore._rowScore(table, queryVector, qnorm, slot); if (score >= threshold) out.push({ nodeRef: result.nodeRef, score, text: result.text }); } return out.sort((a, b) => b.score - a.score); @@ -2277,12 +2561,12 @@ export class EmbeddingStore { const slots = new Int32Array(m); for (let i = 0; i < m; i++) slots[i] = i; slots.sort((a, b) => scores[b] - scores[a] || rows[a] - rows[b]); - const keys = scan.keys; - const embeddings = this._embeddings; + const refs = scan.refs; + const texts = scan.texts; const ranked = (function* (): Generator { for (let i = 0; i < m; i++) { - const record = embeddings.get(keys[rows[slots[i]]]!)!; - yield { nodeRef: record.nodeRef, score: scores[slots[i]], text: record.text }; + const row = rows[slots[i]]; + yield { nodeRef: refs[row]!, score: scores[slots[i]], text: texts[row] }; } })(); return { ranked, approximate: true }; @@ -2302,7 +2586,7 @@ export class EmbeddingStore { if (this._ann === null) { throw new EmbeddingError("this store has no approximate index; create it with ann: 'ivf'"); } - if (force && this._scan !== null) this._scan.ivf = null; + if (force && this._table !== null) this._table.ivf = null; const scan = this._annIndex(); if (scan !== null && scan.ivf !== null && scan.ivf.pending.size > 0) scan.ivf.fold(scan); return this.annStats(); @@ -2310,7 +2594,7 @@ export class EmbeddingStore { /** What the approximate index looks like right now. Never builds anything. */ annStats(): AnnStats { - const scan = this._scan; + const scan = this._table; const ivf = scan === null ? null : scan.ivf; return { enabled: this._ann !== null, @@ -2323,7 +2607,8 @@ export class EmbeddingStore { churn: ivf === null ? 0 : ivf.churn, buildSeconds: ivf === null ? null : ivf.buildSeconds, indexBytes: ivf === null ? 0 : ivf.bytes(), - cacheBytes: scan === null ? 0 : scan.bytes(), + cacheBytes: 0, + vectorBytes: scan === null ? 0 : scan.bytes(), minSize: this._annMinSize, searches: { ...this._annCounts }, }; @@ -2402,7 +2687,7 @@ export class EmbeddingStore { const mid = ordered.length >> 1; return ordered.length % 2 ? ordered[mid] : 0.5 * (ordered[mid - 1] + ordered[mid]); }; - const ivf = this._scan === null ? null : this._scan.ivf; + const ivf = this._table === null ? null : this._table.ivf; return { recall: recalls.length ? recalls.reduce((a, b) => a + b, 0) / recalls.length : 1, minRecall: recalls.length ? Math.min(...recalls) : 1, @@ -2425,15 +2710,25 @@ export class EmbeddingStore { scoreMap(query: string | number[], nodeType?: string): Map { const queryVector = this._resolveQuery(query); const scores = new Map(); - for (const [key, record] of this._embeddings) { - if (nodeType && record.nodeRef[0] !== nodeType) continue; - scores.set(key, this._cosineSimilarity(queryVector, record.vector)); + const table = this._table; + if (table === null) return scores; + let code = -1; + if (nodeType) { + code = table.codeOf.get(nodeType) ?? -2; + if (code === -2) return scores; + } + const qnorm = EmbeddingStore._norm(queryVector); + for (let slot = 0; slot < table.n; slot++) { + const key = table.keys[slot]; + if (key === null) continue; + if (code >= 0 && table.codes[slot] !== code) continue; + scores.set(key, EmbeddingStore._rowScore(table, queryVector, qnorm, slot)); } return scores; } count(): number { - return this._embeddings.size; + return this._table === null ? 0 : this._table.live; } /** @@ -2445,8 +2740,7 @@ export class EmbeddingStore { * wrong length even when empty. */ clear(resetDimension: boolean = false): void { - this._embeddings.clear(); - this._scan = null; + this._table = null; if (resetDimension) this._dimension = null; } @@ -2460,14 +2754,17 @@ export class EmbeddingStore { */ exportEmbeddings(graphDigest?: string | null): ExportedEmbeddings { const embeddings: ExportedEmbedding[] = []; - for (const record of this._embeddings.values()) { + const table = this._table; + for (let slot = 0; table !== null && slot < table.n; slot++) { + const ref = table.refs[slot]; + if (ref === null) continue; embeddings.push({ - type: String(record.nodeRef[0]), - id: record.nodeRef[1], - id_type: typeof record.nodeRef[1] === 'number' ? 'int' : 'str', - text: record.text, - model: record.model, - vector: record.vector, + type: String(ref[0]), + id: ref[1], + id_type: typeof ref[1] === 'number' ? 'int' : 'str', + text: table.texts[slot], + model: table.modelNames[table.modelIx[slot]], + vector: table.vectorAt(slot), }); } const payload: ExportedEmbeddings = { @@ -2519,17 +2816,31 @@ export class EmbeddingStore { const ref = [item.type, id] as NodeRef; this._checkDimension(item.vector, `embedding for :${item.type}:${id}`); const key = this._nodeRefToKey(ref); - this._set(key, { - id: key, - nodeRef: ref, - vector: item.vector, - text: item.text ?? '', - model: item.model ?? 'unknown', - }); + this._put(key, ref, item.text ?? '', item.model ?? 'unknown', item.vector); written++; } return written; } + + /** + * The table, the encoder's model and the version, for the snapshot writer + * (see `snapshot.ts`). Low-level: not part of the stable API. + */ + snapshotState(): { table: VectorTable | null; encoderModel: string | null; encoderVersion: string | null } { + return { table: this._table, encoderModel: this._currentModel(), encoderVersion: this.version }; + } + + /** + * Replace every embedding with the rows of a verified, fully built table + * (see `snapshot.ts`), as `importEmbeddings(payload, true)` would, and record + * the snapshot's metadata as the last import's. Low-level: not part of the + * stable API. Any approximate index is rebuilt lazily. + */ + adoptTable(table: VectorTable, meta: ImportMeta): void { + this._table = table.live === 0 ? null : table; + this._dimension = table.live === 0 ? null : table.dim; + this.lastImportMeta = meta; + } } // ============================================================================= @@ -3783,7 +4094,7 @@ export class SemanticGraph extends ISONGraph { const batch: Array<[NodeRef, string]> = []; for (const node of this.nodes()) { - if (skipExisting && this._embeddingStore.get(node.ref)) continue; + if (skipExisting && this._embeddingStore.has(node.ref)) continue; const text = fn(node); if (text) batch.push([node.ref, text]); } @@ -3800,7 +4111,7 @@ export class SemanticGraph extends ISONGraph { this.embedTextFor(node.properties) || `${node.type}:${node.id}`); const batch: Array<[NodeRef, string]> = []; for (const node of this.nodes()) { - if (skipExisting && this._embeddingStore.get(node.ref)) continue; + if (skipExisting && this._embeddingStore.has(node.ref)) continue; const text = fn(node); if (text) batch.push([node.ref, text]); } diff --git a/isongraph-vector-ts/src/node.ts b/isongraph-vector-ts/src/node.ts index f803229..46e662b 100644 --- a/isongraph-vector-ts/src/node.ts +++ b/isongraph-vector-ts/src/node.ts @@ -31,8 +31,11 @@ import { closeSync, renameSync, unlinkSync, + fstatSync, + readSync, + writeSync, } from 'node:fs'; -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { basename, dirname, extname, join } from 'node:path'; import { ISONGraph } from 'ison-graph-ts'; import { @@ -46,6 +49,20 @@ import { type ExportedEmbeddings, type SemanticGraphOptions, } from './index.js'; +import { + SNAPSHOT_SUFFIX, + SnapshotError, + writeSnapshot, + readSnapshot, + applySnapshot, + findUnrestorableId, + type Sha256Like, + type SidecarId, + type SnapshotIdentity, + type SnapshotInfo, + type SnapshotReadOptions, + type SnapshotWriteOptions, +} from './snapshot.js'; export { EMBEDDING_SIDECAR_SUFFIX }; @@ -128,11 +145,191 @@ export function loadEmbeddings( return store.importEmbeddings(payload, replace); } +// ============================================================================= +// Binary snapshot (optional, beside the JSON sidecar) +// ============================================================================= + +export { SNAPSHOT_SUFFIX, SnapshotError }; +export type { SidecarId, SnapshotIdentity, SnapshotInfo, SnapshotReadOptions, SnapshotWriteOptions }; + +/** `.embeddings.snap`. */ +export function snapshotPathFor(path: string): string { + return path + SNAPSHOT_SUFFIX; +} + +/** SHA-256 over `node:crypto`, in the shape the browser-safe snapshot module asks for. */ +function sha256(): Sha256Like { + const hash = createHash('sha256'); + return { + update: bytes => { + hash.update(bytes); + }, + digest: () => new Uint8Array(hash.digest()), + }; +} + +/** Bytes sampled from the head and tail of a sidecar, and per probe (see DESIGN.md 11.1). */ +const FP_EDGE = 64 * 1024; +const FP_PROBE = 4 * 1024; +const FP_PROBES = 64; + +/** + * Identity of a sidecar file: its length and a SHA-256 over the length (u64 LE), + * its first and last 64 KiB and 64 probes of 4 KiB at `floor(len * i / 65)`. + * Any serializer rewriting the sidecar changes it; an in-place edit that keeps + * the length and touches no sampled byte does not. + */ +export function sidecarIdentity(path: string): SidecarId { + const fd = openSync(path, 'r'); + try { + const len = fstatSync(fd).size; + const hash = createHash('sha256'); + const head = Buffer.alloc(8); + head.writeBigUInt64LE(BigInt(len)); + hash.update(head); + const take = (at: number, want: number): void => { + const n = Math.min(want, Math.max(0, len - at)); + const buf = Buffer.alloc(n); + let got = 0; + while (got < n) { + const read = readSync(fd, buf, got, n - got, at + got); + if (read === 0) throw new EmbeddingError(`${path} changed while its identity was being read`); + got += read; + } + hash.update(buf); + }; + take(0, FP_EDGE); + take(Math.max(0, len - FP_EDGE), FP_EDGE); + for (let i = 1; i <= FP_PROBES; i++) take(Number((BigInt(len) * BigInt(i)) / BigInt(FP_PROBES + 1)), FP_PROBE); + return { len, fingerprint: new Uint8Array(hash.digest()) }; + } finally { + closeSync(fd); + } +} + +/** + * Write whatever `fill` streams to `write` into `path`, so a reader sees the old + * file or the new one: a temp file in the same directory, flushed, renamed over + * `path`. A failure removes the temp file and leaves `path` as it was. + */ +function writeStreamAtomic(path: string, fill: (write: (bytes: Uint8Array) => void) => T): T { + const tmp = join( + dirname(path), + `.${basename(path)}.${process.pid}.${randomBytes(4).toString('hex')}.tmp` + ); + let fd: number | undefined; + try { + fd = openSync(tmp, 'w'); + const handle = fd; + const value = fill(bytes => { + let at = 0; + while (at < bytes.length) at += writeSync(handle, bytes, at, bytes.length - at); + }); + try { + fsyncSync(fd); + } catch { + // fsync unsupported on this filesystem; the rename still gives atomicity. + } + closeSync(fd); + fd = undefined; + renameSync(tmp, path); + return value; + } catch (error) { + if (fd !== undefined) { + try { closeSync(fd); } catch { /* already failing */ } + } + try { unlinkSync(tmp); } catch { /* temp file never created */ } + throw error; + } +} + +/** + * Write a binary snapshot of every embedding in `store` to `path`, atomically (a + * temp file, flushed, renamed over `path`: a failure or crash leaves the + * previous file untouched). `identity` ties it to the graph and sidecar it was + * saved with; a load that finds a different identity refuses it. Prefer + * `saveGraph(graph, path, { snapshot: true })`, which does all of that. + */ +export function saveSnapshot( + store: EmbeddingStore, + path: string, + identity: SnapshotIdentity = {}, + options: SnapshotWriteOptions = {} +): SnapshotInfo { + return writeStreamAtomic(path, write => writeSnapshot(store, identity, sha256(), write, options)); +} + +function readSnapshotFile(path: string, options: SnapshotReadOptions): ReturnType { + const fd = openSync(path, 'r'); + try { + const size = fstatSync(fd).size; + return readSnapshot(into => readSync(fd, into, 0, into.length, null), size, sha256(), options); + } finally { + closeSync(fd); + } +} + +/** + * Replace the store's embeddings with those of a snapshot file. Nothing changes + * unless the whole file verifies (structure and checksum); throws SnapshotError + * otherwise. Does not check identity (pass `accept` to), and ids come back as + * strings unless `resolveId` says otherwise: use `loadGraph`, which does both. + */ +export function loadSnapshot( + store: EmbeddingStore, + path: string, + options: SnapshotReadOptions = {} +): SnapshotInfo { + const result = readSnapshotFile(path, options); + applySnapshot(store, result); + return result.info; +} + +/** + * How a snapshot's id strings map back to node ids, given the graph they were + * saved with: an id that names a node of that type as a string stays a string; + * a canonical integer string that names a node as a number becomes the number; + * anything else stays a string. (The snapshot format carries no id type; the + * JSON sidecar's `id_type` is what this stands in for.) + */ +function idResolverFor(graph: ISONGraph): (type: string, id: string) => string | number { + return (type, id) => { + if (graph.hasNode(type, id)) return id; + const n = Number(id); + if (id !== '' && String(n) === id && graph.hasNode(type, n)) return n; + return id; + }; +} + +function removeQuietly(path: string): void { + try { + unlinkSync(path); + } catch { + // no such file, or it cannot be removed: either way there is nothing to keep stale + } +} + +// ============================================================================= +// Graph files +// ============================================================================= + export interface SaveGraphOptions { /** 'ison', 'isonl', or 'auto' (by extension: .isonl is ISONL, else ISON). */ format?: GraphFileFormat; /** Write the embeddings sidecar (set false for graph-only saves). Default true. */ embeddings?: boolean; + /** + * Also write a binary snapshot of the embeddings at `.embeddings.snap`, + * after and in addition to the JSON sidecar (which stays the portable copy). + * `loadGraph` uses it, when it provably belongs to the graph and sidecar on + * disk, instead of parsing the JSON. Default false. A save without it deletes + * an existing snapshot, which the new sidecar would make stale. If the + * snapshot cannot be written the graph and sidecar are saved anyway, a stale + * snapshot is removed, and the error is thrown (a SnapshotError when the store + * holds something the snapshot cannot carry, such as an integer id for a node + * the graph does not have). + */ + snapshot?: boolean; } /** @@ -164,7 +361,24 @@ export function saveGraph( const stem = basename(path, extname(path)); const reread = format === 'isonl' ? ISONGraph.fromIsonl(text, stem) : ISONGraph.fromIson(text, stem); - saveEmbeddings(graph.embeddingStore, embeddingsPathFor(path), graphDigest(reread)); + const digest = graphDigest(reread); + const sidecar = embeddingsPathFor(path); + const snap = snapshotPathFor(path); + if (!options.snapshot) removeQuietly(snap); // the sidecar is about to change: a snapshot of the old one is stale + saveEmbeddings(graph.embeddingStore, sidecar, digest); + if (options.snapshot) { + try { + const problem = findUnrestorableId(graph.embeddingStore, idResolverFor(reread)); + if (problem !== null) throw new SnapshotError(`not written: ${problem}`); + saveSnapshot(graph.embeddingStore, snap, { + graphDigest: digest, + sidecar: sidecarIdentity(sidecar), + }); + } catch (error) { + removeQuietly(snap); + throw error; + } + } } } @@ -185,6 +399,21 @@ export interface LoadGraphOptions extends SemanticGraphOptions { onMismatch?: GraphMismatchMode; } +/** Where a load's embeddings came from (see {@link loadGraphWithReport}). */ +export interface LoadReport { + /** 'snapshot' (the binary file), 'json' (the sidecar) or 'none' (embeddings off, or no sidecar). */ + source: 'snapshot' | 'json' | 'none'; + /** + * Why a snapshot that exists was not used (and the JSON sidecar was read + * instead); null when it was used or there is none. + */ + snapshotSkipped: string | null; + /** Embeddings restored. */ + embeddings: number; + /** Wall time of the whole load. */ + seconds: number; +} + /** * Number literals this port reads differently from Python, so a digest taken * here can differ from one taken there for an unedited file: integral floats @@ -194,6 +423,22 @@ function hasInexactNumberLiteral(text: string): boolean { return /(?:^|[\s|])[-+]?(?:\d+\.0+|\d{16,})(?=$|[\s|])/m.test(text); } +/** Warn when the sidecar's (or snapshot's) encoder version is not the one in use. */ +function warnOnVersionSkew(graph: SemanticGraph): void { + const storedVersion = graph.embeddingStore.lastImportMeta.encoder_version; + const currentVersion = graph.embeddingStore.version; + if ( + storedVersion !== undefined && + currentVersion !== null && + String(storedVersion) !== String(currentVersion) + ) { + console.warn( + `embeddings were made by encoder version '${storedVersion}', ` + + `but the encoder in use is version '${currentVersion}'` + ); + } +} + /** * Load a graph from a file, restoring its embeddings from the sidecar. * @@ -210,8 +455,24 @@ function hasInexactNumberLiteral(text: string): boolean { * mismatch is reported as a warning whatever `onMismatch` says, since it may * be a false positive. If both the encoder's version and the sidecar's * `encoder_version` are set and differ, a warning is issued. + * + * A binary snapshot beside the sidecar (`saveGraph` with `snapshot: true`, or + * the Rust crate's `save_with_snapshot`) is used instead of the JSON when its + * header names this graph's digest and the sidecar file as it is now and its + * checksum verifies; the result is the same store the JSON gives. Any other + * outcome falls back to the JSON, exactly as without a snapshot. Use + * {@link loadGraphWithReport} to see which one was used. */ export function loadGraph(path: string, options: LoadGraphOptions = {}): SemanticGraph { + return loadGraphWithReport(path, options).graph; +} + +/** {@link loadGraph}, also saying where the embeddings came from. */ +export function loadGraphWithReport( + path: string, + options: LoadGraphOptions = {} +): { graph: SemanticGraph; report: LoadReport } { + const started = typeof performance !== 'undefined' ? performance.now() : Date.now(); const { format: fmt, embeddings, onMismatch = 'error', ...graphOptions } = options; if (onMismatch !== 'error' && onMismatch !== 'warn' && onMismatch !== 'reembed') { throw new EmbeddingError( @@ -228,64 +489,102 @@ export function loadGraph(path: string, options: LoadGraphOptions = {}): Semanti format === 'isonl' ? ISONGraph.fromIsonl(text, name) : ISONGraph.fromIson(text, name); const graph = SemanticGraph.fromISONGraph(base, { ...graphOptions, name }); - if (embeddings ?? true) { - const sidecar = embeddingsPathFor(path); - if (existsSync(sidecar) && statSync(sidecar).isFile()) { - let payload: ExportedEmbeddings; - try { - payload = JSON.parse(readFileSync(sidecar, 'utf8')) as ExportedEmbeddings; - } catch (error) { - throw new EmbeddingError( - `unreadable embeddings sidecar ${sidecar}: ${(error as Error).message}` - ); - } + const report = ( + source: LoadReport['source'], + snapshotSkipped: string | null, + count: number + ): { graph: SemanticGraph; report: LoadReport } => ({ + graph, + report: { + source, + snapshotSkipped, + embeddings: count, + seconds: ((typeof performance !== 'undefined' ? performance.now() : Date.now()) - started) / 1000, + }, + }); - let problem: string | null = null; - let unverifiable = false; - const recorded = payload && typeof payload === 'object' ? payload.graph_digest : undefined; - if ( - recorded !== undefined && - recorded !== null && - (payload.graph_digest_basis ?? GRAPH_DIGEST_BASIS) === GRAPH_DIGEST_BASIS - ) { - const actual = graphDigest(base); - if (recorded !== actual) { - unverifiable = hasInexactNumberLiteral(text); - problem = - `embeddings sidecar ${sidecar} does not belong to graph ${path}: it was written ` + - `for graph digest ${recorded}, but the graph reads as ${actual} (the sidecar is ` + - `from a different save, or the graph was edited since). ` + - (unverifiable - ? `The graph file holds number literals (integral floats such as 1.0, or ` + - `integers beyond 2^53) that this port reads differently from Python, so the ` + - `digests may differ for an unedited file; using the sidecar's vectors anyway.` - : `Pass onMismatch: 'reembed' to rebuild the vectors or onMismatch: 'warn' to ` + - `use them anyway.`); - } - } + if (!(embeddings ?? true)) return report('none', null, 0); + const sidecar = embeddingsPathFor(path); + if (!(existsSync(sidecar) && statSync(sidecar).isFile())) return report('none', null, 0); - if (problem !== null && onMismatch === 'error' && !unverifiable) { - throw new GraphEmbeddingMismatchError(problem); - } - if (problem !== null && onMismatch === 'reembed' && !unverifiable) { - graph.embedAllNodes(); - } else { - if (problem !== null) console.warn(problem); - graph.embeddingStore.importEmbeddings(payload); - const storedVersion = graph.embeddingStore.lastImportMeta.encoder_version; - const currentVersion = graph.embeddingStore.version; - if ( - storedVersion !== undefined && - currentVersion !== null && - String(storedVersion) !== String(currentVersion) - ) { - console.warn( - `embeddings were made by encoder version '${storedVersion}', ` + - `but the encoder in use is version '${currentVersion}'` - ); - } - } + let actualDigest: string | null = null; + const digestNow = (): string => (actualDigest ??= graphDigest(base)); + + // The snapshot first: a graph digest equal to this graph's own (no leniency + // for number literals read differently from Python), the sidecar as it is + // now, a valid body and checksum. Anything else and the JSON is read. + let snapshotSkipped: string | null = null; + const snap = snapshotPathFor(path); + if (existsSync(snap) && statSync(snap).isFile()) { + try { + const current = sidecarIdentity(sidecar); + const digest = digestNow(); + const result = readSnapshotFile(snap, { + resolveId: idResolverFor(base), + accept: info => { + if (info.graphDigest !== digest || info.graphDigestBasis !== GRAPH_DIGEST_BASIS) { + return 'it was saved for a different graph (digest differs)'; + } + if (info.sidecar === null) return 'it records no sidecar identity'; + if ( + info.sidecar.len !== current.len || + !info.sidecar.fingerprint.every((b, i) => b === current.fingerprint[i]) + ) { + return 'the JSON sidecar changed since the snapshot was written (stale snapshot)'; + } + return null; + }, + }); + applySnapshot(graph.embeddingStore, result); + warnOnVersionSkew(graph); + return report('snapshot', null, graph.embeddingStore.count()); + } catch (error) { + snapshotSkipped = error instanceof SnapshotError ? error.reason : (error as Error).message; } } - return graph; + + let payload: ExportedEmbeddings; + try { + payload = JSON.parse(readFileSync(sidecar, 'utf8')) as ExportedEmbeddings; + } catch (error) { + throw new EmbeddingError( + `unreadable embeddings sidecar ${sidecar}: ${(error as Error).message}` + ); + } + + let problem: string | null = null; + let unverifiable = false; + const recorded = payload && typeof payload === 'object' ? payload.graph_digest : undefined; + if ( + recorded !== undefined && + recorded !== null && + (payload.graph_digest_basis ?? GRAPH_DIGEST_BASIS) === GRAPH_DIGEST_BASIS + ) { + const actual = digestNow(); + if (recorded !== actual) { + unverifiable = hasInexactNumberLiteral(text); + problem = + `embeddings sidecar ${sidecar} does not belong to graph ${path}: it was written ` + + `for graph digest ${recorded}, but the graph reads as ${actual} (the sidecar is ` + + `from a different save, or the graph was edited since). ` + + (unverifiable + ? `The graph file holds number literals (integral floats such as 1.0, or ` + + `integers beyond 2^53) that this port reads differently from Python, so the ` + + `digests may differ for an unedited file; using the sidecar's vectors anyway.` + : `Pass onMismatch: 'reembed' to rebuild the vectors or onMismatch: 'warn' to ` + + `use them anyway.`); + } + } + + if (problem !== null && onMismatch === 'error' && !unverifiable) { + throw new GraphEmbeddingMismatchError(problem); + } + if (problem !== null && onMismatch === 'reembed' && !unverifiable) { + graph.embedAllNodes(); + } else { + if (problem !== null) console.warn(problem); + graph.embeddingStore.importEmbeddings(payload); + warnOnVersionSkew(graph); + } + return report('json', snapshotSkipped, graph.embeddingStore.count()); } diff --git a/isongraph-vector-ts/src/snapshot.test.ts b/isongraph-vector-ts/src/snapshot.test.ts new file mode 100644 index 0000000..88f9fa8 --- /dev/null +++ b/isongraph-vector-ts/src/snapshot.test.ts @@ -0,0 +1,594 @@ +/** + * The optional binary snapshot (DESIGN.md section 11 of the Rust crate): the + * golden files the Rust crate wrote, a save -> load round trip of our own, + * every rejection rule with its fall-back to the JSON sidecar, the save + * protocol, and the single-copy table against a plain reference model. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import { + mkdtempSync, rmSync, readFileSync, writeFileSync, copyFileSync, existsSync, readdirSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { + EmbeddingStore, SemanticGraph, MockEncoder, graphDigest, selectTopK, + type NodeRef, type SimilarityResult, +} from './index.js'; +import { + SnapshotError, snapshotToBytes, readSnapshotBytes, applySnapshot, writeSnapshot, + type Sha256Like, +} from './snapshot.js'; +import { + saveGraph, loadGraph, loadGraphWithReport, loadSnapshot, saveSnapshot, snapshotPathFor, + sidecarIdentity, embeddingsPathFor, +} from './node.js'; + +const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'tests', 'fixtures', 'snapshot'); + +const dirs: string[] = []; +function tmp(): string { + const dir = mkdtempSync(join(tmpdir(), 'igv-snap-')); + dirs.push(dir); + return dir; +} +afterEach(() => { + while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }); +}); + +function sha(): Sha256Like { + const h = createHash('sha256'); + return { update: b => { h.update(b); }, digest: () => new Uint8Array(h.digest()) }; +} + +/** Recompute the trailer so that only the structural rule under test can reject the file. */ +function reseal(bytes: Uint8Array): Uint8Array { + const body = bytes.subarray(0, bytes.length - 32); + const out = new Uint8Array(bytes.length); + out.set(body); + out.set(createHash('sha256').update(body).digest(), body.length); + return out; +} + +function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** Everything observable about a store's contents: order, keys, ids, texts, models and the vector bits. */ +function dump(store: EmbeddingStore): unknown[] { + return store.exportEmbeddings().embeddings.map(e => [ + e.type, e.id, e.id_type, e.text, e.model, + Array.from(new Uint32Array(new Float32Array(e.vector).buffer)), + ]); +} + +/** A graph with numeric and string ids, two models, an update, a delete and a duplicate vector. */ +function sampleGraph(): SemanticGraph { + const graph = new SemanticGraph({ name: 'snap', encoder: new MockEncoder(8, 'v1'), autoEmbed: false }); + const next = rng(11); + const vec = (): number[] => Array.from({ length: 8 }, () => Math.fround(next() * 2 - 1)); + for (let i = 1; i <= 30; i++) { + graph.addNode('person', i, { name: `Person ${i}` }); + if (i === 12) graph.embeddingStore.add(['person', i], 'Person 12'); // from the encoder + else graph.embeddingStore.add(['person', i], `Person ${i} \u{1F600} "quoted"\nnewline`, vec()); + } + graph.embeddingStore.add(['doc', 'a:b'], 'id with a colon', vec()); // an embedding without a graph node (an id may hold a colon here) + graph.embeddingStore.add(['person', 3], 'updated', vec()); // update in place + graph.embeddingStore.remove(['person', 7]); // tombstone + const dup = vec(); + graph.embeddingStore.add(['doc', 'x'], 'dup 1', dup); // an exact tie + graph.addNode('doc', 'x', { name: 'x' }); + graph.embeddingStore.add(['doc', 'y'], 'dup 2', dup); + graph.addNode('doc', 'y', { name: 'y' }); + graph.embeddingStore.add(['doc', 'z'], '', new Array(8).fill(0)); // a zero vector + graph.addNode('doc', 'z', { name: 'z' }); + return graph; +} + +function sameAnswers(a: SemanticGraph, b: SemanticGraph, dim: number, seed = 5): void { + const next = rng(seed); + for (let i = 0; i < 12; i++) { + const q = Array.from({ length: dim }, () => next() * 2 - 1); + for (const [k, type, th] of [[10, undefined, -1], [3, 'doc', 0], [null, undefined, 0.2]] as const) { + expect(JSON.stringify(a.similaritySearch(q, k, type, th))).toBe(JSON.stringify(b.similaritySearch(q, k, type, th))); + } + } +} + +describe('snapshot written by the Rust crate (golden fixture)', () => { + function copyFixture(dir: string, withSnap: boolean): string { + const path = join(dir, 'golden.ison'); + for (const name of ['golden.ison', 'golden.ison.embeddings.json', ...(withSnap ? ['golden.ison.embeddings.snap'] : [])]) { + copyFileSync(join(FIXTURES, name), join(dir, name)); + } + return path; + } + + it('is read through the snapshot, and equals a JSON-only load of the same graph and sidecar', () => { + const withSnap = loadGraphWithReport(copyFixture(tmp(), true)); + const jsonOnly = loadGraphWithReport(copyFixture(tmp(), false)); + + expect(withSnap.report.source).toBe('snapshot'); + expect(withSnap.report.snapshotSkipped).toBeNull(); + expect(withSnap.report.embeddings).toBe(36); + expect(jsonOnly.report.source).toBe('json'); + + const a = withSnap.graph.embeddingStore; + const b = jsonOnly.graph.embeddingStore; + expect(a.count()).toBe(36); + expect(a.dimension).toBe(12); + expect(dump(a)).toEqual(dump(b)); // keys, texts, models, vector bits, order + expect(a.models()).toEqual(b.models()); + expect(a.lastImportMeta).toEqual(b.lastImportMeta); + expect(a.lastImportMeta.encoder_model).toBe('second-model'); + sameAnswers(withSnap.graph, jsonOnly.graph, 12); + }); + + it('is reproduced byte for byte by our writer from the loaded store', () => { + const dir = tmp(); + const path = copyFixture(dir, true); + const encoder = new MockEncoder(12); + encoder.modelName = 'second-model'; // the header records the encoder's model + const { graph } = loadGraphWithReport(path, { encoder }); + const meta = graph.embeddingStore.lastImportMeta; + const out = join(dir, 'again.snap'); + saveSnapshot(graph.embeddingStore, out, { + graphDigest: meta.graph_digest, + sidecar: sidecarIdentity(embeddingsPathFor(path)), + }); + expect(readFileSync(out)).toEqual(readFileSync(snapshotPathFor(path))); + }); + + it('keeps unicode, colons, empty and multi-line texts, a zero vector and a tie', () => { + const { graph } = loadGraphWithReport(copyFixture(tmp(), true)); + const store = graph.embeddingStore; + const rows = dump(store) as Array<[string, string, string, string, string, number[]]>; + expect(rows.some(r => r[3].includes('\u{1F600}'))).toBe(true); + expect(rows.some(r => r[1].includes('id:with:colons'))).toBe(true); + expect(rows.some(r => r[3] === '')).toBe(true); + expect(rows.some(r => r[3].includes('\n'))).toBe(true); + expect(rows.some(r => r[5].every(bits => bits === 0))).toBe(true); + expect(new Set(rows.map(r => r[4])).size).toBe(2); + }); + + it('falls back to the JSON, untouched, when a copy of the snapshot is corrupt or truncated', () => { + const jsonOnly = loadGraphWithReport(copyFixture(tmp(), false)).graph.embeddingStore; + const original = new Uint8Array(readFileSync(join(FIXTURES, 'golden.ison.embeddings.snap'))); + const damaged: Array<[string, Uint8Array]> = [ + ['a flipped float byte', (() => { const b = original.slice(); b[b.length - 100] ^= 0x40; return b; })()], + ['a flipped trailer byte', (() => { const b = original.slice(); b[b.length - 1] ^= 1; return b; })()], + ['half of the file', original.slice(0, original.length >> 1)], + ['one byte short', original.slice(0, original.length - 1)], + ['only the magic', original.slice(0, 8)], + ['an empty file', new Uint8Array(0)], + ]; + for (const [what, bytes] of damaged) { + const dir = tmp(); + const path = copyFixture(dir, false); + writeFileSync(snapshotPathFor(path), bytes); + const { graph, report } = loadGraphWithReport(path); + expect(report.source, what).toBe('json'); + expect(report.snapshotSkipped, what).not.toBeNull(); + expect(dump(graph.embeddingStore), what).toEqual(dump(jsonOnly)); + } + }); +}); + +describe('snapshot round trip of our own save', () => { + it('saveGraph({ snapshot: true }) then loadGraph reads the snapshot and gives the same store', () => { + const dir = tmp(); + const path = join(dir, 'g.ison'); + const graph = sampleGraph(); + saveGraph(graph, path, { snapshot: true }); + expect(existsSync(snapshotPathFor(path))).toBe(true); + + const viaSnap = loadGraphWithReport(path, { encoder: new MockEncoder(8, 'v1') }); + expect(viaSnap.report.source).toBe('snapshot'); + expect(viaSnap.report.embeddings).toBe(graph.embeddingStore.count()); + + const noSnap = tmp(); + copyFileSync(path, join(noSnap, 'g.ison')); + copyFileSync(embeddingsPathFor(path), join(noSnap, 'g.ison.embeddings.json')); + const viaJson = loadGraphWithReport(join(noSnap, 'g.ison'), { encoder: new MockEncoder(8, 'v1') }); + expect(viaJson.report.source).toBe('json'); + + expect(dump(viaSnap.graph.embeddingStore)).toEqual(dump(viaJson.graph.embeddingStore)); + expect(dump(viaSnap.graph.embeddingStore)).toEqual(dump(graph.embeddingStore)); // numeric ids stayed numbers + expect(viaSnap.graph.getEmbedding(['person', 3])!.nodeRef).toEqual(['person', 3]); + expect(typeof viaSnap.graph.getEmbedding(['person', 3])!.nodeRef[1]).toBe('number'); + expect(viaSnap.graph.getEmbedding(['person', 7])).toBeNull(); + sameAnswers(viaSnap.graph, viaJson.graph, 8); + // text search works against the restored store too + expect(viaSnap.graph.similaritySearch('Person 12', 1)[0].nodeRef).toEqual(['person', 12]); + }); + + it('writes the graph file and the JSON sidecar exactly as it does without a snapshot', () => { + const graph = sampleGraph(); + const a = tmp(); + const b = tmp(); + saveGraph(graph, join(a, 'g.ison')); + saveGraph(graph, join(b, 'g.ison'), { snapshot: true }); + expect(readFileSync(join(b, 'g.ison'))).toEqual(readFileSync(join(a, 'g.ison'))); + expect(readFileSync(embeddingsPathFor(join(b, 'g.ison')))).toEqual(readFileSync(embeddingsPathFor(join(a, 'g.ison')))); + }); + + it('spans several blocks, and the DataView path writes and reads the same bytes as the fast path', () => { + const store = new EmbeddingStore(); + const next = rng(3); + for (let i = 0; i < 2600; i++) { + store.add(['n', `id${i}`], `text ${i}`, Array.from({ length: 6 }, () => Math.fround(next() - 0.5))); + } + const fast = snapshotToBytes(store, {}, sha()); + const portable = snapshotToBytes(store, {}, sha(), { portable: true }); + expect(portable.bytes).toEqual(fast.bytes); + expect(fast.info.count).toBe(2600); + + for (const options of [{}, { portable: true }, { windowBytes: 64 }, { windowBytes: 1000, portable: true }]) { + const copy = new EmbeddingStore(); + applySnapshot(copy, readSnapshotBytes(fast.bytes, sha(), options)); + expect(dump(copy)).toEqual(dump(store)); + } + const small = snapshotToBytes(store, {}, sha(), { blockRows: 7 }); // the reader honours the header's block size + const copy = new EmbeddingStore(); + applySnapshot(copy, readSnapshotBytes(small.bytes, sha())); + expect(dump(copy)).toEqual(dump(store)); + }); + + it('round-trips an empty store', () => { + const store = new EmbeddingStore(); + const { bytes } = snapshotToBytes(store, {}, sha()); + const again = new EmbeddingStore(); + applySnapshot(again, readSnapshotBytes(bytes, sha())); + expect(again.count()).toBe(0); + expect(again.dimension).toBeNull(); + }); + + it('matches the byte layout of the specification example (123 bytes)', () => { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'x', [1.0, -2.0]); + store.version = null; + const { bytes } = snapshotToBytes(store, {}, sha()); + const u32 = (n: number): number[] => [n & 255, (n >> 8) & 255, (n >> 16) & 255, (n >>> 24) & 255]; + const ascii = (s: string): number[] => Array.from(s).map(c => c.charCodeAt(0)); + const expected = [ + ...ascii('ISONVSNP'), ...u32(1), ...u32(0), + ...u32(2), ...u32(1024), ...u32(1), ...u32(0), + ...u32(0xffffffff), ...u32(0xffffffff), ...u32(0xffffffff), ...u32(0xffffffff), + ...u32(1), ...u32(8), ...ascii('provided'), + ...u32(1), ...ascii('t'), ...u32(1), ...ascii('a'), ...u32(1), ...ascii('x'), ...u32(0), + 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0, + ]; + expect(expected).toHaveLength(91); + expect(bytes).toHaveLength(123); + expect(Array.from(bytes.subarray(0, 91))).toEqual(expected); + expect(Buffer.from(bytes.subarray(91)).toString('hex')).toBe(createHash('sha256').update(Uint8Array.from(expected)).digest('hex')); + }); +}); + +describe('snapshot rejection rules', () => { + /** A two-row snapshot ('t','a','x') and ('t','b','y'), the store it came from, and its bytes. */ + function twoRows(): { store: EmbeddingStore; bytes: Uint8Array } { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'x', [1, 0, 0]); + store.add(['t', 'b'], 'y', [0, 1, 0]); + return { store, bytes: snapshotToBytes(store, {}, sha()).bytes }; + } + const dv = (b: Uint8Array): DataView => new DataView(b.buffer, b.byteOffset, b.byteLength); + function find(bytes: Uint8Array, needle: number[], from = 0): number { + outer: for (let i = from; i <= bytes.length - needle.length; i++) { + for (let j = 0; j < needle.length; j++) if (bytes[i + j] !== needle[j]) continue outer; + return i; + } + throw new Error('pattern not found'); + } + const reject = (bytes: Uint8Array, reason: RegExp, options = {}): void => { + const target = new EmbeddingStore(); + target.add(['keep', 1], 'kept', [9, 9, 9]); + const before = dump(target); + expect(() => applySnapshot(target, readSnapshotBytes(bytes, sha(), options))).toThrow(SnapshotError); + try { + readSnapshotBytes(bytes, sha(), options); + } catch (error) { + expect((error as SnapshotError).reason).toMatch(reason); + } + expect(dump(target)).toEqual(before); // nothing was installed + expect(target.dimension).toBe(3); + }; + + it('rejects bad magic, an unknown version, unknown flags and an implausible shape', () => { + const { bytes } = twoRows(); + let b = bytes.slice(); b[0] = 0x58; reject(reseal(b), /bad magic/); + b = bytes.slice(); dv(b).setUint32(8, 2, true); reject(reseal(b), /version 2/); + b = bytes.slice(); dv(b).setUint32(12, 2, true); reject(reseal(b), /unknown flags/); + b = bytes.slice(); dv(b).setUint32(20, 0, true); reject(reseal(b), /block size/); + b = bytes.slice(); dv(b).setUint32(20, (1 << 20) + 1, true); reject(reseal(b), /block size/); + b = bytes.slice(); dv(b).setUint32(16, 0, true); reject(reseal(b), /implausible shape/); + b = bytes.slice(); dv(b).setUint32(24, 0xffffffff, true); dv(b).setUint32(28, 1, true); reject(reseal(b), /implausible shape/); + b = bytes.slice(); dv(b).setUint32(24, 1000, true); reject(reseal(b), /fewer bytes than the rows need/); + }); + + it('rejects a repeated key, a model index out of range and a string past the end', () => { + const { bytes } = twoRows(); + const dup = bytes.slice(); + const idB = find(dup, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x62]); // type 't', id 'b' + dup[idB + 9] = 0x61; // ... now 'a': the key t:a twice + reject(reseal(dup), /appears twice/); + + const model = bytes.slice(); + const rowA = find(model, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61, 1, 0, 0, 0, 0x78]); + dv(model).setUint32(rowA + 15, 5, true); // model_index 5 of 1 + reject(reseal(model), /model that is not listed/); + + const long = bytes.slice(); + dv(long).setUint32(find(long, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61]), 0x7ffffff0, true); + reject(reseal(long), /longer than the file/); + + const absent = bytes.slice(); + dv(absent).setUint32(find(absent, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61]), 0xffffffff, true); + reject(reseal(absent), /marked absent/); + }); + + it('rejects bytes between the last block and the trailer, bytes after it, and a wrong checksum', () => { + const { bytes } = twoRows(); + const between = new Uint8Array(bytes.length + 1); + between.set(bytes.subarray(0, bytes.length - 32)); + between.set(bytes.subarray(bytes.length - 32), bytes.length - 31); + reject(reseal(between), /left over/); + + const after = new Uint8Array(bytes.length + 1); + after.set(bytes); + reject(after, /./); + + const bad = bytes.slice(); + bad[bytes.length - 40] ^= 1; // inside the last row's floats + reject(bad, /checksum/); + }); + + it('rejects a truncation at every length', () => { + const { bytes } = twoRows(); + for (let n = 0; n < bytes.length; n++) { + expect(() => readSnapshotBytes(bytes.slice(0, n), sha()), `length ${n}`).toThrow(SnapshotError); + } + }); + + it('rejects a string that is not UTF-8', () => { + const { bytes } = twoRows(); + const b = bytes.slice(); + b[find(b, [1, 0, 0, 0, 0x74, 1, 0, 0, 0, 0x61, 1, 0, 0, 0, 0x78]) + 14] = 0xff; // the text of row a + reject(reseal(b), /not UTF-8/); + }); + + it('refuses what `accept` refuses before reading any row', () => { + const { bytes } = twoRows(); + expect(() => readSnapshotBytes(bytes, sha(), { accept: () => 'not for you' })).toThrow(/not for you/); + }); + + it('refuses to write a lone surrogate (UTF-8 cannot carry it)', () => { + const store = new EmbeddingStore(); + store.add(['t', 'a'], 'bad \ud800 text', [1]); + expect(() => snapshotToBytes(store, {}, sha())).toThrow(SnapshotError); + }); +}); + +describe('snapshot protocol: which file wins, and why a snapshot is skipped', () => { + function saved(): { dir: string; path: string; graph: SemanticGraph } { + const dir = tmp(); + const path = join(dir, 'g.ison'); + const graph = sampleGraph(); + saveGraph(graph, path, { snapshot: true }); + return { dir, path, graph }; + } + + it('skips a snapshot whose sidecar changed since (stale) and loads the new JSON', () => { + const { path, graph } = saved(); + const sidecar = embeddingsPathFor(path); + const payload = JSON.parse(readFileSync(sidecar, 'utf8')); + payload.embeddings[0].text = 'edited after the snapshot'; + writeFileSync(sidecar, JSON.stringify(payload)); + const { graph: loaded, report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/stale/); + expect(loaded.embeddingStore.get(['person', 1])!.text).toBe('edited after the snapshot'); + expect(graph.embeddingStore.get(['person', 1])!.text).not.toBe('edited after the snapshot'); + }); + + it('skips a snapshot saved for a different graph digest', () => { + const { path, graph } = saved(); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { + graphDigest: 'sha256:' + '0'.repeat(64), + sidecar: sidecarIdentity(embeddingsPathFor(path)), + }); + const { report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/different graph/); + }); + + it('skips a snapshot that records no digest or no sidecar identity', () => { + const { path, graph } = saved(); + const digest = graphDigest(graph); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { sidecar: sidecarIdentity(embeddingsPathFor(path)) }); + expect(loadGraphWithReport(path).report.snapshotSkipped).toMatch(/different graph/); + saveSnapshot(graph.embeddingStore, snapshotPathFor(path), { graphDigest: digest }); + expect(loadGraphWithReport(path).report.snapshotSkipped).toMatch(/different graph|no sidecar identity/); + }); + + it('skips a snapshot when the graph file was edited (with onMismatch: warn for the JSON)', () => { + const { path } = saved(); + writeFileSync(path, readFileSync(path, 'utf8').replace('Person 5', 'Person five')); + const quiet = (): void => {}; + const warn = console.warn; + console.warn = quiet; + try { + const { report } = loadGraphWithReport(path, { onMismatch: 'warn' }); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toMatch(/different graph/); + } finally { + console.warn = warn; + } + }); + + it('never uses a snapshot when the sidecar is missing', () => { + const { path } = saved(); + rmSync(embeddingsPathFor(path)); + const { graph, report } = loadGraphWithReport(path); + expect(report.source).toBe('none'); + expect(graph.embeddingStore.count()).toBe(0); + }); + + it('embeddings: false ignores the snapshot too', () => { + const { path } = saved(); + expect(loadGraphWithReport(path, { embeddings: false }).report.source).toBe('none'); + }); + + it('a save without a snapshot deletes the old one, and a plain load then reads the JSON', () => { + const { path, graph } = saved(); + graph.embeddingStore.add(['person', 1], 'changed', new Array(8).fill(0.25)); + saveGraph(graph, path); + expect(existsSync(snapshotPathFor(path))).toBe(false); + const { graph: loaded, report } = loadGraphWithReport(path); + expect(report.source).toBe('json'); + expect(report.snapshotSkipped).toBeNull(); + expect(loaded.embeddingStore.get(['person', 1])!.text).toBe('changed'); + }); + + it('a snapshot that cannot be written still saves the pair, removes a stale snapshot and throws', () => { + const { path, graph } = saved(); + graph.embeddingStore.add(['orphan', 5], 'an integer id for a node the graph does not have', new Array(8).fill(0.5)); + expect(() => saveGraph(graph, path, { snapshot: true })).toThrow(SnapshotError); + expect(existsSync(snapshotPathFor(path))).toBe(false); + expect(readdirSync(dirname(path)).filter(f => f.endsWith('.tmp'))).toEqual([]); + const { graph: loaded, report } = loadGraphWithReport(path); // the pair was saved: JSON reads fine + expect(report.source).toBe('json'); + expect(loaded.embeddingStore.get(['orphan', 5])!.text).toMatch(/orphan|integer id/); + expect(typeof loaded.embeddingStore.get(['orphan', 5])!.nodeRef[1]).toBe('number'); + }); + + it('a failing write leaves the previous snapshot untouched and no temp file behind', () => { + const { dir, path, graph } = saved(); + const before = readFileSync(snapshotPathFor(path)); + const bad = new EmbeddingStore(); + bad.add(['t', 'a'], 'lone \ud800 surrogate', [1, 2]); + expect(() => saveSnapshot(bad, snapshotPathFor(path), {})).toThrow(SnapshotError); + expect(readFileSync(snapshotPathFor(path))).toEqual(before); + expect(readdirSync(dir).filter(f => f.endsWith('.tmp'))).toEqual([]); + expect(loadGraphWithReport(path).report.source).toBe('snapshot'); + expect(graph.embeddingStore.count()).toBeGreaterThan(0); + }); + + it('loadSnapshot replaces the store only when the whole file verifies', () => { + const { path, graph } = saved(); + const snap = snapshotPathFor(path); + const store = new EmbeddingStore(); + store.add(['keep', 1], 'kept', new Array(8).fill(1)); + const bytes = new Uint8Array(readFileSync(snap)); + const broken = bytes.slice(); + broken[bytes.length - 50] ^= 8; + writeFileSync(snap, broken); + expect(() => loadSnapshot(store, snap)).toThrow(SnapshotError); + expect(store.count()).toBe(1); + expect(store.get(['keep', 1])!.text).toBe('kept'); + writeFileSync(snap, bytes); + const info = loadSnapshot(store, snap); + expect(info.count).toBe(graph.embeddingStore.count()); + expect(store.get(['keep', 1])).toBeNull(); + expect(store.count()).toBe(info.count); + expect(store.get(['person', '1'])!.nodeRef).toEqual(['person', '1']); // no id type: a bare load reads ids as text + }); + + it('is deterministic: saving the same store twice gives the same bytes', () => { + const { path, graph } = saved(); + const first = readFileSync(snapshotPathFor(path)); + saveGraph(graph, path, { snapshot: true }); + expect(readFileSync(snapshotPathFor(path))).toEqual(first); + }); +}); + +describe('the single-copy table against a plain reference', () => { + /** What the store computed before the rework: float64 cosine over number[] copies, in arrival order. */ + class Reference { + rows = new Map(); + add(ref: NodeRef, text: string, vector: number[]): void { + this.rows.set(`${ref[0]}:${ref[1]}`, { ref, text, vector: vector.map(Math.fround) }); + } + remove(ref: NodeRef): void { + this.rows.delete(`${ref[0]}:${ref[1]}`); + } + private static cosine(q: number[], v: number[]): number { + let dot = 0, n1 = 0, n2 = 0; + for (let i = 0; i < q.length; i++) { dot += q[i] * v[i]; n1 += q[i] * q[i]; n2 += v[i] * v[i]; } + n1 = Math.sqrt(n1); n2 = Math.sqrt(n2); + return n1 === 0 || n2 === 0 ? 0 : dot / (n1 * n2); + } + search(q: number[], k: number | null, type: string | undefined, threshold: number): SimilarityResult[] { + const out: SimilarityResult[] = []; + for (const row of this.rows.values()) { + if (type && row.ref[0] !== type) continue; + const score = Reference.cosine(q, row.vector); + if (score >= threshold) out.push({ nodeRef: row.ref, score, text: row.text }); + } + return k === null ? out.sort((a, b) => b.score - a.score) : selectTopK(out, k); + } + scoreMap(q: number[]): Array<[string, number]> { + return [...this.rows].map(([key, row]) => [key, Reference.cosine(q, row.vector)]); + } + } + + it('returns the same ids, scores (bit for bit) and tie order after updates, deletes and compaction', () => { + const next = rng(21); + const dim = 9; + const vec = (): number[] => Array.from({ length: dim }, () => Math.fround(next() * 2 - 1)); + const store = new EmbeddingStore(); + const ref = new Reference(); + const types = ['a', 'b', 'c:d']; + const refOf = (i: number): NodeRef => [types[i % 3], i % 5 === 0 ? `s${i}` : i]; + const pool: number[][] = []; + for (let i = 0; i < 2300; i++) { // crosses two chunk boundaries + const r = next(); + const v = r < 0.06 && pool.length ? pool[Math.floor(next() * pool.length)] : r < 0.09 ? new Array(dim).fill(0) : vec(); + pool.push(v); + store.add(refOf(i), `t${i}`, v); + ref.add(refOf(i), `t${i}`, v); + } + for (let i = 0; i < 2300; i += 4) { const v = vec(); store.add(refOf(i), `u${i}`, v); ref.add(refOf(i), `u${i}`, v); } + for (let i = 1; i < 2300; i += 3) { store.remove(refOf(i)); ref.remove(refOf(i)); } // more than a quarter: compaction + for (let i = 0; i < 30; i++) { const v = vec(); store.add(refOf(i), `again${i}`, v); ref.add(refOf(i), `again${i}`, v); } + expect(store.count()).toBe(ref.rows.size); + + for (let i = 0; i < 25; i++) { + const q = i === 0 ? new Array(dim).fill(0) : i < 3 ? pool[i].slice() : vec(); + for (const k of [1, 7, 10, 300, 400, null]) for (const type of [undefined, 'b', 'nope']) for (const th of [0, -1, 0.4]) { + expect(store.similaritySearch(q, k, type, th), `k=${k} type=${type} th=${th}`).toEqual(ref.search(q, k, type, th)); + } + expect([...store.scoreMap(q)]).toEqual(ref.scoreMap(q)); + } + }); + + it('gives a get() copy: changing its vector changes nothing in the store', () => { + const store = new EmbeddingStore(); + store.add(['t', 1], 'x', [1, 2, 3]); + const record = store.get(['t', 1])!; + record.vector[0] = 99; + expect(store.get(['t', 1])!.vector).toEqual([1, 2, 3]); + expect(store.similaritySearch([1, 2, 3], 1)[0].score).toBeCloseTo(1, 12); + }); + + it('keeps one copy: the index reports no cache and the vectors take about 4 bytes per component', () => { + const store = new EmbeddingStore(undefined, { ann: 'ivf', annMinSize: 0, annNlist: 16, annNprobe: 4 }); + const next = rng(2); + for (let i = 0; i < 3000; i++) store.add(['d', i], '', Array.from({ length: 32 }, () => next() - 0.5)); + store.buildAnnIndex(); + const stats = store.annStats(); + expect(stats.active).toBe(true); + expect(stats.cacheBytes).toBe(0); + expect(stats.vectorBytes).toBeGreaterThanOrEqual(3000 * 32 * 4); + expect(stats.vectorBytes).toBeLessThan(3000 * 32 * 4 * 1.2); + }); +}); diff --git a/isongraph-vector-ts/src/snapshot.ts b/isongraph-vector-ts/src/snapshot.ts new file mode 100644 index 0000000..e389679 --- /dev/null +++ b/isongraph-vector-ts/src/snapshot.ts @@ -0,0 +1,638 @@ +/** + * Binary snapshot of an EmbeddingStore (optional, beside the JSON sidecar). + * + * The byte layout is version 1 of the snapshot format in the Rust crate's + * DESIGN.md, section 11.2; every integer is little endian and every float an + * IEEE-754 binary32 (read and written through a DataView with an explicit byte + * order, with a plain byte copy as the fast path on little-endian hosts). The + * JSON sidecar stays the portable, authoritative format; a snapshot is a cache + * of it, taken only when it provably belongs to the graph and sidecar on disk. + * + * This module is browser-safe: it touches no file and no `node:` API, and it + * does not hash anything itself. The SHA-256 trailer is computed by a + * {@link Sha256Like} the caller supplies (`node:crypto` in `./node`, which also + * does the file handling and the sidecar fingerprint). + * + * Differences from the Rust crate that are forced by this port: + * - rows are written in the order of this store's JSON sidecar (arrival order, + * live rows only), so that a snapshot load and a JSON load give the same + * slots; the Rust crate's sidecar happens to be sorted, ours is not; + * - the format has no `id_type`, so an integer node id comes back as a string + * unless the caller's `resolveId` says otherwise (`./node` asks the graph). + */ + +import { + EmbeddingError, + EmbeddingStore, + VectorTable, + GRAPH_DIGEST_BASIS, + hostIsLittleEndian, + type ImportMeta, + type NodeRef, +} from './index.js'; + +/** Appended to the graph file's name: `.embeddings.snap`. */ +export const SNAPSHOT_SUFFIX = '.embeddings.snap'; +/** The only layout version this build reads and writes. */ +export const SNAPSHOT_VERSION = 1; +const MAGIC = 'ISONVSNP'; +/** Rows per block (a writer's choice, recorded in the header; readers honour it). */ +const BLOCK_ROWS = 1024; +/** Length value of an absent optional string. */ +const ABSENT = 0xffffffff; +/** Header flag: the header carries the sidecar's identity. */ +const FLAG_SIDECAR = 1; +const KNOWN_FLAGS = FLAG_SIDECAR; +const TRAILER = 32; +const TWO_32 = 4294967296; +const DEFAULT_WINDOW = 4 << 20; + +/** The snapshot is not usable (malformed, truncated, stale, or not the one asked for). */ +export class SnapshotError extends EmbeddingError { + /** Why, without the "invalid snapshot" prefix of `message`. */ + readonly reason: string; + + constructor(reason: string) { + super(`invalid snapshot: ${reason}`); + this.name = 'SnapshotError'; + this.reason = reason; + } +} + +/** + * A cheap identity for a sidecar file: its length and a SHA-256 over its + * length, its first and last 64 KiB and 64 probes of 4 KiB spread over the rest + * (see `sidecarIdentity` in `./node`). + */ +export interface SidecarId { + len: number; + /** 32 bytes. */ + fingerprint: Uint8Array; +} + +/** What a snapshot is tied to. */ +export interface SnapshotIdentity { + /** `graphDigest` of the graph file (the JSON sidecar records the same). */ + graphDigest?: string | null; + /** The JSON sidecar written for the same save. */ + sidecar?: SidecarId | null; +} + +/** The header of a snapshot, and its size. */ +export interface SnapshotInfo { + version: number; + dimension: number; + count: number; + graphDigest: string | null; + graphDigestBasis: string | null; + encoderModel: string | null; + encoderVersion: string | null; + sidecar: SidecarId | null; + /** Size in bytes, trailer included. */ + bytes: number; +} + +/** The part of a SHA-256 implementation this module needs. `digest()` is called once, at the end. */ +export interface Sha256Like { + update(data: Uint8Array): void; + /** The 32-byte digest of everything passed to `update`. */ + digest(): Uint8Array; +} + +/** Reads up to `into.length` bytes into `into`; returns how many, 0 at the end of the data (like `fs.readSync`). */ +export type FillFn = (into: Uint8Array) => number; + +export interface SnapshotWriteOptions { + /** Rows per block (default 1024; the reader honours whatever the header says). */ + blockRows?: number; + /** Write floats through a DataView even on a little-endian host (used by the tests). */ + portable?: boolean; +} + +export interface SnapshotReadOptions { + /** + * Called with the header before any row is read; return a reason to refuse + * the snapshot (nothing is read, nothing changes), or null to accept it. + */ + accept?: (info: SnapshotInfo) => string | null; + /** + * The id a row's id string stands for. The format stores ids as strings; a + * caller that knows an id is an integer (a graph with such a node) returns the + * number. Default: the string. + */ + resolveId?: (type: string, id: string) => string | number; + /** Read floats through a DataView even on a little-endian host (used by the tests). */ + portable?: boolean; + /** Size of the read-ahead window in bytes (default 4 MiB; grows to fit a block). */ + windowBytes?: number; +} + +const encoder = new TextEncoder(); +const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }); + +// ============================================================================= +// Writing +// ============================================================================= + +/** Appends little-endian values to a growing buffer. */ +class ByteWriter { + buf: Uint8Array; + dv: DataView; + len = 0; + + constructor(capacity: number) { + this.buf = new Uint8Array(capacity); + this.dv = new DataView(this.buf.buffer); + } + + reset(): void { + this.len = 0; + } + + /** Make room for `n` more bytes. */ + reserve(n: number): void { + if (this.len + n <= this.buf.length) return; + const grown = new Uint8Array(Math.max(this.len + n, this.buf.length * 2)); + grown.set(this.buf.subarray(0, this.len)); + this.buf = grown; + this.dv = new DataView(grown.buffer); + } + + u32(value: number): void { + this.reserve(4); + this.dv.setUint32(this.len, value, true); + this.len += 4; + } + + u64(value: number): void { + this.reserve(8); + this.dv.setUint32(this.len, value % TWO_32, true); + this.dv.setUint32(this.len + 4, Math.floor(value / TWO_32), true); + this.len += 8; + } + + bytes(data: Uint8Array): void { + this.reserve(data.length); + this.buf.set(data, this.len); + this.len += data.length; + } + + /** A length-prefixed string, already encoded. */ + str(data: Uint8Array): void { + this.u32(data.length); + this.bytes(data); + } + + /** An optional string: absent is a length of 0xFFFFFFFF. */ + opt(value: string | null | undefined): void { + if (value === null || value === undefined) this.u32(ABSENT); + else this.str(encoder.encode(value)); + } + + view(): Uint8Array { + return this.buf.subarray(0, this.len); + } +} + +/** A string the format can carry: UTF-8 cannot hold a lone surrogate. */ +function wellFormed(s: string): boolean { + const check = (s as unknown as { isWellFormed?: () => boolean }).isWellFormed; + if (typeof check === 'function') return check.call(s); + return !/[\ud800-\udbff](?![\udc00-\udfff])|(? string | number +): string | null { + const table = store.snapshotState().table; + if (table === null) return null; + for (let slot = 0; slot < table.n; slot++) { + const ref = table.refs[slot]; + if (ref === null) continue; + const back = resolveId(String(ref[0]), String(ref[1])); + if (back !== ref[1]) { + return ( + `the id ${JSON.stringify(ref[1])} of a '${String(ref[0])}' embedding would read back as ` + + `${JSON.stringify(back)}: a snapshot stores ids as text, and the graph does not say which kind this is` + ); + } + } + return null; +} + +/** + * Write the store as a snapshot: the bytes go to `sink` in order (header, then + * one piece per block, then the 32-byte SHA-256 trailer taken from `hasher`), + * each piece valid only until `sink` returns. Rows are the live ones in arrival + * order. Throws SnapshotError for a value the format cannot carry; the caller + * decides what to do with a partial write (`./node` writes to a temp file). + */ +export function writeSnapshot( + store: EmbeddingStore, + identity: SnapshotIdentity, + hasher: Sha256Like, + sink: (bytes: Uint8Array) => void, + options: SnapshotWriteOptions = {} +): SnapshotInfo { + const blockRows = options.blockRows ?? BLOCK_ROWS; + if (!Number.isInteger(blockRows) || blockRows < 1 || blockRows > 1 << 20) { + throw new EmbeddingError(`blockRows must be an integer in 1..${1 << 20}, got ${blockRows}`); + } + const littleEndian = hostIsLittleEndian() && !options.portable; + const { table, encoderModel, encoderVersion } = store.snapshotState(); + + const slots: number[] = []; + if (table !== null) for (let s = 0; s < table.n; s++) if (table.keys[s] !== null) slots.push(s); + const count = slots.length; + const dim = table === null || count === 0 ? 0 : table.dim; + + // models, in order of first appearance + const modelIndex = new Map(); + const models: string[] = []; + if (table !== null) { + for (const s of slots) { + const name = table.modelNames[table.modelIx[s]]; + if (!modelIndex.has(name)) { + modelIndex.set(name, models.length); + models.push(name); + } + } + } + + const digest = identity.graphDigest ?? null; + const basis = digest === null ? null : GRAPH_DIGEST_BASIS; + const sidecar = identity.sidecar ?? null; + if (sidecar !== null && sidecar.fingerprint.length !== 32) { + throw new EmbeddingError(`sidecar fingerprint must be 32 bytes, got ${sidecar.fingerprint.length}`); + } + + const out = new ByteWriter(4096); + out.bytes(encoder.encode(MAGIC)); + out.u32(SNAPSHOT_VERSION); + out.u32(sidecar !== null ? FLAG_SIDECAR : 0); + out.u32(dim); + out.u32(blockRows); + out.u64(count); + out.opt(digest); + out.opt(basis); + out.opt(encoderModel); + out.opt(encoderVersion); + if (sidecar !== null) { + out.u64(sidecar.len); + out.bytes(sidecar.fingerprint); + } + out.u32(models.length); + for (const name of models) out.str(encodeText(name, 'a model name')); + + let written = 0; + const emit = (): void => { + const bytes = out.view(); + hasher.update(bytes); + sink(bytes); + written += bytes.length; + }; + emit(); + + for (let start = 0; start < count; start += blockRows) { + const rows = Math.min(blockRows, count - start); + out.reset(); + for (let i = 0; i < rows; i++) { + const s = slots[start + i]; + const ref = table!.refs[s]!; + out.str(encodeText(String(ref[0]), 'a node type')); + out.str(encodeText(String(ref[1]), 'a node id')); + out.str(encodeText(table!.texts[s], 'an embedding text')); + out.u32(modelIndex.get(table!.modelNames[table!.modelIx[s]])!); + } + out.reserve(rows * dim * 4); + for (let i = 0; i < rows; i++) { + const s = slots[start + i]; + const chunk = table!.chunks[s >>> 10]; + const base = (s & 1023) * dim; + if (littleEndian) { + out.buf.set(new Uint8Array(chunk.buffer, chunk.byteOffset + base * 4, dim * 4), out.len); + out.len += dim * 4; + } else { + for (let j = 0; j < dim; j++) { + out.dv.setFloat32(out.len, chunk[base + j], true); + out.len += 4; + } + } + } + emit(); + } + + const trailer = hasher.digest(); + if (trailer.length !== TRAILER) throw new EmbeddingError('the hasher did not return 32 bytes'); + sink(trailer); + written += TRAILER; + return { + version: SNAPSHOT_VERSION, + dimension: dim, + count, + graphDigest: digest, + graphDigestBasis: basis, + encoderModel, + encoderVersion, + sidecar, + bytes: written, + }; +} + +/** {@link writeSnapshot} into memory. */ +export function snapshotToBytes( + store: EmbeddingStore, + identity: SnapshotIdentity, + hasher: Sha256Like, + options: SnapshotWriteOptions = {} +): { bytes: Uint8Array; info: SnapshotInfo } { + const pieces: Uint8Array[] = []; + const info = writeSnapshot(store, identity, hasher, piece => pieces.push(piece.slice()), options); + const bytes = new Uint8Array(info.bytes); + let at = 0; + for (const piece of pieces) { + bytes.set(piece, at); + at += piece.length; + } + return { bytes, info }; +} + +// ============================================================================= +// Reading +// ============================================================================= + +/** + * A read-ahead window over a stream of known size. It hands out the next `n` + * bytes of the body (everything before the trailer) as a view that is valid until + * the next call, hashes each byte of the body once as it enters the window, and + * refuses to hand out more than the body holds. + */ +class Window { + buf: Uint8Array; + dv: DataView; + pos = 0; + end = 0; + consumed = 0; + private _pulled = 0; + private readonly _body: number; + + constructor( + private readonly _fill: FillFn, + size: number, + private readonly _hasher: Sha256Like, + capacity: number + ) { + this._body = size - TRAILER; + this.buf = new Uint8Array(capacity); + this.dv = new DataView(this.buf.buffer); + } + + remaining(): number { + return this._body - this.consumed; + } + + /** Make `n` bytes available at `pos`. */ + private _ensure(n: number): void { + if (this.end - this.pos >= n) return; + if (this.pos > 0) { + this.buf.copyWithin(0, this.pos, this.end); + this.end -= this.pos; + this.pos = 0; + } + if (this.buf.length < n) { + const grown = new Uint8Array(Math.max(n, this.buf.length * 2)); + grown.set(this.buf.subarray(0, this.end)); + this.buf = grown; + this.dv = new DataView(grown.buffer); + } + while (this.end < n) { + const got = this._fill(this.buf.subarray(this.end)); + if (!(got > 0)) throw new SnapshotError('truncated'); + const hashable = Math.max(0, Math.min(got, this._body - this._pulled)); + if (hashable > 0) this._hasher.update(this.buf.subarray(this.end, this.end + hashable)); + this._pulled += got; + this.end += got; + } + } + + take(n: number): Uint8Array { + if (n > this.remaining()) throw new SnapshotError('truncated'); + this._ensure(n); + const view = this.buf.subarray(this.pos, this.pos + n); + this.pos += n; + this.consumed += n; + return view; + } + + u32(): number { + if (4 > this.remaining()) throw new SnapshotError('truncated'); + this._ensure(4); + const value = this.dv.getUint32(this.pos, true); + this.pos += 4; + this.consumed += 4; + return value; + } + + u64(): number { + const low = this.u32(); + const high = this.u32(); + return high * TWO_32 + low; + } + + /** A length-prefixed string; `optional` lets the length be ABSENT. */ + string(optional: boolean): string | null { + const len = this.u32(); + if (len === ABSENT) { + if (optional) return null; + throw new SnapshotError('a required string is marked absent'); + } + if (len > this.remaining()) throw new SnapshotError('a string is longer than the file'); + const bytes = this.take(len); + try { + return decoder.decode(bytes); + } catch { + throw new SnapshotError('a string is not UTF-8'); + } + } + + /** The trailer and a check that nothing follows it. Call after the body is consumed. */ + trailer(): Uint8Array { + this._ensureRaw(TRAILER); + const view = this.buf.slice(this.pos, this.pos + TRAILER); + this.pos += TRAILER; + return view; + } + + private _ensureRaw(n: number): void { + if (this.end - this.pos >= n) return; + if (this.pos > 0) { + this.buf.copyWithin(0, this.pos, this.end); + this.end -= this.pos; + this.pos = 0; + } + while (this.end < n) { + const got = this._fill(this.buf.subarray(this.end)); + if (!(got > 0)) throw new SnapshotError('truncated'); + this._pulled += got; + this.end += got; + } + } + + /** True when the stream has more bytes than the size it was declared with. */ + hasMore(): boolean { + if (this.end > this.pos) return true; + const probe = new Uint8Array(1); + return this._fill(probe) > 0; + } +} + +function sameBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +/** + * Read and fully verify a snapshot of `size` bytes whose bytes `fill` supplies, + * applying every rejection rule of the format: bad magic, version or flags, + * implausible shape, a length past the end, a model index out of range, a + * repeated key, bytes between the last block and the trailer or after it, a + * checksum mismatch. The rows go into a fresh table; nothing else is touched, so + * a failure (a SnapshotError, or whatever `fill` throws) leaves every store as + * it was. Install the result with {@link applySnapshot}. + */ +export function readSnapshot( + fill: FillFn, + size: number, + hasher: Sha256Like, + options: SnapshotReadOptions = {} +): { table: VectorTable | null; info: SnapshotInfo } { + if (size < TRAILER + MAGIC.length) throw new SnapshotError('file too short'); + const portable = options.portable ?? false; + const resolveId = options.resolveId ?? ((_type: string, id: string): string | number => id); + const w = new Window(fill, size, hasher, Math.max(64, options.windowBytes ?? DEFAULT_WINDOW)); + + if (!sameBytes(w.take(MAGIC.length), encoder.encode(MAGIC))) throw new SnapshotError('not a snapshot (bad magic)'); + const version = w.u32(); + if (version !== SNAPSHOT_VERSION) { + throw new SnapshotError(`unsupported snapshot version ${version} (this build reads ${SNAPSHOT_VERSION})`); + } + const flags = w.u32(); + if ((flags & ~KNOWN_FLAGS) !== 0) throw new SnapshotError(`unknown flags 0x${flags.toString(16)}`); + const dim = w.u32(); + const blockRows = w.u32(); + const count = w.u64(); + if (blockRows === 0 || blockRows > 1 << 20) throw new SnapshotError(`implausible block size ${blockRows}`); + if (count > 0xffffffff || (count > 0 && dim === 0)) throw new SnapshotError(`implausible shape ${count} x ${dim}`); + if (count * dim * 4 > w.remaining()) throw new SnapshotError('fewer bytes than the rows need'); + const graphDigest = w.string(true); + const graphDigestBasis = w.string(true); + const encoderModel = w.string(true); + const encoderVersion = w.string(true); + let sidecar: SidecarId | null = null; + if ((flags & FLAG_SIDECAR) !== 0) { + const len = w.u64(); + sidecar = { len, fingerprint: w.take(32).slice() }; + } + const modelCount = w.u32(); + if (modelCount > w.remaining() / 4) throw new SnapshotError('more models than the file can hold'); + const models: string[] = []; + for (let i = 0; i < modelCount; i++) models.push(w.string(false)!); + + const info: SnapshotInfo = { + version, + dimension: dim, + count, + graphDigest, + graphDigestBasis, + encoderModel, + encoderVersion, + sidecar, + bytes: size, + }; + const refusal = options.accept ? options.accept(info) : null; + if (refusal !== null) throw new SnapshotError(refusal); + + const table = count > 0 ? new VectorTable(dim) : null; + let left = count; + while (left > 0) { + const rows = Math.min(left, blockRows); + const types: string[] = new Array(rows); + const ids: string[] = new Array(rows); + const texts: string[] = new Array(rows); + const modelOf = new Int32Array(rows); + for (let i = 0; i < rows; i++) { + types[i] = w.string(false)!; + ids[i] = w.string(false)!; + texts[i] = w.string(false)!; + const m = w.u32(); + if (m >= models.length) throw new SnapshotError('a row names a model that is not listed'); + modelOf[i] = m; + } + const need = rows * dim * 4; + if (need > w.remaining()) throw new SnapshotError('truncated'); + const floats = w.take(need); + for (let i = 0; i < rows; i++) { + const key = `${types[i]}:${ids[i]}`; + if (table!.index.has(key)) throw new SnapshotError(`the key ${key} appears twice`); + const ref = [types[i], resolveId(types[i], ids[i])] as NodeRef; + table!.putLittleEndian(key, ref, texts[i], models[modelOf[i]], floats.subarray(i * dim * 4, (i + 1) * dim * 4), portable); + } + left -= rows; + } + if (w.remaining() !== 0) throw new SnapshotError('bytes left over after the last block'); + const expected = hasher.digest(); + const stored = w.trailer(); + if (!sameBytes(stored, expected)) throw new SnapshotError('checksum mismatch'); + if (w.hasMore()) throw new SnapshotError('bytes after the checksum'); + return { table, info }; +} + +/** {@link readSnapshot} from a byte array. */ +export function readSnapshotBytes( + bytes: Uint8Array, + hasher: Sha256Like, + options: SnapshotReadOptions = {} +): { table: VectorTable | null; info: SnapshotInfo } { + let at = 0; + const fill: FillFn = into => { + const n = Math.min(into.length, bytes.length - at); + if (n <= 0) return 0; + into.set(bytes.subarray(at, at + n)); + at += n; + return n; + }; + return readSnapshot(fill, bytes.length, hasher, options); +} + +/** + * Replace the store's embeddings with a verified snapshot's rows (and remember + * its header as the store's `lastImportMeta`, as a JSON import does). The + * approximate-index settings stay; the index is rebuilt lazily. + */ +export function applySnapshot( + store: EmbeddingStore, + result: { table: VectorTable | null; info: SnapshotInfo } +): void { + const info = result.info; + const meta: ImportMeta = {}; + if (info.encoderModel !== null) meta.encoder_model = info.encoderModel; + if (info.encoderVersion !== null) meta.encoder_version = info.encoderVersion; + if (info.graphDigest !== null) meta.graph_digest = info.graphDigest; + if (info.graphDigestBasis !== null) meta.graph_digest_basis = info.graphDigestBasis; + store.adoptTable(result.table ?? new VectorTable(1), meta); +} From 64cf4bb15298dfc87a19773f40e1effb991a2151 Mon Sep 17 00:00:00 2001 From: Mahesh Vaikri Date: Tue, 17 Nov 2026 08:45:44 -0600 Subject: [PATCH 4/7] Python: one copy of every vector, numpy batch search, optional binary snapshot In-memory stores no longer keep a SQLite copy next to the scan matrix; the contiguous float32 matrix is the table (pure-Python path: one array('f') per row, patched in place with tombstones and compaction). IVF pending rows fold at write time (256). similarity_search_batch does one matrix-matrix product per group of queries. Optional .embeddings.snap per Rust DESIGN.md section 11 (save(snapshot=True), load_with_report; verified accepted by the Rust reader). Bounded exact search now returns canonical scores: BLAS only narrows the field and every row within a derived error margin of the k-th score is rescored with one kernel that gives the same bits whatever else is in the call, which is what makes batch equal single bit for bit. Scores can differ from the previous release in the last float32 bit and near-ties can reorder; store.canonical_scores = False restores the old behaviour. Breaking: in-memory stores have no SQLite table (_get_conn is empty), IVF pending is below 256 between writes. Co-Authored-By: Claude Sonnet 5.5 --- benchmarks/stage-d-python/bench_stage_d.py | 221 +++ isongraph_vector-py/__init__.py | 1543 ++++++++++++++++++-- tests/test_isongraph_vector.py | 31 +- tests/test_stage_d_python.py | 969 ++++++++++++ 4 files changed, 2619 insertions(+), 145 deletions(-) create mode 100644 benchmarks/stage-d-python/bench_stage_d.py create mode 100644 tests/test_stage_d_python.py diff --git a/benchmarks/stage-d-python/bench_stage_d.py b/benchmarks/stage-d-python/bench_stage_d.py new file mode 100644 index 0000000..8fafc21 --- /dev/null +++ b/benchmarks/stage-d-python/bench_stage_d.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +""" +Stage D benchmark for the Python port: memory, single-query and batch search, +snapshot against JSON. + +Each measurement runs in a fresh child process (RSS is only meaningful there) +and is repeated ``--runs`` times; the parent prints medians and ranges. + + python benchmarks/stage-d-python/bench_stage_d.py --impl new --runs 3 + python benchmarks/stage-d-python/bench_stage_d.py --impl path/to/old/__init__.py --runs 3 + +``--impl new`` (default) is this checkout's package; any other value is the path +of an ``__init__.py`` to load instead (used to compare against the code as it was +before Stage D). Data are random unit vectors (fixed seed), 5,000 per add_batch +call as the competitive benchmark does. Timings are indicative only when the +machine is shared; RSS figures are not affected by that. +""" + +import argparse +import gc +import importlib.util +import json +import os +import statistics +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..")) + + +def load_impl(spec: str): + path = os.path.join(REPO, "isongraph_vector-py", "__init__.py") if spec == "new" else spec + name = "isongraph_vector_bench_" + str(abs(hash(path))) + module_spec = importlib.util.spec_from_file_location( + name, path, submodule_search_locations=[os.path.dirname(path)]) + module = importlib.util.module_from_spec(module_spec) + sys.modules[name] = module + module_spec.loader.exec_module(module) + return module + + +def rss_mb(proc) -> float: + return proc.memory_info().rss / 2**20 + + +def child(args) -> dict: + import numpy as np + import psutil + + mod = load_impl(args.impl) + proc = psutil.Process() + gc.collect() + base = rss_mb(proc) + out = {"impl": args.impl, "n": args.n, "dim": args.dim, "baseline_mb": round(base, 1), + "raw_mb": round(args.n * args.dim * 4 / 2**20, 1)} + + kwargs = {} + if args.ann: + kwargs["ann"] = "ivf" + db_dir = tempfile.TemporaryDirectory() if args.file_db else None + db_path = os.path.join(db_dir.name, "emb.db") if db_dir else ":memory:" + store = mod.EmbeddingStore(db_path, dimension=args.dim, **kwargs) + rng = np.random.default_rng(1234) + centers = None + if args.clustered: # 256 cluster centres + noise: the shape of real embeddings, for IVF recall + centers = rng.standard_normal((256, args.dim)).astype(np.float32) + centers /= np.linalg.norm(centers, axis=1, keepdims=True) + + def draw(count, gen): + if centers is None: + return gen.standard_normal((count, args.dim)).astype(np.float32) + which = gen.integers(0, 256, count) + return centers[which] + 0.5 * gen.standard_normal((count, args.dim)).astype(np.float32) / np.sqrt(args.dim) + + started = time.perf_counter() + for start in range(0, args.n, 5000): + count = min(5000, args.n - start) + block = draw(count, rng) + block /= np.linalg.norm(block, axis=1, keepdims=True) + store.add_batch([(("doc", start + i), "", v) for i, v in enumerate(block.tolist())]) + del block + out["ingest_s"] = round(time.perf_counter() - started, 2) + gc.collect() + out["rss_after_ingest_mb"] = round(rss_mb(proc) - base, 1) + + qrng = np.random.default_rng(99) + queries = draw(512, qrng) + queries /= np.linalg.norm(queries, axis=1, keepdims=True) + qlist = queries.tolist() + + started = time.perf_counter() + store.similarity_search(qlist[0], 10) + out["first_query_s"] = round(time.perf_counter() - started, 3) + gc.collect() + out["rss_steady_mb"] = round(rss_mb(proc) - base, 1) + out["rss_ratio_to_raw"] = round(out["rss_steady_mb"] / out["raw_mb"], 2) + + if args.ann: + started = time.perf_counter() + store.build_ann_index() + out["ivf_build_s"] = round(time.perf_counter() - started, 2) + gc.collect() + out["rss_with_ivf_mb"] = round(rss_mb(proc) - base, 1) + + for q in qlist[:20]: + store.similarity_search(q, 10) + timed = qlist[:args.queries] + started = time.perf_counter() + for q in timed: + store.similarity_search(q, 10) + elapsed = time.perf_counter() - started + out["single_qps"] = round(len(timed) / elapsed, 1) + out["single_ms"] = round(1000 * elapsed / len(timed), 3) + if args.ann: + report = store.recall_check(qlist[:50], k=10) + out["ivf_recall_at_10"] = round(report.recall, 4) + + if hasattr(store, "similarity_search_batch") and not args.ann: + for batch in (16, 64, 256): + best = None + for _ in range(3): + started = time.perf_counter() + store.similarity_search_batch(qlist[:batch], 10) + took = time.perf_counter() - started + best = took if best is None else min(best, took) + out[f"batch{batch}_qps"] = round(batch / best, 1) + # the same queries one at a time, for a like-for-like ratio + started = time.perf_counter() + for q in qlist[:256]: + store.similarity_search(q, 10) + out["single_256_qps"] = round(256 / (time.perf_counter() - started), 1) + + if args.files and not args.ann: + with tempfile.TemporaryDirectory() as tmp: + json_path = os.path.join(tmp, "e.embeddings.json") + started = time.perf_counter() + store.save_embeddings(json_path) + out["json_save_s"] = round(time.perf_counter() - started, 2) + out["json_mb"] = round(os.path.getsize(json_path) / 2**20, 1) + gc.collect() + other = mod.EmbeddingStore(":memory:", dimension=args.dim) + started = time.perf_counter() + other.load_embeddings(json_path) + out["json_load_s"] = round(time.perf_counter() - started, 2) + assert other.count() == args.n + other.close() + del other + gc.collect() + if hasattr(store, "save_snapshot"): + snap_path = os.path.join(tmp, "e.embeddings.snap") + started = time.perf_counter() + store.save_snapshot(snap_path) + out["snap_save_s"] = round(time.perf_counter() - started, 2) + out["snap_mb"] = round(os.path.getsize(snap_path) / 2**20, 1) + other = mod.EmbeddingStore(":memory:") + started = time.perf_counter() + other.load_snapshot(snap_path) + out["snap_load_s"] = round(time.perf_counter() - started, 2) + assert other.count() == args.n + other.close() + store.close() + if db_dir: + db_dir.cleanup() + return out + + +def parent(args) -> None: + runs = [] + for i in range(args.runs): + cmd = [sys.executable, os.path.abspath(__file__), "--child", "--impl", args.impl, + "--n", str(args.n), "--dim", str(args.dim), "--queries", str(args.queries)] + if args.ann: + cmd.append("--ann") + if args.files: + cmd.append("--files") + if args.clustered: + cmd.append("--clustered") + if args.file_db: + cmd.append("--file-db") + done = subprocess.run(cmd, capture_output=True, text=True) + if done.returncode != 0: + sys.stderr.write(done.stderr) + raise SystemExit(done.returncode) + runs.append(json.loads(done.stdout.strip().splitlines()[-1])) + print(f"run {i + 1}/{args.runs}: {runs[-1]}", flush=True) + keys = [k for k, v in runs[0].items() if isinstance(v, (int, float)) and k not in ("n", "dim")] + summary = {} + for k in keys: + values = [r[k] for r in runs if k in r] + summary[k] = {"median": statistics.median(values), "min": min(values), "max": max(values)} + print(f"== impl={args.impl} ann={args.ann} clustered={args.clustered} file_db={args.file_db} n={args.n} dim={args.dim} runs={args.runs}") + for k, v in summary.items(): + print(f"{k:24s} median {v['median']:>10} range [{v['min']} .. {v['max']}]") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--impl", default="new") + ap.add_argument("--n", type=int, default=100_000) + ap.add_argument("--dim", type=int, default=384) + ap.add_argument("--runs", type=int, default=3) + ap.add_argument("--queries", type=int, default=200) + ap.add_argument("--ann", action="store_true", help="measure the IVF index instead of the exact scan") + ap.add_argument("--files", action="store_true", help="also time JSON and snapshot save/load") + ap.add_argument("--file-db", action="store_true", + help="back the store with a SQLite file instead of :memory:") + ap.add_argument("--clustered", action="store_true", + help="clustered data instead of uniform random (uniform is the worst case for IVF recall)") + ap.add_argument("--child", action="store_true") + args = ap.parse_args() + if args.child: + print(json.dumps(child(args))) + else: + parent(args) + + +if __name__ == "__main__": + main() diff --git a/isongraph_vector-py/__init__.py b/isongraph_vector-py/__init__.py index ae49539..d6078ad 100644 --- a/isongraph_vector-py/__init__.py +++ b/isongraph_vector-py/__init__.py @@ -46,6 +46,7 @@ import os import sqlite3 import struct +import sys import hashlib import heapq import json @@ -57,6 +58,7 @@ import urllib.request import uuid import warnings +from array import array from collections import deque from abc import ABC, abstractmethod from dataclasses import dataclass @@ -87,8 +89,7 @@ # Approximate (IVF) index policy; see EmbeddingStore._ann_index. _ANN_MIN_SIZE = 50_000 # below this the exact scan is used -_ANN_FOLD_MIN = 512 # fold pending rows into cells at >= this many ... -_ANN_FOLD_FRACTION = 0.02 # ... and at least this fraction of the indexed rows +_ANN_FOLD_BATCH = 256 # a write that leaves this many rows pending folds them into cells _ANN_RETRAIN_FRACTION = 0.5 # retrain once writes+deletes exceed this x trained size __version__ = "1.0.0" @@ -221,6 +222,322 @@ def dump(obj: Any) -> str: return "sha256:" + hashlib.sha256(body.encode('utf-8')).hexdigest() +# ============================================================================= +# Binary snapshot (optional, beside the JSON sidecar) +# ============================================================================= + +#: Suffix appended to a graph path to locate its binary snapshot. +SNAPSHOT_SUFFIX = ".embeddings.snap" +SNAPSHOT_VERSION = 1 + +_SNAP_MAGIC = b"ISONVSNP" +_SNAP_BLOCK_ROWS = 1024 +_SNAP_ABSENT = 0xFFFFFFFF +_SNAP_FLAG_SIDECAR = 1 +_SNAP_TRAILER = 32 +_FP_EDGE = 64 * 1024 +_FP_PROBE = 4 * 1024 +_FP_PROBES = 64 +_U32 = struct.Struct(' FilePath: + """``.embeddings.snap``.""" + return FilePath(str(path) + SNAPSHOT_SUFFIX) + + +def sidecar_identity(path: Union[str, FilePath]) -> Tuple[int, bytes]: + """ + ``(length, fingerprint)`` of a sidecar file, as a snapshot records it. + + The fingerprint is SHA-256 over the length (u64 LE), the first and last + 64 KiB, and 64 probes of 4 KiB at ``floor(len * i / 65)`` (each clipped to + the end of the file). + """ + with open(path, 'rb') as handle: + length = os.fstat(handle.fileno()).st_size + digest = hashlib.sha256() + digest.update(_U64.pack(length)) + + def take(at: int, count: int) -> None: + count = min(count, max(0, length - at)) + handle.seek(at) + data = handle.read(count) + if len(data) != count: + raise SnapshotError(f"short read of {path}") + digest.update(data) + + take(0, _FP_EDGE) + take(max(0, length - _FP_EDGE), _FP_EDGE) + for i in range(1, _FP_PROBES + 1): + take(length * i // (_FP_PROBES + 1), _FP_PROBE) + return length, digest.digest() + + +def _atomic_write_stream(path: Union[str, FilePath], fill: Callable[[Any], Any]) -> Any: + """ + Run ``fill(handle)`` against a temp file beside ``path``, flush it to disk + and rename it over ``path``; a failure removes the temp file and leaves + ``path`` as it was. Returns whatever ``fill`` returns. + """ + path = FilePath(path) + tmp = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp") + try: + with open(tmp, 'wb') as handle: + value = fill(handle) + handle.flush() + try: + os.fsync(handle.fileno()) + except OSError: # pragma: no cover - fsync unsupported on this fs + pass + os.replace(tmp, path) + return value + except BaseException: + try: + tmp.unlink() + except OSError: + pass + raise + + +def _snap_str(text: str) -> bytes: + try: + raw = text.encode('utf-8') + except UnicodeEncodeError as exc: # a lone surrogate: JSON can carry it, UTF-8 cannot + raise SnapshotError(f"a text cannot be stored as UTF-8: {exc}") from exc + return _U32.pack(len(raw)) + raw + + +def _snap_opt(text: Optional[str]) -> bytes: + return _U32.pack(_SNAP_ABSENT) if text is None else _snap_str(text) + + +class _StagedSnapshot: + """A verified snapshot's rows, held apart until they are installed.""" + + __slots__ = ('info', 'models', 'types', 'ids', 'texts', 'model_ix', 'matrix', 'vectors') + + def __init__(self, info: SnapshotInfo): + self.info = info + self.models: List[str] = [] + self.types: List[str] = [] + self.ids: List[str] = [] + self.texts: List[str] = [] + self.model_ix: List[int] = [] + self.matrix = None # numpy: (capacity, dim) float32, first ``count`` rows used + self.vectors: Optional[List[Any]] = None # no numpy: one array('f') per row + + +def _read_snapshot(path: Union[str, FilePath], + accept: Optional[Callable[[SnapshotInfo], None]] = None) -> _StagedSnapshot: + """ + Read and fully verify the snapshot at ``path``. + + ``accept`` sees the header first and raises :class:`SnapshotError` to + refuse the file before its body is read. Every rule of the layout is + checked (lengths, model indexes, repeated keys, leftover bytes, the + trailing checksum); any failure raises :class:`SnapshotError` and nothing + has been installed anywhere. + """ + import mmap + + def bad(message: str) -> SnapshotError: + return SnapshotError(message) + + try: + handle = open(path, 'rb') + except OSError as exc: + raise bad(f"cannot open the snapshot {path}: {exc}") from exc + mapped = None + try: + size = os.fstat(handle.fileno()).st_size + if size < _SNAP_TRAILER + len(_SNAP_MAGIC): + raise bad("file too short") + try: + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + buf = mapped + except (OSError, ValueError): # pragma: no cover - mmap unavailable + buf = handle.read() + end = size - _SNAP_TRAILER # first byte of the trailer + pos = 0 + + def need(count: int) -> None: + if count < 0 or pos + count > end: + raise bad("truncated") + + def u32() -> int: + nonlocal pos + need(4) + value = _U32.unpack_from(buf, pos)[0] + pos += 4 + return value + + def u64() -> int: + nonlocal pos + need(8) + value = _U64.unpack_from(buf, pos)[0] + pos += 8 + return value + + def string(optional: bool) -> Optional[str]: + nonlocal pos + length = u32() + if length == _SNAP_ABSENT: + if optional: + return None + raise bad("a required string is marked absent") + if length > end - pos: + raise bad("a string is longer than the file") + raw = buf[pos:pos + length] + pos += length + try: + return raw.decode('utf-8') + except UnicodeDecodeError: + raise bad("a string is not UTF-8") from None + + if buf[0:8] != _SNAP_MAGIC: + raise bad("not a snapshot (bad magic)") + pos = 8 + version = u32() + if version != SNAPSHOT_VERSION: + raise bad(f"unsupported snapshot version {version} (this build reads {SNAPSHOT_VERSION})") + flags = u32() + if flags & ~_SNAP_FLAG_SIDECAR: + raise bad(f"unknown flags {flags:#x}") + dim = u32() + block_rows = u32() + count = u64() + if block_rows == 0 or block_rows > 1 << 20: + raise bad(f"implausible block size {block_rows}") + if count > 0xFFFFFFFF or (count > 0 and dim == 0): + raise bad(f"implausible shape {count} x {dim}") + if count * dim * 4 > end - pos: + raise bad("fewer bytes than the rows need") + graph_digest_text = string(True) + basis = string(True) + encoder_model = string(True) + encoder_version = string(True) + side_len = side_print = None + if flags & _SNAP_FLAG_SIDECAR: + side_len = u64() + need(32) + side_print = bytes(buf[pos:pos + 32]) + pos += 32 + model_count = u32() + if model_count > (end - pos) // 4: + raise bad("more models than the file can hold") + models = [string(False) for _ in range(model_count)] + info = SnapshotInfo( + version=version, dimension=dim, count=count, graph_digest=graph_digest_text, + graph_digest_basis=basis, encoder_model=encoder_model, + encoder_version=encoder_version, sidecar_len=side_len, + sidecar_fingerprint=side_print, bytes=size) + if accept is not None: + accept(info) + + stage = _StagedSnapshot(info) + stage.models = models + capacity = count + count // 32 + 16 + if _np is not None: + stage.matrix = _np.empty((capacity, dim), dtype=_np.float32) if count else None + else: + stage.vectors = [] + seen = set() + left = count + filled = 0 + while left > 0: + rows = min(left, block_rows) + for _ in range(rows): + ty = string(False) + ident = string(False) + text = string(False) + model = u32() + if model >= len(models): + raise bad("a row names a model that is not listed") + key = ty + ":" + ident + if key in seen: + raise bad(f"the key {key} appears twice") + seen.add(key) + stage.types.append(sys.intern(ty)) + stage.ids.append(ident) + stage.texts.append(text) + stage.model_ix.append(model) + nbytes = rows * dim * 4 + need(nbytes) + if _np is not None: + block = _np.frombuffer(buf, dtype=' str: class _ScanCache: """ - Decoded, incrementally maintained snapshot of the whole table. - - Every search reads every row (brute force), so decoding blobs and, with - numpy, stacking them into one matrix pays for itself immediately on the - second query: the per-query cost drops from a Python loop over N x d - floats to a single matrix-vector product. + The one in-memory copy of every stored vector, plus what a scan needs. - With numpy the snapshot is one C-contiguous float32 matrix with spare row - capacity, the vector norms, a type code per row and a liveness flag. Writes - patch it in place instead of invalidating it: an insert appends a row, an - update overwrites one, a delete tombstones one (compacted once a quarter - of the rows are dead). Rows stay in insertion order, which is the order - the table's rowids give, so ties break exactly as a rebuild would. + Every search reads every row (brute force), so the vectors live in one + place laid out for that: with numpy a single C-contiguous float32 matrix + with spare row capacity, the vector norms, a type code per row and a + liveness flag. Writes patch it in place instead of rebuilding it: an + insert appends a row, an update overwrites one, a delete tombstones one + (compacted once a quarter of the rows are dead). Rows stay in insertion + order, which is the order the table's rowids give, so ties break exactly as + a rebuild would. ``n`` counts used rows including tombstones, ``dead`` the tombstones. - Without numpy (or with ragged vector lengths) ``matrix`` is None and - ``vectors``/``types`` hold plain lists; that form is rebuilt, not patched. + Without numpy (or with ragged vector lengths) ``matrix`` is None and the + rows are plain Python data: ``vectors`` holds one compact ``array('f')`` per + row (None for a tombstone), ``pnorms`` the matching double-precision + norms. Either form is patched in place. + + For an in-memory store this object *is* the table (see + :meth:`EmbeddingStore._table_mode`): ``models`` then holds each row's model + name too. For a file-backed store SQLite is the durable copy and this is + decoded from it on first use. """ __slots__ = ('refs', 'types', 'texts', 'vectors', 'matrix', 'norms', - 'n', 'dead', 'alive', 'codes', 'code_of', 'pos', 'ivf') + 'n', 'dead', 'alive', 'codes', 'code_of', 'pos', 'ivf', + 'models', 'pnorms') def __init__(self, refs, types, texts, vectors, matrix, norms): self.refs = refs @@ -696,6 +1018,8 @@ def __init__(self, refs, types, texts, vectors, matrix, norms): self.code_of = None self.pos = None self.ivf = None # optional approximate index (see _IVFIndex) + self.models = None # per-row model name (in-memory stores only) + self.pnorms = None # per-row norm, list form only def _is_number(value: Any) -> bool: @@ -728,8 +1052,12 @@ class _IVFIndex: * a **new** or **updated** row is *pending*: it has no cell (``assign`` is -1) and every query scores the pending rows exactly, so a write is visible immediately and costs the index nothing; - * pending rows are folded into their nearest cell in one matrix product - once there are enough of them (see ``EmbeddingStore._ann_index``); + * pending rows are folded into their nearest cell in one matrix product as + soon as ``_ANN_FOLD_BATCH`` of them have piled up, by the write that + reaches that number, so a search never waits for a fold; a fold of rows + that were never listed before splices them into the cell lists (one pass + over ``order``), and only a fold that includes an updated or slot-reusing + row regroups everything; * a **delete** just clears the row's cell (the cache has already tombstoned it); stale entries in ``order`` are skipped at query time; * the cache's compaction remaps row numbers, and :meth:`compact` carries @@ -742,7 +1070,7 @@ class _IVFIndex: """ __slots__ = ('centroids', 'nlist', 'assign', 'order', 'offsets', - 'pending', 'churn', 'trained_n', 'build_seconds') + 'pending', 'churn', 'trained_n', 'build_seconds', 'listed') def __init__(self, centroids, assign, trained_n, build_seconds): self.centroids = centroids @@ -750,6 +1078,7 @@ def __init__(self, centroids, assign, trained_n, build_seconds): self.assign = assign # int32 per row slot, -1 = not in a cell self.order = None self.offsets = None + self.listed = None # bool per slot: has an entry in ``order`` self.pending = set() # live rows with no cell yet self.churn = 0 self.trained_n = trained_n @@ -825,15 +1154,39 @@ def relist(self, n: int) -> None: self.order = rows[_np.argsort(picked, kind='stable')].astype(_np.int32) counts = _np.bincount(picked, minlength=self.nlist) self.offsets = _np.concatenate(([0], _np.cumsum(counts))) + if self.listed is None or self.listed.shape[0] != self.assign.shape[0]: + self.listed = _np.zeros(self.assign.shape[0], dtype=bool) + else: + self.listed[:] = False + self.listed[self.order] = True def fold(self, cache) -> None: - """Give every pending row its nearest cell and regroup.""" - if self.pending: - ids = _np.fromiter(sorted(self.pending), dtype=_np.int64, count=len(self.pending)) - self.assign[ids] = self._nearest( - self._unit(cache.matrix[ids], cache.norms[ids]), self.centroids) - self.pending.clear() - self.relist(cache.n) + """Give every pending row its nearest cell. + + Rows that have no entry in ``order`` yet (new rows, the normal case) + are spliced into their cells' runs in one pass; if any pending row + still has a stale entry from an earlier life (an update, a reused + slot), everything is regrouped so no row is ever listed twice. + """ + if not self.pending: + if self.order is None: + self.relist(cache.n) + return + ids = _np.fromiter(sorted(self.pending), dtype=_np.int64, count=len(self.pending)) + cells = self._nearest( + self._unit(cache.matrix[ids], cache.norms[ids]), self.centroids) + self.assign[ids] = cells + self.pending.clear() + if self.order is None or self.listed[ids].any(): + self.relist(cache.n) + return + by_cell = _np.lexsort((ids, cells)) + ids, cells = ids[by_cell], cells[by_cell] + # each new row goes at the end of its cell's run (rows ascend within a run) + self.order = _np.insert(self.order, self.offsets[cells + 1], ids.astype(_np.int32)) + counts = _np.bincount(cells, minlength=self.nlist) + self.offsets = self.offsets + _np.concatenate(([0], _np.cumsum(counts))) + self.listed[ids] = True # -- incremental upkeep --------------------------------------------------- @@ -842,6 +1195,10 @@ def note_put(self, i: int, capacity: int) -> None: grown = _np.full(max(capacity, i + 1), -1, dtype=_np.int32) grown[:self.assign.shape[0]] = self.assign self.assign = grown + listed = _np.zeros(grown.shape[0], dtype=bool) + if self.listed is not None: + listed[:self.listed.shape[0]] = self.listed + self.listed = listed self.assign[i] = -1 self.pending.add(i) self.churn += 1 @@ -865,7 +1222,7 @@ def compact(self, live, kept: int) -> None: def nbytes(self) -> int: total = self.centroids.nbytes + self.assign.nbytes if self.order is not None: - total += self.order.nbytes + self.offsets.nbytes + total += self.order.nbytes + self.offsets.nbytes + self.listed.nbytes return total @@ -894,7 +1251,14 @@ class RecallReport: class EmbeddingStore: """ - SQLite-backed embedding storage with similarity search. + Embedding storage with similarity search; SQLite-backed when it has to be. + + Every vector lives in one place in memory: the scan cache's float32 matrix + (a plain per-row ``array('f')`` without numpy). A store opened on a file + (``db_path``) or with ``sqlite_vec=True`` also writes each row to SQLite, + which is the durable copy (and vec0's index); a store on ``":memory:"``, + the default, has nothing to persist and keeps its rows only in the cache, + without a SQLite table at all. Brute-force cosine similarity, vectorized through numpy when it is installed and falling back to a pure-Python loop when it is not. Search @@ -1020,13 +1384,30 @@ def __init__( self._cache: Optional[_ScanCache] = None self._sqlite_vec = sqlite_vec self._vec_ready = False - with self._lock: - self._get_conn() - if self._dimension is None: - self._adopt_stored_dimension() - if self._sqlite_vec and self._dimension is not None: + #: Whether SQLite holds the rows. A store that is neither file-backed + #: nor sqlite-vec has nothing to persist, so its only copy of the data + #: is the :class:`_ScanCache` (see :meth:`_table_mode`). + self._sql = bool(sqlite_vec) or not self._is_memory_path(self.db_path) + #: Rescore the bounded exact search with the canonical kernel (see + #: :meth:`_finish_exact`). False restores the raw BLAS scores. + self._rescore = True + self._model_names: Dict[str, str] = {} + if self._sql: with self._lock: - self._ensure_vec_table(self._dimension) + self._get_conn() + if self._dimension is None: + self._adopt_stored_dimension() + if self._sqlite_vec and self._dimension is not None: + with self._lock: + self._ensure_vec_table(self._dimension) + else: + self._models_cache = {} + + @staticmethod + def _is_memory_path(db_path: str) -> bool: + """Whether ``db_path`` names a private in-memory SQLite database.""" + return (db_path in ("", ":memory:") or db_path.startswith("file::memory:") + or "mode=memory" in db_path) @property def sqlite_vec(self) -> bool: @@ -1050,6 +1431,31 @@ def ann_nprobe(self, value: Optional[int]) -> None: raise ValueError(f"ann_nprobe must be an integer >= 1, got {value!r}") self._ann_nprobe = value + @property + def canonical_scores(self) -> bool: + """ + Whether bounded exact searches return canonical scores (default True). + + BLAS sums a row's products in an order that depends on where the row + sits in the matrix and on the shape of the call, so the same vector can + score a few float32 ulps differently after a compaction, or in a batch. + With ``canonical_scores`` on, an exact top-k search (and + :meth:`similarity_search_batch`) uses BLAS only to find the candidate + rows and scores those with a kernel whose result for a (row, query) pair + depends on nothing else, so every score is reproducible and a batch + answer is bit-identical to the one-query answer. Turn it off for the raw + BLAS scores of earlier releases (batches then answer query by query). + The approximate index ranks the rows it probed the same way, so a score + it returns is the very number an exact search gives that row. + :meth:`score_map` and the lazily ranked stream behind ``node_filter`` + searches always return raw BLAS scores. + """ + return self._rescore + + @canonical_scores.setter + def canonical_scores(self, value: bool) -> None: + self._rescore = bool(value) + @property def version(self) -> Optional[str]: """Encoder/model version recorded in the sidecar, or None if unknown.""" @@ -1066,10 +1472,20 @@ def _invalidate(self) -> None: Writes patch the caches in place (see :class:`_ScanCache`); this is the fallback for the ones that cannot, and for a write that failed - part-way and left the derived state unknown. + part-way and left the derived state unknown. An in-memory store has + nothing to rebuild the scan cache from, so only the model counts are + dropped there (a write validates and converts the vector before it + changes anything, so a failure leaves the cache whole). """ - self._cache = None - self._models_cache = None + if self._sql: + self._cache = None + self._models_cache = None + else: + self._models_cache = None + + def _table_mode(self) -> bool: + """True when the scan cache is the only copy of the rows (no SQLite).""" + return not self._sql def _models_adjust(self, old: Optional[str], new: Optional[str]) -> None: """Move one row between model buckets in the cached ``models()`` counts.""" @@ -1310,10 +1726,19 @@ def models(self) -> Dict[str, int]: """ with self._lock: if self._models_cache is None: - rows = self._get_conn().execute( - "SELECT model, COUNT(*) AS n FROM embeddings GROUP BY model" - ).fetchall() - self._models_cache = {row['model']: row['n'] for row in rows} + if self._sql: + rows = self._get_conn().execute( + "SELECT model, COUNT(*) AS n FROM embeddings GROUP BY model" + ).fetchall() + self._models_cache = {row['model']: row['n'] for row in rows} + else: + counts: Dict[str, int] = {} + cache = self._cache + if cache is not None: + for ref, model in zip(cache.refs, cache.models): + if ref is not None: + counts[model] = counts.get(model, 0) + 1 + self._models_cache = counts return dict(self._models_cache) def _current_model(self) -> Optional[str]: @@ -1368,6 +1793,24 @@ def stale( except (AttributeError, TypeError): dimension = None with self._lock: + if not self._sql: + cache = self._cache + out = [] + if cache is None: + return out + # every row of an in-memory store has the store's one width + width = self._dimension + for ref, row_model in zip(cache.refs, cache.models): + if ref is None: + continue + by_model = ( + model is not None and row_model != model + and (include_unknown or row_model not in UNKNOWN_MODELS) + ) + by_dim = dimension is not None and width != dimension + if by_model or by_dim: + out.append(ref) + return out rows = self._get_conn().execute(""" SELECT node_type, node_id, node_id_type, model, LENGTH(vector) AS n FROM embeddings ORDER BY rowid @@ -1397,13 +1840,15 @@ def _reset_dimension(self) -> None: def _write_row( self, - conn: sqlite3.Connection, + conn: Optional[sqlite3.Connection], node_ref: NodeRef, vector: Sequence[float], text: str, model: str, - node_id_type: Optional[str] = None - ) -> str: + node_id_type: Optional[str] = None, + blob: Optional[bytes] = None, + want_id: bool = True + ) -> Optional[str]: """ Insert or update one embedding, keeping the vec0 index in step. @@ -1413,10 +1858,21 @@ def _write_row( of (node_type, node_id), so a conflict on one implies a conflict on the other and this single conflict target catches both. + An in-memory store (``conn`` is None) has no table: the row goes straight + into the scan cache, after the vector has been converted, so a bad + vector fails before anything changes. + Caller holds the lock and commits. """ + if blob is None: + blob = self._vector_to_bytes(vector) + if conn is None: + id_type = node_id_type or self._node_id_type(node_ref[1]) + old_model = self._cache_put(node_ref, id_type, blob, text, model) + if self._models_cache is not None: + self._models_adjust(old_model, model) + return self._embedding_id(node_ref) if want_id else None emb_id = self._embedding_id(node_ref) - blob = self._vector_to_bytes(vector) old_model = None if self._models_cache is not None: previous = conn.execute( @@ -1455,7 +1911,7 @@ def _write_row( # Patch what is derived from the table rather than dropping it. self._models_adjust(old_model, model) - self._cache_put(node_ref, node_id_type or self._node_id_type(node_ref[1]), blob, text) + self._cache_put(node_ref, node_id_type or self._node_id_type(node_ref[1]), blob, text, model) return emb_id def add( @@ -1484,10 +1940,11 @@ def add( vector = self._encode(text) self._check_dimension(vector, "vector") - conn = self._get_conn() + conn = self._get_conn() if self._sql else None try: emb_id = self._write_row(conn, node_ref, vector, text, self._model_name()) - conn.commit() + if conn is not None: + conn.commit() except BaseException: self._invalidate() raise @@ -1533,11 +1990,13 @@ def add_batch( for ref, text, vector in items: self._check_dimension(vector, "vector") - conn = self._get_conn() + conn = self._get_conn() if self._sql else None try: + self._cache_reserve(len(items)) for ref, text, vector in items: - self._write_row(conn, ref, vector, text, model) - conn.commit() + self._write_row(conn, ref, vector, text, model, want_id=False) + if conn is not None: + conn.commit() except BaseException: self._invalidate() raise @@ -1550,6 +2009,12 @@ def _embedding_id(node_ref: NodeRef) -> str: def remove(self, node_ref: NodeRef) -> bool: """Remove embedding for a node.""" with self._lock: + if not self._sql: + old_model = self._cache_drop(node_ref) + if old_model is None: + return False + self._models_adjust(old_model, None) + return True conn = self._get_conn() try: row = conn.execute( @@ -1583,12 +2048,14 @@ def clear(self, reset_dimension: bool = False) -> None: vectors of the wrong length even when empty. """ with self._lock: - conn = self._get_conn() - conn.execute("DELETE FROM embeddings") - if self._sqlite_vec and self._vec_ready: - conn.execute(f"DELETE FROM {VEC_TABLE}") - conn.commit() - self._invalidate() + if self._sql: + conn = self._get_conn() + conn.execute("DELETE FROM embeddings") + if self._sqlite_vec and self._vec_ready: + conn.execute(f"DELETE FROM {VEC_TABLE}") + conn.commit() + self._cache = None + self._models_cache = None if self._sql else {} if reset_dimension: self._reset_dimension() @@ -1597,6 +2064,18 @@ def clear(self, reset_dimension: bool = False) -> None: def get(self, node_ref: NodeRef) -> Optional[EmbeddingRecord]: """Get embedding for a node.""" with self._lock: + if not self._sql: + cache = self._cache + i = None if cache is None else cache.pos.get((str(node_ref[0]), str(node_ref[1]))) + if i is None: + return None + if cache.matrix is not None: + vector = cache.matrix[i].tolist() + else: + vector = cache.vectors[i].tolist() + return EmbeddingRecord( + id=self._embedding_id(node_ref), node_ref=cache.refs[i], vector=vector, + text=cache.texts[i], model=cache.models[i]) row = self._get_conn().execute(""" SELECT id, node_type, node_id, node_id_type, vector, text, model FROM embeddings @@ -1618,6 +2097,8 @@ def get(self, node_ref: NodeRef) -> Optional[EmbeddingRecord]: def count(self) -> int: """Count total embeddings.""" with self._lock: + if not self._sql: + return 0 if self._cache is None else self._cache.n - self._cache.dead return self._get_conn().execute("SELECT COUNT(*) FROM embeddings").fetchone()[0] # -- incremental scan cache ------------------------------------------- @@ -1625,10 +2106,15 @@ def count(self) -> int: _CACHE_CHUNK = 8192 def _ensure_cache(self) -> _ScanCache: - """Build (or reuse) the decoded snapshot every search reads. Locked.""" + """The scan cache: decoded from SQLite on first use, or (for an + in-memory store, where it is the table itself) created empty. Locked.""" if self._cache is not None: return self._cache + if not self._sql: + self._cache = self._new_cache(self._dimension) + return self._cache + cursor = self._get_conn().cursor() cursor.row_factory = None # plain tuples: no Row overhead per field query = ("SELECT node_type, node_id, node_id_type, vector, text " @@ -1643,8 +2129,44 @@ def _ensure_cache(self) -> _ScanCache: self._cache = cache return cache + def _new_cache(self, width: Optional[int], capacity: int = 0) -> _ScanCache: + """An empty cache: numpy form when numpy is importable and the vector + width is known, otherwise the list form (which adopts numpy at its first + row, see :meth:`_cache_put`).""" + if _np is not None and width: + capacity = max(1024, capacity) + cache = _ScanCache([], None, [], None, + _np.empty((capacity, width), dtype=_np.float32), + _np.zeros(capacity, dtype=_np.float32)) + cache.alive = _np.zeros(capacity, dtype=bool) + cache.codes = _np.zeros(capacity, dtype=_np.int32) + cache.code_of = {} + else: + cache = _ScanCache([], [], [], [], None, None) + cache.pnorms = [] + cache.pos = {} + if not self._sql: + cache.models = [] + return cache + + def _cache_reserve(self, extra: int) -> None: + """Room for ``extra`` more rows in one allocation (a bulk write knows + its size, so the matrix grows once instead of step by step).""" + if extra <= 0 or _np is None: + return + cache = self._cache + if cache is None or (cache.matrix is None and cache.n == 0): + if not self._sql and self._dimension: + self._cache = self._new_cache(self._dimension, extra) + return + if cache.matrix is None: + return + need = cache.n + extra + if need > cache.matrix.shape[0]: + self._cache_grow(cache, need) + def _build_numpy_cache(self, cursor, total: int) -> Optional[_ScanCache]: - """Stream the table into a spare-capacity float32 matrix. + """Stream the table into a float32 matrix with a little spare room. Returns None when the stored vectors are not all one length (the caller then falls back to the list form, which tolerates that). @@ -1656,7 +2178,7 @@ def _build_numpy_cache(self, cursor, total: int) -> Optional[_ScanCache]: if width == 0 or width % 4: return None dim = width // 4 - capacity = total + total // 8 + 16 + capacity = total + total // 32 + 16 matrix = _np.empty((capacity, dim), dtype=_np.float32) norms = _np.zeros(capacity, dtype=_np.float32) codes = _np.zeros(capacity, dtype=_np.int32) @@ -1681,6 +2203,7 @@ def _build_numpy_cache(self, cursor, total: int) -> Optional[_ScanCache]: if t not in code_of: code_of[t] = len(code_of) codes[filled:filled + m] = [code_of[t] for t in types] + types = [sys.intern(t) for t in types] # one str per type, not one per row keys = list(zip(types, ids)) if 'int' in id_types: refs.extend([(t, self._restore_node_id(i, k)) @@ -1701,15 +2224,23 @@ def _build_numpy_cache(self, cursor, total: int) -> Optional[_ScanCache]: def _build_list_cache(self, rows) -> _ScanCache: """Pure-Python form: no numpy, or vectors of mixed length.""" refs = [(r[0], self._restore_node_id(r[1], r[2])) for r in rows] - return _ScanCache( - refs, [r[0] for r in rows], [r[4] for r in rows], - [self._bytes_to_vector(r[3]) for r in rows], None, None) + vectors = [] + for r in rows: + vec = array('f') + vec.frombytes(r[3]) + vectors.append(vec) + cache = _ScanCache( + refs, [r[0] for r in rows], [r[4] for r in rows], vectors, None, None) + cache.pnorms = [sum(b * b for b in vec) ** 0.5 for vec in vectors] + cache.pos = {(r[0], str(r[1])): j for j, r in enumerate(refs)} + return cache @staticmethod - def _cache_grow(cache: _ScanCache) -> None: - """Make room for more rows: grow capacity by a quarter (min 1024).""" + def _cache_grow(cache: _ScanCache, at_least: int = 0) -> None: + """Make room for more rows: grow capacity by a quarter (min 1024), + or to ``at_least`` rows when a bulk write asked for more.""" capacity, dim = cache.matrix.shape - new_capacity = capacity + max(1024, capacity // 4) + new_capacity = max(capacity + max(1024, capacity // 4), at_least) n = cache.n matrix = _np.empty((new_capacity, dim), dtype=_np.float32) matrix[:n] = cache.matrix[:n] @@ -1721,21 +2252,61 @@ def _cache_grow(cache: _ScanCache) -> None: alive[:n] = cache.alive[:n] cache.matrix, cache.norms, cache.codes, cache.alive = matrix, norms, codes, alive - def _cache_put(self, node_ref: NodeRef, id_type: str, blob: bytes, text: str) -> None: - """Mirror one upserted row into the cache. Caller holds the lock.""" + def _cache_put(self, node_ref: NodeRef, id_type: str, blob: bytes, text: str, + model: Optional[str] = None) -> Optional[str]: + """Mirror one upserted row into the cache. Caller holds the lock. + + For an in-memory store this *is* the write, and the return value is the + model the row carried before (None if it is new). + """ cache = self._cache - if cache is None: - return - matrix = cache.matrix - if matrix is None or len(blob) != matrix.shape[1] * 4: + table = not self._sql + width = len(blob) // 4 + if table: + if cache is None: + cache = self._cache = self._new_cache(width) + elif cache.matrix is None and cache.n == 0 and _np is not None: + cache = self._cache = self._new_cache(width) # first row: adopt numpy + elif cache.matrix is not None and width != cache.matrix.shape[1]: + raise DimensionMismatchError( + f"vector has {width} dimensions, store holds {cache.matrix.shape[1]}") + elif cache is None: + return None + elif cache.matrix is None or width != cache.matrix.shape[1]: self._cache = None # list form / new width: rebuild lazily - return - node_type = str(node_ref[0]) + return None + node_type = sys.intern(str(node_ref[0])) id_text = str(node_ref[1]) key = (node_type, id_text) ref = key if id_type != 'int' else (node_type, int(id_text)) - row = _np.frombuffer(blob, dtype=_np.float32) + if model is not None: + model = self._model_names.setdefault(model, model) # one str per model i = cache.pos.get(key) + old_model = None + if cache.matrix is None: # list form (in-memory only) + vec = array('f') + vec.frombytes(blob) + pnorm = sum(b * b for b in vec) ** 0.5 + if i is None: + i = cache.n + cache.n = i + 1 + cache.refs.append(ref) + cache.types.append(node_type) + cache.texts.append(text) + cache.vectors.append(vec) + cache.pnorms.append(pnorm) + cache.models.append(model) + cache.pos[key] = i + else: + old_model = cache.models[i] + cache.refs[i] = ref + cache.texts[i] = text + cache.vectors[i] = vec + cache.pnorms[i] = pnorm + cache.models[i] = model + return old_model + matrix = cache.matrix + row = _np.frombuffer(blob, dtype=_np.float32) if i is None: if cache.n >= matrix.shape[0]: self._cache_grow(cache) @@ -1744,6 +2315,8 @@ def _cache_put(self, node_ref: NodeRef, id_type: str, blob: bytes, text: str) -> cache.n = i + 1 cache.refs.append(ref) cache.texts.append(text) + if table: + cache.models.append(model) cache.pos[key] = i code = cache.code_of.get(node_type) if code is None: @@ -1751,43 +2324,78 @@ def _cache_put(self, node_ref: NodeRef, id_type: str, blob: bytes, text: str) -> cache.codes[i] = code cache.alive[i] = True else: # overwrite in place: keeps its position + if table: + old_model = cache.models[i] + cache.models[i] = model cache.refs[i] = ref cache.texts[i] = text matrix[i] = row cache.norms[i] = _np.linalg.norm(row.reshape(1, -1), axis=1)[0] - if cache.ivf is not None: - cache.ivf.note_put(i, matrix.shape[0]) + ivf = cache.ivf + if ivf is not None: + ivf.note_put(i, matrix.shape[0]) + if len(ivf.pending) >= _ANN_FOLD_BATCH: + ivf.fold(cache) # the write that fills the batch pays for the fold + return old_model + + def _cache_drop(self, node_ref: NodeRef) -> Optional[str]: + """Mirror one deleted row into the cache. Caller holds the lock. - def _cache_drop(self, node_ref: NodeRef) -> None: - """Mirror one deleted row into the cache. Caller holds the lock.""" + Returns the model the row carried (in-memory stores), else None. + """ cache = self._cache if cache is None: - return - if cache.matrix is None: - self._cache = None - return + return None i = cache.pos.pop((str(node_ref[0]), str(node_ref[1])), None) if i is None: - return - cache.matrix[i] = 0.0 - cache.norms[i] = 0.0 - cache.alive[i] = False + return None + old_model = cache.models[i] if cache.models is not None else None + if cache.matrix is not None: + cache.matrix[i] = 0.0 + cache.norms[i] = 0.0 + cache.alive[i] = False + else: + cache.vectors[i] = None + cache.types[i] = None + cache.pnorms[i] = 0.0 cache.refs[i] = None cache.texts[i] = None + if cache.models is not None: + cache.models[i] = None cache.dead += 1 if cache.ivf is not None: cache.ivf.note_drop(i) - while cache.n and not cache.alive[cache.n - 1]: # trailing tombstones + while cache.n and cache.refs[cache.n - 1] is None: # trailing tombstones cache.n -= 1 cache.refs.pop() cache.texts.pop() + if cache.matrix is None: + cache.vectors.pop() + cache.types.pop() + cache.pnorms.pop() + if cache.models is not None: + cache.models.pop() cache.dead -= 1 if cache.dead > 32 and cache.dead * 4 > cache.n: self._cache_compact(cache) + return old_model @staticmethod def _cache_compact(cache: _ScanCache) -> None: """Squeeze tombstones out, keeping the live rows in their order.""" + if cache.matrix is None: + keep = [i for i in range(cache.n) if cache.refs[i] is not None] + cache.refs = [cache.refs[i] for i in keep] + cache.texts = [cache.texts[i] for i in keep] + cache.vectors = [cache.vectors[i] for i in keep] + cache.types = [cache.types[i] for i in keep] + cache.pnorms = [cache.pnorms[i] for i in keep] + if cache.models is not None: + cache.models = [cache.models[i] for i in keep] + cache.pos = {(r[0], str(r[1])): j for j, r in enumerate(cache.refs)} + cache.n = len(keep) + cache.dead = 0 + return live = _np.flatnonzero(cache.alive[:cache.n]) step = 16384 for start in range(0, live.size, step): @@ -1805,6 +2413,8 @@ def _cache_compact(cache: _ScanCache) -> None: order = live.tolist() cache.refs = [cache.refs[i] for i in order] cache.texts = [cache.texts[i] for i in order] + if cache.models is not None: + cache.models = [cache.models[i] for i in order] cache.pos = {(r[0], str(r[1])): j for j, r in enumerate(cache.refs)} cache.n = kept cache.dead = 0 @@ -1867,6 +2477,12 @@ def _rank(scores, k: Optional[int]): def _np_search(self, cache, query_vector, top_k, node_type, threshold): if top_k is not None and top_k <= 0: return [] + if self._rescore: + if cache.n - cache.dead == 0: + return [] + q32, qnorm = self._query_parts(query_vector) + approx = self._np_scores(cache, query_vector) + return self._finish_exact(cache, q32, qnorm, approx, top_k, node_type, threshold) scores = self._np_candidates(cache, query_vector, node_type, threshold) if scores is None: return [] @@ -1875,6 +2491,200 @@ def _np_search(self, cache, query_vector, top_k, node_type, threshold): return [SimilarityResult(node_ref=refs[i], score=s, text=texts[i]) for i, s in zip(best.tolist(), scores[best].tolist())] + # -- canonical rescoring: one definition of a score, for one query or many -- + # + # BLAS does not give a row the same float32 result in every call: a + # matrix-vector product, a matrix-matrix product, the same product over a + # block or a gathered subset all sum a row's products in different orders + # (measured: a block of rows differs from the same rows inside the full + # product in the last bit for a fraction of a percent of them). So the + # bounded exact search does not return BLAS scores. It uses the product + # (one query: matrix @ vector; a batch: matrix @ matrices) only to find a + # candidate set that provably contains the true top k, then scores those + # candidates with a kernel whose result for a (row, query) pair depends on + # nothing else: float32 products, summed pairwise along the row. Single + # and batch searches finish through the same code, so a batch answer is + # bit-identical to the one-query answer. + + @staticmethod + def _query_parts(query_vector): + q32 = _np.asarray(query_vector, dtype=_np.float32) + return q32, float(_np.linalg.norm(q32)) + + @staticmethod + def _canonical_scores(cache, q32, qnorm, rows): + """Cosine of ``rows`` (row numbers) against ``q32``, order-independent. + + Same formula as :meth:`_np_scores` (zero denominator scores 0), but the + dot product is ``sum(row * q)`` over float32 products with numpy's + pairwise summation, which does not depend on which other rows are in + the call. Processed in blocks of about 8 MB. + """ + out = _np.zeros(rows.size, dtype=_np.float32) + if qnorm == 0.0 or rows.size == 0: + return out + step = max(64, (1 << 21) // cache.matrix.shape[1]) + with _np.errstate(divide='ignore', invalid='ignore', over='ignore'): + for start in range(0, rows.size, step): + sel = rows[start:start + step] + block = cache.matrix[sel] # a gathered copy + block *= q32 + dots = block.sum(axis=1) + denom = cache.norms[sel] * qnorm + _np.divide(dots, denom, out=out[start:start + step], where=denom > 0) + return out + + @staticmethod + def _score_margin(cache) -> float: + """Bound on ``|BLAS cosine - canonical cosine|`` for one row. + + Each dot product is off by at most ``gamma_d * |row| * |query|`` with + ``gamma_d = d * 2**-24`` whatever the summation order, so two kernels + differ by at most twice that in cosine units; padded for the float32 + norms and the division. + """ + return 2.1 * cache.matrix.shape[1] * 2.0 ** -24 + 5e-7 + + def _finish_exact(self, cache, q32, qnorm, approx, top_k, node_type, threshold, rows=None): + """Top ``top_k`` by canonical score, given BLAS scores to narrow the field. + + ``approx`` is a float32 array of cosines computed however was + convenient, one per used row of the cache, or (with ``rows``, an + ascending array of row numbers) one per listed row, -inf for a row that + was excluded already. Let ``e`` be :meth:`_score_margin` and ``A`` the + k-th best approximate score among the rows that pass the filters (with + the threshold relaxed by ``e``): every row of the true top k has an + approximate score of at least ``A - 2e``, so rescoring exactly the rows + above that line and ranking them gives the true top k whichever kernel + produced ``approx``. If fewer than k of them pass the exact threshold + the candidate set was too tight (a threshold close to the scores) and + every filtered row is rescored instead. + """ + n = approx.shape[0] + e = self._score_margin(cache) + # compared in float32 with the bound rounded down: a superset of the rows + # whose canonical score passes the (exact, float64) threshold below + floor = _np.nextafter(_np.float32(threshold - e), _np.float32(-_np.inf)) + valid = approx >= floor + if rows is None: + if cache.dead: + valid &= cache.alive[:n] + if node_type is not None: + code = cache.code_of.get(node_type) + if code is None: + return [] + valid &= cache.codes[:n] == code + else: + valid &= approx > -_np.inf + passing = _np.flatnonzero(valid) + nv = passing.size + if nv == 0: + return [] + k = nv if (top_k is None or top_k >= nv) else top_k + if k == nv: + cand = passing + else: + # partition only the rows that pass: a partition over the whole array, + # with the rejected rows set to -inf, is several times slower + values = approx[passing] + kth = float(_np.partition(values, nv - k)[nv - k]) + cutoff = _np.nextafter(_np.float32(kth - 2 * e), _np.float32(-_np.inf)) + cand = passing[values >= cutoff] + cand_rows = cand if rows is None else rows[cand] + scores = self._canonical_scores(cache, q32, qnorm, cand_rows) + ok = scores.astype(_np.float64) >= threshold + if k != nv and int(_np.count_nonzero(ok)) < k: + cand_rows = passing if rows is None else rows[passing] # threshold hugs the scores + scores = self._canonical_scores(cache, q32, qnorm, cand_rows) + ok = scores.astype(_np.float64) >= threshold + cand_rows = cand_rows[ok] + scores = scores[ok] + if cand_rows.size == 0: + return [] + order = _np.argsort(-scores, kind='stable') # ties keep row order + if top_k is not None: + order = order[:top_k] + refs, texts = cache.refs, cache.texts + return [SimilarityResult(node_ref=refs[i], score=sc, text=texts[i]) + for i, sc in zip(cand_rows[order].tolist(), scores[order].tolist())] + + #: Rows of float32 scores one matrix-matrix product may produce at a time. + _BATCH_SCORE_BYTES = 1 << 26 + + def similarity_search_batch( + self, + queries: Sequence[Union[str, Sequence[float]]], + top_k: Optional[int] = 10, + node_type: Optional[str] = None, + threshold: float = 0.0, + exact: bool = False + ) -> List[List[SimilarityResult]]: + """ + Answer many queries at once; entry ``i`` of the result is exactly what + ``similarity_search(queries[i], top_k, node_type, threshold, exact)`` + returns (same nodes, same order, bit-identical scores). + + With numpy, a bounded ``top_k`` and an exact scan (no ``ann`` index in + play, no sqlite-vec) the queries go through the stored matrix together: + one matrix-matrix product per group of queries instead of one + matrix-vector product each, which reads the matrix once per group and + is several times faster per query on a big store. The product only + picks candidates; the returned scores come from the canonical kernel + (see :meth:`_finish_exact`), the same one a single search finishes + with. Every other configuration answers the queries one after the + other, which is the same answer by definition. + + Text queries are encoded one at a time with ``encoder.encode`` (not + ``encode_batch``) so that each gets the very vector a single search + would give it. + + Args: + queries: Texts and/or vectors + top_k: Results per query (None returns every match; that and + ``top_k <= 0`` take the one-at-a-time path) + node_type: Filter by node type (optional) + threshold: Minimum similarity score + exact: Skip the approximate index (as in :meth:`similarity_search`) + + Returns: + One list of SimilarityResult per query, in query order + """ + items = list(queries) + if not items: + return [] + with self._lock: + vectors = [self._resolve_query(q) for q in items] + batched = ( + self._rescore and _np is not None and top_k is not None and top_k > 0 + and not (self._sqlite_vec) + and (self._ann is None or exact) + ) + cache = self._ensure_cache() if batched else None + if cache is None or cache.matrix is None: + return [self.similarity_search(v, top_k, node_type, threshold, exact) + for v in vectors] + if self._ann is not None: + self._ann_counts["exact"] += len(vectors) + if cache.n - cache.dead == 0: + return [[] for _ in vectors] + n = cache.n + group = max(1, min(64, self._BATCH_SCORE_BYTES // (4 * n))) + matrix_t = cache.matrix[:n].T + results: List[List[SimilarityResult]] = [] + for start in range(0, len(vectors), group): + parts = [self._query_parts(v) for v in vectors[start:start + group]] + block = _np.stack([q32 for q32, _ in parts]) + dots = block @ matrix_t # (group, n) + for row, (q32, qnorm) in zip(dots, parts): + approx = _np.zeros(n, dtype=_np.float32) + if qnorm != 0.0: + denom = cache.norms[:n] * qnorm + with _np.errstate(divide='ignore', invalid='ignore'): + _np.divide(row, denom, out=approx, where=denom > 0) + results.append(self._finish_exact( + cache, q32, qnorm, approx, top_k, node_type, threshold)) + return results + def _score_all( self, query_vector: Sequence[float], @@ -1901,10 +2711,18 @@ def _score_all( for i, ref in enumerate(cache.refs) if ref is not None and (code is None or codes[i] == code)] - scores = [self._cosine_similarity(query_vector, v) for v in cache.vectors] - return [(ref, float(scores[i]), cache.texts[i]) - for i, ref in enumerate(cache.refs) - if node_type is None or cache.types[i] == node_type] + query_norm = sum(a * a for a in query_vector) ** 0.5 + out = [] + for i, ref in enumerate(cache.refs): + if ref is None or (node_type is not None and cache.types[i] != node_type): + continue + norm = cache.pnorms[i] + if query_norm == 0 or norm == 0: + score = 0.0 + else: + score = sum(a * b for a, b in zip(query_vector, cache.vectors[i])) / (query_norm * norm) + out.append((ref, float(score), cache.texts[i])) + return out def _ranked( self, @@ -1973,9 +2791,9 @@ def _ann_index(self, cache) -> Optional[_IVFIndex]: Policy: * built lazily by the first search (never persisted), k-means on a sample of up to 40 x nlist live vectors, then every row assigned; - * new and updated rows are scored exactly until at least - ``max(512, 2%)`` of the indexed rows are pending, then folded - into their nearest cell; + * new and updated rows are scored exactly until ``_ANN_FOLD_BATCH`` + (256) of them are pending, then the write that reaches that number + folds them into their nearest cell (so no search waits for a fold); * deletes tombstone (nothing to do but skip them); * once writes + deletes since training exceed 50% of the trained size, the index is retrained from the current data - with an @@ -1996,7 +2814,7 @@ def _ann_index(self, cache) -> Optional[_IVFIndex]: nlist = self._ann_nlist or self._ann_auto_nlist(live) ivf = cache.ivf = _IVFIndex.build(cache, nlist, self._ann_seed) self._ann_counts["builds"] += 1 - elif len(ivf.pending) >= max(_ANN_FOLD_MIN, _ANN_FOLD_FRACTION * ivf.trained_n): + elif len(ivf.pending) >= _ANN_FOLD_BATCH: ivf.fold(cache) return ivf @@ -2084,18 +2902,30 @@ def _ann_search(self, cache, query_vector, top_k, node_type, threshold): lists = self._ann_lists(ivf, scale) if lists is None: return None - got = self._ann_scores(cache, ivf, query_vector, node_type, threshold, lists) + relax = self._score_margin(cache) if self._rescore else 0.0 + got = self._ann_scores(cache, ivf, query_vector, node_type, threshold - relax, lists) if got is None: return None rows, scores = got - if int(_np.count_nonzero(scores > -_np.inf)) >= top_k: - best = self._rank(scores, top_k) - refs, texts = cache.refs, cache.texts + if self._rescore: + # the probed rows are ranked with the exact search's kernel, so a + # returned score is the very number an exact search gives that row + q32, qnorm = self._query_parts(query_vector) + found = (self._finish_exact(cache, q32, qnorm, scores, top_k, None, threshold, rows) + if rows.size else []) + enough = len(found) >= top_k + else: + enough = int(_np.count_nonzero(scores > -_np.inf)) >= top_k + if enough: + best = self._rank(scores, top_k) + refs, texts = cache.refs, cache.texts + found = [SimilarityResult(node_ref=refs[r], score=sc, text=texts[r]) + for r, sc in zip(rows[best].tolist(), scores[best].tolist())] + if enough: self._ann_counts["approximate"] += 1 if scale > 1: self._ann_counts["widened"] += 1 - return [SimilarityResult(node_ref=refs[r], score=sc, text=texts[r]) - for r, sc in zip(rows[best].tolist(), scores[best].tolist())] + return found scale *= 4 def _ranked_ann(self, query, node_type, threshold, hint, scale): @@ -2407,19 +3237,22 @@ def export_embeddings(self, graph_digest: Optional[str] = None) -> Dict[str, Any ``graph_digest_basis``. Absent values are omitted, not written as null. """ with self._lock: - rows = self._get_conn().execute(""" - SELECT node_type, node_id, node_id_type, vector, text, model - FROM embeddings ORDER BY rowid - """).fetchall() - - embeddings = [{ - "type": row['node_type'], - "id": self._restore_node_id(row['node_id'], row['node_id_type']), - "id_type": row['node_id_type'], - "text": row['text'], - "model": row['model'], - "vector": self._bytes_to_vector(row['vector']), - } for row in rows] + if not self._sql: + embeddings = self._table_embeddings() + else: + rows = self._get_conn().execute(""" + SELECT node_type, node_id, node_id_type, vector, text, model + FROM embeddings ORDER BY rowid + """).fetchall() + + embeddings = [{ + "type": row['node_type'], + "id": self._restore_node_id(row['node_id'], row['node_id_type']), + "id_type": row['node_id_type'], + "text": row['text'], + "model": row['model'], + "vector": self._bytes_to_vector(row['vector']), + } for row in rows] payload: Dict[str, Any] = { "format": EMBEDDING_FORMAT, @@ -2438,6 +3271,33 @@ def export_embeddings(self, graph_digest: Optional[str] = None) -> Dict[str, Any payload["graph_digest_basis"] = GRAPH_DIGEST_BASIS return payload + def _table_embeddings(self) -> List[Dict[str, Any]]: + """The live rows of an in-memory store as sidecar entries, in row order.""" + cache = self._cache + out: List[Dict[str, Any]] = [] + if cache is None: + return out + step = 4096 + for start in range(0, cache.n, step): + stop = min(cache.n, start + step) + if cache.matrix is not None: + vectors = cache.matrix[start:stop].tolist() + else: + vectors = [None if v is None else v.tolist() for v in cache.vectors[start:stop]] + for i, vector in zip(range(start, stop), vectors): + ref = cache.refs[i] + if ref is None: + continue + out.append({ + "type": ref[0], + "id": ref[1], + "id_type": 'int' if isinstance(ref[1], int) else 'str', + "text": cache.texts[i], + "model": cache.models[i], + "vector": vector, + }) + return out + def import_embeddings(self, payload: Dict[str, Any], replace: bool = False) -> int: """ Load embeddings produced by :meth:`export_embeddings`. @@ -2473,9 +3333,10 @@ def import_embeddings(self, payload: Dict[str, Any], replace: bool = False) -> i self._dimension = payload.get("dimension") written = 0 - conn = self._get_conn() + conn = self._get_conn() if self._sql else None + items = payload.get("embeddings", []) try: - for item in payload.get("embeddings", []): + for item in items: node_id = item["id"] if item.get("id_type") == 'int' and not isinstance(node_id, int): node_id = int(node_id) @@ -2485,10 +3346,14 @@ def import_embeddings(self, payload: Dict[str, Any], replace: bool = False) -> i self._write_row( conn, ref, vector, item.get("text", ""), item.get("model", "unknown"), - node_id_type=item.get("id_type") or self._node_id_type(node_id) + node_id_type=item.get("id_type") or self._node_id_type(node_id), + want_id=False ) written += 1 - conn.commit() + if written == 1: + self._cache_reserve(len(items) - 1) + if conn is not None: + conn.commit() except BaseException: self._invalidate() raise @@ -2515,6 +3380,328 @@ def load_embeddings(self, path: Union[str, FilePath], replace: bool = False) -> payload = json.loads(FilePath(path).read_text(encoding='utf-8')) return self.import_embeddings(payload, replace=replace) + # -- binary snapshot ------------------------------------------------ + + def _snapshot_models(self) -> Tuple[int, List[str]]: + """``(live row count, model names in order of first use)``. Locked.""" + order: Dict[str, int] = {} + count = 0 + if not self._sql: + cache = self._cache + if cache is not None: + for ref, model in zip(cache.refs, cache.models): + if ref is not None: + count += 1 + if model not in order: + order[model] = len(order) + else: + for (model,) in self._get_conn().execute( + "SELECT model FROM embeddings ORDER BY rowid"): + count += 1 + if model not in order: + order[model] = len(order) + return count, list(order) + + def _snapshot_blocks(self, rows_per_block: int): + """Live rows in table order as ``(metadata, vector bytes)`` blocks. + + ``metadata`` is a list of ``(type, id text, text, model, is_int_id)``; + the vector bytes are little-endian IEEE-754 float32, row after row. + Locked by the caller. + """ + if not self._sql: + cache = self._cache + if cache is None: + return + live = [i for i in range(cache.n) if cache.refs[i] is not None] + for start in range(0, len(live), rows_per_block): + chunk = live[start:start + rows_per_block] + meta = [(cache.refs[i][0], str(cache.refs[i][1]), cache.texts[i], + cache.models[i], isinstance(cache.refs[i][1], int)) for i in chunk] + if cache.matrix is not None: + data = _np.ascontiguousarray(cache.matrix[chunk], dtype=' SnapshotInfo: + """ + Write a binary snapshot of every live embedding to ``path``, atomically. + + The snapshot is a cache of the JSON sidecar (the portable, authoritative + format): the same rows in the same order, as little-endian binary, so a + load is a read and a copy instead of a parse. The file is written to a + temp file, flushed and renamed over ``path``; a failure leaves the + previous file untouched. :meth:`SemanticGraph.save` calls this with the + graph digest and the sidecar it just wrote. + + Args: + path: Snapshot file + graph_digest: :func:`graph_digest` of the graph this belongs to + sidecar: The JSON sidecar written for the same save; its length and + fingerprint are recorded, and a load refuses the snapshot + once that file has changed + int_ids: How a loader will tell integer ids from string ids (a + snapshot stores every id as text): a container of + ``(type, id text)`` pairs that are integers, or a + ``callable(type, id_text) -> bool``. When given, a row + whose id kind that rule would get wrong makes the call + raise :class:`SnapshotError` instead of writing a snapshot + that loads differently from the JSON sidecar. + + Returns: + The header that was written (and the file size) + """ + def is_int(ty: str, ident: str) -> bool: + if int_ids is None: + return False + return bool(int_ids(ty, ident)) if callable(int_ids) else (ty, ident) in int_ids + + with self._lock: + count, models = self._snapshot_models() + dim = self._dimension or 0 + identity = sidecar_identity(sidecar) if sidecar is not None else None + basis = GRAPH_DIGEST_BASIS if graph_digest is not None else None + encoder_model = self._current_model() + encoder_version = str(self.version) if self.version is not None else None + head = bytearray(_SNAP_MAGIC) + head += _U32.pack(SNAPSHOT_VERSION) + head += _U32.pack(_SNAP_FLAG_SIDECAR if identity is not None else 0) + head += _U32.pack(dim) + head += _U32.pack(_SNAP_BLOCK_ROWS) + head += _U64.pack(count) + head += _snap_opt(graph_digest) + _snap_opt(basis) + head += _snap_opt(encoder_model) + _snap_opt(encoder_version) + if identity is not None: + head += _U64.pack(identity[0]) + identity[1] + head += _U32.pack(len(models)) + for model in models: + head += _snap_str(model) + model_ix = {m: i for i, m in enumerate(models)} + + def fill(handle: Any) -> int: + digest = hashlib.sha256() + written = 0 + + def emit(data: bytes) -> None: + nonlocal written + handle.write(data) + digest.update(data) + written += len(data) + + emit(bytes(head)) + rows = 0 + for meta, data in self._snapshot_blocks(_SNAP_BLOCK_ROWS): + parts = [] + for ty, ident, text, model, row_is_int in meta: + if int_ids is not None and row_is_int != is_int(ty, ident): + raise SnapshotError( + f"the id {ident!r} of a {ty!r} embedding is " + f"{'an integer' if row_is_int else 'a string'}, which a " + f"snapshot (it stores ids as text) cannot say") + parts.append(_snap_str(ty) + _snap_str(ident) + _snap_str(text) + + _U32.pack(model_ix[model])) + emit(b"".join(parts)) + emit(data) + rows += len(meta) + if rows != count: # pragma: no cover + raise SnapshotError("the store changed while it was written") + handle.write(digest.digest()) + return written + _SNAP_TRAILER + + size = _atomic_write_stream(path, fill) + return SnapshotInfo( + version=SNAPSHOT_VERSION, dimension=dim, count=count, + graph_digest=graph_digest, graph_digest_basis=basis, + encoder_model=encoder_model, encoder_version=encoder_version, + sidecar_len=identity[0] if identity else None, + sidecar_fingerprint=identity[1] if identity else None, bytes=size) + + def load_snapshot( + self, + path: Union[str, FilePath], + replace: bool = False, + int_ids: Optional[Any] = None + ) -> SnapshotInfo: + """ + Read a snapshot written by :meth:`save_snapshot` (or by another port). + + The whole file is read and verified (structure, every length, repeated + keys, the trailing SHA-256) before the store is touched; any problem + raises :class:`SnapshotError` and leaves the store as it was. Does not + check which graph or sidecar the snapshot belongs to: use + :meth:`SemanticGraph.load` for that. + + A snapshot stores ids as text, so ``int_ids`` says which of them are + integers (a container of ``(type, id text)`` pairs or a + ``callable(type, id_text) -> bool``); by default none are. + + Args: + path: Snapshot file + replace: Drop the existing embeddings first (otherwise the + snapshot's rows are added, overwriting equal keys, as + :meth:`load_embeddings` does) + int_ids: See above + + Returns: + The snapshot's header + """ + with self._lock: + stage = _read_snapshot(path) + self._adopt_staged(stage, int_ids, replace) + return stage.info + + def _adopt_snapshot_for(self, path, sidecar, graph_digest_value, int_ids) -> SnapshotInfo: + """Use the snapshot at ``path`` for the graph whose digest is + ``graph_digest_value`` and whose JSON sidecar is ``sidecar``, if it + provably belongs to them. :class:`SnapshotError` (its message is the + reason) leaves the store untouched.""" + try: + current = sidecar_identity(sidecar) + except (OSError, SnapshotError) as exc: + raise SnapshotError(f"cannot read the sidecar {sidecar}: {exc}") from exc + + def accept(info: SnapshotInfo) -> None: + if (info.graph_digest != graph_digest_value + or info.graph_digest_basis != GRAPH_DIGEST_BASIS): + raise SnapshotError("it was saved for a different graph (digest differs)") + if info.sidecar_len is None: + raise SnapshotError("it records no sidecar identity") + if (info.sidecar_len, info.sidecar_fingerprint) != current: + raise SnapshotError( + "the JSON sidecar changed since the snapshot was written (stale snapshot)") + + with self._lock: + stage = _read_snapshot(path, accept) + self._adopt_staged(stage, int_ids, False) + return stage.info + + def _adopt_staged(self, stage: _StagedSnapshot, int_ids: Optional[Any], replace: bool) -> None: + """Install a verified snapshot's rows. Locked.""" + info = stage.info + if info.count and not replace and self._dimension not in (None, info.dimension): + raise SnapshotError( + f"the snapshot holds {info.dimension}-dimensional vectors, " + f"the store {self._dimension}") + + def is_int(ty: str, ident: str) -> bool: + if int_ids is None: + return False + return bool(int_ids(ty, ident)) if callable(int_ids) else (ty, ident) in int_ids + + if replace: + self.clear() + self._dimension = info.dimension or None + meta = {key: value for key, value in ( + ("encoder_model", info.encoder_model), ("encoder_version", info.encoder_version), + ("graph_digest", info.graph_digest), ("graph_digest_basis", info.graph_digest_basis)) + if value is not None} + count = info.count + if not self._sql and self.count() == 0 and count: + self._install_table(stage, is_int) + elif count: + conn = self._get_conn() if self._sql else None + try: + for k in range(count): + if stage.matrix is not None: + row = stage.matrix[k] + blob = row.tobytes() + else: + row = stage.vectors[k] + blob = row.tobytes() + self._check_dimension(row, "snapshot vector") + ident = stage.ids[k] + id_type = 'int' if is_int(stage.types[k], ident) else 'str' + ref = (stage.types[k], int(ident) if id_type == 'int' else ident) + self._write_row(conn, ref, row, stage.texts[k], + stage.models[stage.model_ix[k]], node_id_type=id_type, + blob=blob, want_id=False) + if conn is not None: + conn.commit() + except BaseException: + self._invalidate() + raise + self.last_import_meta = meta + + def _install_table(self, stage: _StagedSnapshot, is_int: Callable[[str, str], bool]) -> None: + """Move a staged snapshot into an empty in-memory store without copying + the vectors again. Locked.""" + info = stage.info + count, dim = info.count, info.dimension + types, ids = stage.types, stage.ids + names = [self._model_names.setdefault(m, m) for m in stage.models] + models = [names[i] for i in stage.model_ix] + refs = [(t, int(i)) if is_int(t, i) else (t, i) for t, i in zip(types, ids)] + if stage.matrix is not None: + capacity = stage.matrix.shape[0] + norms = _np.zeros(capacity, dtype=_np.float32) + step = 16384 + for start in range(0, count, step): + stop = min(count, start + step) + norms[start:stop] = _np.linalg.norm(stage.matrix[start:stop], axis=1) + code_of: Dict[str, int] = {} + for t in types: + if t not in code_of: + code_of[t] = len(code_of) + codes = _np.zeros(capacity, dtype=_np.int32) + codes[:count] = [code_of[t] for t in types] + alive = _np.zeros(capacity, dtype=bool) + alive[:count] = True + cache = _ScanCache(refs, None, list(stage.texts), None, stage.matrix, norms) + cache.alive, cache.codes, cache.code_of = alive, codes, code_of + else: + vectors = stage.vectors + cache = _ScanCache(refs, list(types), list(stage.texts), vectors, None, None) + cache.pnorms = [sum(b * b for b in vec) ** 0.5 for vec in vectors] + cache.pos = {(t, i): k for k, (t, i) in enumerate(zip(types, ids))} + cache.models = models + counts: Dict[str, int] = {} + for m in models: + counts[m] = counts.get(m, 0) + 1 + self._cache = cache + self._models_cache = counts + self._dimension = dim + # -- lifecycle ------------------------------------------------------ def close(self) -> None: @@ -2523,6 +3710,8 @@ def close(self) -> None: if self._conn: self._conn.close() self._conn = None + self._cache = None # an in-memory store ends with its rows + self._models_cache = None if self._sql else {} self._invalidate() def __enter__(self) -> 'EmbeddingStore': @@ -2643,6 +3832,9 @@ def __init__( self.auto_embed = auto_embed self.embed_fields = embed_fields or ['name', 'description', 'content', 'text'] + #: How the embeddings of the last :meth:`load` were restored (None for + #: a graph that was not loaded); see :class:`LoadReport`. + self.load_report: Optional[LoadReport] = None # -- encoder -------------------------------------------------------- @@ -3600,7 +4792,8 @@ def save( self, path: Union[str, FilePath], format: str = "auto", - embeddings: bool = True + embeddings: bool = True, + snapshot: bool = False ) -> None: """ Save graph to file, and its embeddings alongside it. @@ -3622,10 +4815,26 @@ def save( With ``embeddings=False`` an existing sidecar is left in place (and will fail the digest check on load, as it no longer matches). + With ``snapshot=True`` a binary snapshot of the embeddings is written as + well, at ``.embeddings.snap`` (after the sidecar, atomically). The + JSON sidecar stays the portable, authoritative copy and is byte-for-byte + what it would be without the snapshot; :meth:`load` takes the snapshot + when it provably belongs to the graph and the sidecar on disk (and its + checksum verifies) and reads the JSON otherwise. It loads several times + faster and is a third of the size. A save *without* ``snapshot`` deletes + any existing snapshot first, since the sidecar is about to change. If the + snapshot cannot be written (``SnapshotError``, ``OSError``) the graph and + sidecar are saved anyway, a stale snapshot is removed, and the error is + raised. A snapshot stores ids as text, so it is written only when every + embedding's id is an integer exactly when the graph has an integer node of + that type and number (the rule :meth:`load` uses to restore them); + otherwise ``SnapshotError``. + Args: path: Graph file path format: File format embeddings: Write the sidecar (set False for graph-only saves) + snapshot: Also write the binary snapshot (needs ``embeddings``) """ path = FilePath(path) if format == "auto": @@ -3637,8 +4846,34 @@ def save( if embeddings: reread = (ISONGraph.from_isonl(text, path.stem) if format == "isonl" else ISONGraph.from_ison(text, path.stem)) - self.embedding_store.save_embeddings( - self.embeddings_path_for(path), graph_digest=graph_digest(reread)) + digest = graph_digest(reread) + sidecar = self.embeddings_path_for(path) + snap = snapshot_path_for(path) + if not snapshot: + # the sidecar is about to change: a snapshot of the old one is stale + try: + snap.unlink() + except OSError: + pass + self.embedding_store.save_embeddings(sidecar, graph_digest=digest) + if snapshot: + try: + self.embedding_store.save_snapshot( + snap, graph_digest=digest, sidecar=sidecar, + int_ids=self._int_id_texts(reread)) + except BaseException: + try: + snap.unlink() + except OSError: + pass + raise + + @staticmethod + def _int_id_texts(graph: ISONGraph) -> set: + """``(type, id text)`` of every integer-id node: how a snapshot's text + ids are turned back into the integers they were.""" + return {(n.ref[0], str(n.ref[1])) for n in graph.nodes() + if isinstance(n.ref[1], int) and not isinstance(n.ref[1], bool)} @classmethod def load( @@ -3698,6 +4933,16 @@ def load( never saved; it is rebuilt from the loaded vectors on the first search. + A binary snapshot beside the sidecar (``.embeddings.snap``, written + by ``save(..., snapshot=True)`` or by the Rust crate) is used instead of + the sidecar when its header names this graph's digest and the sidecar + file as it is on disk, and the whole file verifies; the store is then + the same as a JSON load would give. Anything else wrong with it (stale, + truncated, corrupt, another version, a repeated key) is ignored: the + JSON sidecar is read exactly as without a snapshot. ``load_report`` on + the result (or :meth:`load_with_report`) says which path ran and why a + snapshot that exists was not used. + Returns: SemanticGraph instance @@ -3706,6 +4951,7 @@ def load( ``on_mismatch="error"`` ValueError: ``on_mismatch`` is not one of the three modes """ + started = time.perf_counter() if on_mismatch not in ("error", "warn", "reembed"): raise ValueError( f"on_mismatch must be 'error', 'warn' or 'reembed', got {on_mismatch!r}") @@ -3753,9 +4999,25 @@ def load( graph.add_edge(edge.rel_type, edge.source, edge.target, **edge.properties) + source = "none" + skipped: Optional[str] = None + restored = 0 if embeddings: sidecar = cls.embeddings_path_for(path) if sidecar.is_file(): + source = "json" + snap = snapshot_path_for(path) + if snap.is_file(): + try: + info = graph.embedding_store._adopt_snapshot_for( + snap, sidecar, graph_digest(base), cls._int_id_texts(base)) + except Exception as exc: # a snapshot is a cache: any failure falls back + skipped = str(exc) if isinstance(exc, SnapshotError) else f"{type(exc).__name__}: {exc}" + else: + cls._warn_version_skew(graph) + graph.load_report = LoadReport( + "snapshot", None, info.count, time.perf_counter() - started) + return graph try: payload = json.loads(sidecar.read_text(encoding='utf-8')) except ValueError as exc: @@ -3787,17 +5049,29 @@ def load( if problem is not None: warnings.warn(problem, UserWarning, stacklevel=2) graph.embedding_store.import_embeddings(payload) - stored_version = graph.embedding_store.last_import_meta.get("encoder_version") - current_version = graph.embedding_store.version - if (stored_version is not None and current_version is not None - and str(stored_version) != str(current_version)): - warnings.warn( - f"embeddings were made by encoder version {stored_version!r}, " - f"but the encoder in use is version {current_version!r}", - UserWarning, stacklevel=2) + cls._warn_version_skew(graph) + restored = graph.embedding_store.count() + graph.load_report = LoadReport(source, skipped, restored, time.perf_counter() - started) return graph + @staticmethod + def _warn_version_skew(graph: 'SemanticGraph') -> None: + stored_version = graph.embedding_store.last_import_meta.get("encoder_version") + current_version = graph.embedding_store.version + if (stored_version is not None and current_version is not None + and str(stored_version) != str(current_version)): + warnings.warn( + f"embeddings were made by encoder version {stored_version!r}, " + f"but the encoder in use is version {current_version!r}", + UserWarning, stacklevel=3) + + @classmethod + def load_with_report(cls, path: Union[str, FilePath], **kwargs: Any) -> Tuple['SemanticGraph', LoadReport]: + """:meth:`load` (same arguments), also returning the :class:`LoadReport`.""" + graph = cls.load(path, **kwargs) + return graph, graph.load_report + def close(self) -> None: """Release the embedding database connection.""" self.embedding_store.close() @@ -3825,7 +5099,11 @@ def __repr__(self) -> str: 'EMBEDDING_SIDECAR_SUFFIX', 'GRAPH_DIGEST_BASIS', 'UNKNOWN_MODELS', + 'SNAPSHOT_SUFFIX', + 'SNAPSHOT_VERSION', 'graph_digest', + 'snapshot_path_for', + 'sidecar_identity', # Errors 'EmbeddingError', @@ -3833,6 +5111,7 @@ def __repr__(self) -> str: 'NoEncoderError', 'ModelMismatchError', 'GraphEmbeddingMismatchError', + 'SnapshotError', # Encoders 'EmbeddingEncoder', @@ -3843,6 +5122,8 @@ def __repr__(self) -> str: 'EmbeddingStore', 'EmbeddingRecord', 'SimilarityResult', + 'LoadReport', + 'SnapshotInfo', # Graph 'SemanticGraph', diff --git a/tests/test_isongraph_vector.py b/tests/test_isongraph_vector.py index bf88fd4..f67d320 100644 --- a/tests/test_isongraph_vector.py +++ b/tests/test_isongraph_vector.py @@ -1952,8 +1952,15 @@ def _assert_cache_matches_rebuild(store): """The patched cache must equal one built from scratch from the table.""" np = pytest.importorskip('numpy') patched = _cache_snapshot(store) - store._cache = None # force a rebuild from SQLite - fresh = _cache_snapshot(store) + if store._sql: + store._cache = None # force a rebuild from SQLite + fresh = _cache_snapshot(store) + else: + # an in-memory store has no table behind its cache: rebuild a second + # store from the export instead (every row written once, in order) + twin = EmbeddingStore(db_path=':memory:') + twin.import_embeddings(store.export_embeddings()) + fresh = _cache_snapshot(twin) assert patched['refs'] == fresh['refs'] assert patched['texts'] == fresh['texts'] assert np.array_equal(patched['matrix'], fresh['matrix']) @@ -2205,10 +2212,8 @@ def test_failed_write_does_not_leave_a_stale_cache(self): refs_now = {r.node_ref for r in store.similarity_search( [1, 0, 0], top_k=None, threshold=-1.0)} # whatever the connection kept of the half-done batch, the cache must agree - rows = store._get_conn().execute( - "SELECT node_type, node_id FROM embeddings").fetchall() - assert refs_now == {(r['node_type'], int(r['node_id']) if r['node_id'].isdigit() - else r['node_id']) for r in rows} + rows = store.export_embeddings()['embeddings'] + assert refs_now == {(r['type'], r['id']) for r in rows} _assert_cache_matches_rebuild(store) store.close() @@ -2303,13 +2308,10 @@ def reader(tid): assert cache is not None and cache.n - cache.dead == store.count() _assert_cache_matches_rebuild(store) # and the final state agrees with an independent brute-force read of the table - rows = store._get_conn().execute( - "SELECT node_type, node_id, node_id_type, vector, text FROM embeddings ORDER BY rowid" - ).fetchall() + rows = store.export_embeddings()['embeddings'] model = _RefModel() for row in rows: - ref = (row['node_type'], store._restore_node_id(row['node_id'], row['node_id_type'])) - model.put(ref, store._bytes_to_vector(row['vector']), row['text']) + model.put((row['type'], row['id']), row['vector'], row['text']) q = [random.Random(9).uniform(-1, 1) for _ in range(dim)] _assert_same_ranking(store.similarity_search(q, top_k=25, threshold=-1.0), model.search(q, 25, None, -1.0)) @@ -2549,10 +2551,11 @@ def test_pending_rows_fold_and_heavy_churn_retrains(self): builds = store.ann_stats()['searches']['builds'] extra = draw(700) store.add_batch([(('late', i), '', extra[i].tolist()) for i in range(700)]) - assert store.ann_stats()['pending'] == 700 - store.similarity_search(extra[0].tolist(), 5) # >= 512 pending: folded + # folded at write time, 256 at a time: 700 = 2 x 256 + 188 still pending + assert store.ann_stats()['pending'] == 188 + store.similarity_search(extra[0].tolist(), 5) # a search folds nothing stats = store.ann_stats() - assert stats['pending'] == 0 and stats['searches']['builds'] == builds + assert stats['pending'] == 188 and stats['searches']['builds'] == builds more = draw(1200) store.add_batch([(('later', i), '', more[i].tolist()) for i in range(1200)]) store.similarity_search(more[0].tolist(), 5) # churn 1900 > 50% of 3000 diff --git a/tests/test_stage_d_python.py b/tests/test_stage_d_python.py new file mode 100644 index 0000000..63c9c80 --- /dev/null +++ b/tests/test_stage_d_python.py @@ -0,0 +1,969 @@ +#!/usr/bin/env python3 +""" +Stage D tests for the Python port: one copy of every vector, the numpy batch +search, and the optional binary snapshot (DESIGN.md sections 1, 3, 5 and 11). + +The numpy and pure-Python paths are both exercised (the latter by hiding numpy +from the module), and every store-level behaviour is run for an in-memory store +(whose scan cache is the table) and a file-backed one (SQLite is the durable +copy). +""" + +import hashlib +import os +import random +import shutil +import struct +import warnings +from pathlib import Path + +import pytest + +import isongraph_vector as iv +from isongraph_vector import ( + EmbeddingStore, + LoadReport, + MockEncoder, + SemanticGraph, + SnapshotError, + graph_digest, + snapshot_path_for, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "snapshot" + + +# ============================================================================= +# helpers +# ============================================================================= + +def make_store(mode, tmp_path, **kwargs): + """An in-memory store or a file-backed one.""" + if mode == "memory": + return EmbeddingStore(db_path=":memory:", **kwargs) + return EmbeddingStore(db_path=str(tmp_path / "emb.db"), **kwargs) + + +MODES = ["memory", "file"] + + +def rand_vec(rng, dim): + return [rng.uniform(-1, 1) for _ in range(dim)] + + +def brute(store, query, top_k, node_type, threshold): + """The definition of a bounded exact search: canonical score of every live + row, filtered, ranked by (score desc, row asc).""" + np = pytest.importorskip("numpy") + cache = store._ensure_cache() + if cache.n - cache.dead == 0: + return [] + q32, qnorm = store._query_parts(query) + rows = np.flatnonzero(cache.alive[:cache.n]) + if node_type is not None: + code = cache.code_of.get(node_type) + if code is None: + return [] + rows = rows[cache.codes[rows] == code] + scores = store._canonical_scores(cache, q32, qnorm, rows) + keep = scores.astype(np.float64) >= threshold + rows, scores = rows[keep], scores[keep] + order = np.argsort(-scores, kind="stable") + if top_k is not None: + order = order[:top_k] + return [(cache.refs[i], float(s), cache.texts[i]) + for i, s in zip(rows[order].tolist(), scores[order].tolist())] + + +def as_tuples(results): + return [(r.node_ref, r.score, r.text) for r in results] + + +def fill(store, rng, n, dim, types=("a", "b", "c"), dup_every=7, zero_every=29): + base = rand_vec(rng, dim) + entries = [] + for i in range(n): + if i % dup_every == 3: + v = list(base) # exact duplicates: ties + elif i % zero_every == 5: + v = [0.0] * dim + elif i % 11 == 4: + v = [x + 1e-6 * rng.uniform(-1, 1) for x in base] # near ties + else: + v = rand_vec(rng, dim) + entries.append(((types[i % len(types)], i if i % 3 else f"s{i}"), f"t{i}", v)) + store.add_batch(entries) + return base + + +# ============================================================================= +# 1. one copy of every vector +# ============================================================================= + +class TestSingleCopy: + def test_in_memory_store_keeps_rows_only_in_the_scan_cache(self): + pytest.importorskip("numpy") + store = EmbeddingStore(db_path=":memory:", dimension=8) + store.add_batch([(("p", i), f"t{i}", [float(i + 1)] * 8) for i in range(50)]) + assert store._table_mode() + assert store._conn is None, "no SQLite connection is opened for an in-memory store" + assert store.count() == 50 + cache = store._cache + assert cache.matrix is not None and cache.vectors is None + assert cache.matrix.dtype.name == "float32" and cache.matrix.flags["C_CONTIGUOUS"] + # the private table, if something does ask for it, holds no rows + assert store._get_conn().execute("SELECT COUNT(*) FROM embeddings").fetchone()[0] == 0 + + def test_file_backed_store_keeps_sqlite_as_the_durable_copy(self, tmp_path): + pytest.importorskip("numpy") + path = str(tmp_path / "e.db") + store = EmbeddingStore(db_path=path, dimension=4) + store.add_batch([(("p", i), "x", [1.0, i, 0.0, 1.0]) for i in range(30)]) + store.remove(("p", 3)) + store.close() + again = EmbeddingStore(db_path=path) + assert again.count() == 29 and again.dimension == 4 + assert again.get(("p", 4)).vector == [1.0, 4.0, 0.0, 1.0] + assert again.get(("p", 3)) is None + top = again.similarity_search([1.0, 29.0, 0.0, 1.0], 1, threshold=-1)[0] + assert top.node_ref == ("p", 29) + again.close() + + @pytest.mark.parametrize("mode", MODES) + def test_records_round_trip_through_every_accessor(self, mode, tmp_path): + enc = MockEncoder(6) + store = make_store(mode, tmp_path, encoder=enc, dimension=6) + store.add(("person", 7), "seven", vector=[0.5] * 6) # int id + store.add(("person", "007"), "bond", vector=[1, 2, 3, 4, 5, 6]) # numeric-looking str id + store.add(("doc", True), "flag", vector=[0.25] * 6) # bool id -> "True" + rec = store.get(("person", 7)) + assert rec.node_ref == ("person", 7) and isinstance(rec.node_ref[1], int) + assert rec.vector == [0.5] * 6 and rec.text == "seven" and rec.model == store._model_name() + assert store.get(("person", "007")).node_ref == ("person", "007") + assert store.get(("person", "007")).vector == [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + assert store.get(("doc", True)).node_ref == ("doc", "True") + assert store.get(("nope", 1)) is None + assert store.count() == 3 + exported = store.export_embeddings()["embeddings"] + assert [(e["type"], e["id"], e["id_type"]) for e in exported] == [ + ("person", 7, "int"), ("person", "007", "str"), ("doc", "True", "str")] + assert store.models() == {store._model_name(): 3} + assert store.remove(("person", 7)) and not store.remove(("person", 7)) + assert store.count() == 2 and store.get(("person", 7)) is None + assert store.stale() == [] + store.close() + + @pytest.mark.parametrize("mode", MODES) + def test_a_rejected_vector_changes_nothing(self, mode, tmp_path): + store = make_store(mode, tmp_path, dimension=3) + store.add_batch([(("a", i), "t", [1.0, float(i), 0.0]) for i in range(10)]) + for bad in ([1.0, 2.0], [1.0, "x", 2.0], [None, 0.0, 0.0]): + with pytest.raises(Exception): + store.add(("a", 99), "bad", vector=bad) + assert store.count() == 10 and store.get(("a", 99)) is None + assert len(store.similarity_search([1, 0, 0], top_k=None, threshold=-1)) == 10 + store.close() + + def test_growth_is_amortised_and_a_bulk_write_allocates_once(self): + np = pytest.importorskip("numpy") + store = EmbeddingStore(db_path=":memory:", dimension=4) + store.add(("g", 0), "x", vector=[1.0, 0.0, 0.0, 0.0]) + small = store._cache.matrix.shape[0] + store.add_batch([(("g", i), "x", [1.0, float(i), 0.0, 0.0]) for i in range(1, 20_000)]) + cache = store._cache + assert cache.n == 20_000 and cache.matrix.shape[0] >= 20_000 > small + # one allocation for the batch: no more than the batch itself plus the first chunk + assert cache.matrix.shape[0] <= 20_000 + 1024 + assert cache.matrix.base is None or cache.matrix.flags["OWNDATA"] + assert np.array_equal(cache.matrix[5], np.array([1.0, 5.0, 0.0, 0.0], dtype=np.float32)) + + @pytest.mark.parametrize("mode", MODES) + def test_random_writes_match_a_reference_model(self, mode, tmp_path): + pytest.importorskip("numpy") + rng = random.Random(41) + dim = 7 + store = make_store(mode, tmp_path, dimension=dim) + model = {} # insertion-ordered like the table + for step in range(1500): + r = rng.random() + if r < 0.5 or not model: + ref = (rng.choice("abc"), rng.randrange(400) if rng.random() < .6 else f"k{rng.randrange(400)}") + v = rand_vec(rng, dim) + store.add(ref, f"v{step}", vector=v) + if ref in model: + model[ref] = (v, f"v{step}") + else: + model[ref] = (v, f"v{step}") + elif r < 0.8: + ref = rng.choice(list(model)) + assert store.remove(ref) + del model[ref] + else: + q = rand_vec(rng, dim) + got = store.similarity_search(q, 5, threshold=-1.0) + assert as_tuples(got) == brute(store, q, 5, None, -1.0) + assert store.count() == len(model) + for ref, (v, text) in model.items(): + rec = store.get(ref) + assert rec.text == text and rec.vector == pytest.approx(v, abs=1e-6) + assert [(e["type"], e["id"]) for e in store.export_embeddings()["embeddings"]] \ + == list(model) + store.close() + + @pytest.mark.parametrize("mode", MODES) + def test_pure_python_form_is_one_copy_and_agrees_with_numpy(self, mode, tmp_path, monkeypatch): + pytest.importorskip("numpy") + rng = random.Random(3) + dim = 9 + data = [((("a", "b")[i % 2], i), f"t{i}", rand_vec(rng, dim)) for i in range(120)] + fast = make_store(mode, tmp_path, dimension=dim) + fast.add_batch(data) + queries = [rand_vec(rng, dim) for _ in range(6)] + want = [[(r.node_ref, r.text) for r in fast.similarity_search(q, 8, threshold=-1)] + for q in queries] + fast.close() + monkeypatch.setattr(iv, "_np", None) + slow = EmbeddingStore(db_path=":memory:", dimension=dim) + slow.add_batch(data) + cache = slow._cache + assert cache.matrix is None and cache.pos is not None + assert all(v.typecode == "f" for v in cache.vectors) # compact: 4 bytes a float + for q, expected in zip(queries, want): + got = [(r.node_ref, r.text) for r in slow.similarity_search(q, 8, threshold=-1)] + assert got == expected + # writes patch the list form in place (no rebuild), tombstones and compaction included + slow.remove(("a", 2)) + slow.add(("a", 4), "again", vector=[1.0] * dim) + for i in range(0, 120, 2): + if i != 4: + slow.remove(("a", i)) + assert slow._cache is cache # patched, never rebuilt + assert slow.count() == 61 == len(slow.export_embeddings()["embeddings"]) # ("a", 4) + 60 "b" + assert slow.get(("a", 4)).text == "again" + assert slow.get(("a", 6)) is None + top = slow.similarity_search([1.0] * dim, 1, threshold=-1)[0] + assert top.node_ref == ("a", 4) and top.score == pytest.approx(1.0, abs=1e-6) + slow.close() + + +# ============================================================================= +# 2. the bounded exact search and the batch search +# ============================================================================= + +class TestCanonicalSearchAndBatch: + @pytest.mark.parametrize("mode", MODES) + def test_bounded_search_is_the_definition_on_adversarial_data(self, mode, tmp_path): + pytest.importorskip("numpy") + rng = random.Random(5) + dim = 24 + store = make_store(mode, tmp_path, dimension=dim) + base = fill(store, rng, 700, dim) + for i in range(0, 700, 5): # deletes and updates + if i % 2: + store.remove((("a", "b", "c")[i % 3], i if i % 3 else f"s{i}")) + queries = [rand_vec(rng, dim) for _ in range(8)] + [base, [0.0] * dim] + # thresholds equal to scores that really occur, so the candidate set is tight + cache = store._ensure_cache() + q32, qn = store._query_parts(queries[0]) + occurring = sorted(set(store._canonical_scores( + cache, q32, qn, __import__("numpy").arange(cache.n)).tolist())) + thresholds = [-1.0, 0.0, 0.2, occurring[len(occurring) // 2], occurring[-3], 2.0] + for q in queries: + for top_k in (1, 3, 10, 64, 100000, None): + for ntype in (None, "a", "zzz"): + for thr in thresholds: + got = store.similarity_search(q, top_k, ntype, thr) + assert as_tuples(got) == brute(store, q, top_k, ntype, thr), \ + (top_k, ntype, thr) + store.close() + + @pytest.mark.parametrize("mode", MODES) + def test_canonical_scores_agree_with_float64_to_float32_precision(self, mode, tmp_path): + np = pytest.importorskip("numpy") + rng = random.Random(8) + dim = 64 + store = make_store(mode, tmp_path, dimension=dim) + fill(store, rng, 300, dim) + q = rand_vec(rng, dim) + for r in store.similarity_search(q, 20, threshold=-1.0): + v = np.array(store.get(r.node_ref).vector, dtype=np.float64) + qq = np.array(q, dtype=np.float64) + denom = np.linalg.norm(v) * np.linalg.norm(qq) + expected = float(v @ qq / denom) if denom else 0.0 + assert r.score == pytest.approx(expected, abs=2e-6) + store.close() + + @pytest.mark.parametrize("mode", MODES) + def test_batch_equals_single_bit_for_bit(self, mode, tmp_path): + pytest.importorskip("numpy") + rng = random.Random(6) + dim = 16 + store = make_store(mode, tmp_path, dimension=dim, encoder=MockEncoder(dim)) + base = fill(store, rng, 900, dim) + for i in range(0, 900, 6): + store.remove((("a", "b", "c")[i % 3], i if i % 3 else f"s{i}")) + queries = [rand_vec(rng, dim) for _ in range(150)] + [base, [0.0] * dim, "some text", "other"] + for top_k in (1, 5, 10, 100000): + for ntype in (None, "b", "zzz"): + for thr in (-1.0, 0.0, 0.4, 0.95): + batch = store.similarity_search_batch(queries, top_k, ntype, thr) + assert len(batch) == len(queries) + for q, got in zip(queries, batch): + assert as_tuples(got) == as_tuples( + store.similarity_search(q, top_k, ntype, thr)) + store.close() + + def test_batch_edge_cases(self): + pytest.importorskip("numpy") + store = EmbeddingStore(db_path=":memory:", dimension=3) + assert store.similarity_search_batch([]) == [] + assert store.similarity_search_batch([[1, 0, 0], [0, 1, 0]]) == [[], []] # empty store + store.add(("x", 1), "one", vector=[1.0, 0.0, 0.0]) + assert [[r.node_ref for r in res] for res in + store.similarity_search_batch([[1, 0, 0]] * 3, top_k=5)] == [[("x", 1)]] * 3 + assert store.similarity_search_batch([[1, 0, 0]], top_k=0) == [[]] + with pytest.raises(iv.DimensionMismatchError): + store.similarity_search_batch([[1, 0, 0], [1, 0]]) + with pytest.raises(iv.NoEncoderError): + store.similarity_search_batch(["text"]) + # one result list per query, in query order, with top_k=None meaning every match + res = store.similarity_search_batch([[1, 0, 0], [0, 1, 0]], top_k=None, threshold=-1) + assert [len(r) for r in res] == [1, 1] + store.close() + + def test_batch_groups_split_large_batches_without_changing_answers(self): + pytest.importorskip("numpy") + rng = random.Random(1) + dim = 8 + store = EmbeddingStore(db_path=":memory:", dimension=dim) + fill(store, rng, 400, dim) + store._BATCH_SCORE_BYTES = 4 * 400 * 3 # three queries per matrix product + queries = [rand_vec(rng, dim) for _ in range(10)] + for got, q in zip(store.similarity_search_batch(queries, 7), queries): + assert as_tuples(got) == as_tuples(store.similarity_search(q, 7)) + + def test_batch_without_canonical_scores_answers_one_by_one(self): + pytest.importorskip("numpy") + rng = random.Random(2) + store = EmbeddingStore(db_path=":memory:", dimension=8) + fill(store, rng, 200, 8) + store.canonical_scores = False + assert store.canonical_scores is False + queries = [rand_vec(rng, 8) for _ in range(5)] + for got, q in zip(store.similarity_search_batch(queries, 6), queries): + assert as_tuples(got) == as_tuples(store.similarity_search(q, 6)) + + def test_raw_blas_scores_depend_on_row_position_but_canonical_ones_do_not(self): + """Why the bounded search rescoring exists: one vector, two slots.""" + np = pytest.importorskip("numpy") + rng = np.random.default_rng(0) + dim = 384 + data = rng.standard_normal((6001, dim)).astype(np.float32) + q = rng.standard_normal(dim).astype(np.float32) + shifted = np.concatenate([data[:1], data]) # every row moves down one slot + a = data @ q + b = (shifted @ q)[1:] + raw_differs = int((a.view(np.uint32) != b.view(np.uint32)).sum()) + store = EmbeddingStore(db_path=":memory:", dimension=dim) + store.add_batch([(("r", i), "", data[i].tolist()) for i in range(6001)]) + store.add(("pad", 0), "", vector=data[0].tolist()) # a second copy of row 0 at the end + cache = store._ensure_cache() + q32, qn = store._query_parts(q.tolist()) + canon = store._canonical_scores(cache, q32, qn, np.array([0, cache.n - 1])) + assert canon[0] == canon[1] + # raw BLAS rows that moved one slot usually changed in the last bit (a few percent on + # OpenBLAS, none on a BLAS that is position independent): nothing to assert but the bound + assert 0 <= raw_differs <= 6001 + + @pytest.mark.parametrize("mode", MODES) + def test_batch_with_ann_off_exact_and_loop_when_approximate(self, mode, tmp_path): + pytest.importorskip("numpy") + rng = random.Random(9) + dim = 12 + store = make_store(mode, tmp_path, dimension=dim, ann="ivf", ann_min_size=0, + ann_nlist=8, ann_nprobe=2) + fill(store, rng, 600, dim) + queries = [rand_vec(rng, dim) for _ in range(6)] + for exact in (False, True): + for got, q in zip(store.similarity_search_batch(queries, 5, exact=exact), queries): + assert as_tuples(got) == as_tuples(store.similarity_search(q, 5, exact=exact)) + store.close() + + def test_pure_python_batch_is_the_loop(self, monkeypatch): + monkeypatch.setattr(iv, "_np", None) + rng = random.Random(4) + store = EmbeddingStore(db_path=":memory:", dimension=5) + fill(store, rng, 80, 5) + queries = [rand_vec(rng, 5) for _ in range(4)] + for got, q in zip(store.similarity_search_batch(queries, 4), queries): + assert as_tuples(got) == as_tuples(store.similarity_search(q, 4)) + + def test_text_queries_in_a_batch_use_the_single_query_encoding(self): + pytest.importorskip("numpy") + enc = MockEncoder(16) + store = EmbeddingStore(db_path=":memory:", encoder=enc) + store.add_batch([(("d", i), f"document number {i}") for i in range(60)]) + texts = ["document number 3", "something else", "document number 41"] + for got, text in zip(store.similarity_search_batch(texts, 3), texts): + assert as_tuples(got) == as_tuples(store.similarity_search(text, 3)) + assert store.similarity_search_batch(["document number 3"], 1)[0][0].node_ref == ("d", 3) + + +# ============================================================================= +# 3. IVF: folds at write time +# ============================================================================= + +def _ivf_store(n=3000, dim=16, seed=1, **kw): + np = pytest.importorskip("numpy") + rng = np.random.default_rng(seed) + centers = rng.standard_normal((30, dim)).astype(np.float32) + + def draw(count): + which = rng.integers(0, 30, count) + return (centers[which] + 0.3 * rng.standard_normal((count, dim))).astype(np.float32) + + kw.setdefault("ann", "ivf") + kw.setdefault("ann_min_size", 0) + kw.setdefault("ann_nlist", 32) + kw.setdefault("ann_nprobe", 6) + store = EmbeddingStore(db_path=":memory:", dimension=dim, **kw) + data = draw(n) + store.add_batch([(("doc", i), "", data[i].tolist()) for i in range(n)]) + return store, draw + + +def _check_ivf_invariants(store): + """Every live row is in exactly one cell list (at its own cell) or pending.""" + np = pytest.importorskip("numpy") + cache = store._cache + ivf = cache.ivf + n = cache.n + listed_rows = [] + for c in range(ivf.nlist): + part = ivf.order[ivf.offsets[c]:ivf.offsets[c + 1]] + mine = part[ivf.assign[part] == c] + listed_rows.extend(mine.tolist()) + assert len(listed_rows) == len(set(listed_rows)), "a row is listed twice" + live = set(np.flatnonzero(cache.alive[:n]).tolist()) + assert set(listed_rows) | set(ivf.pending) == live + assert not (set(listed_rows) & set(ivf.pending)) + assert all(ivf.assign[i] == -1 for i in ivf.pending) + assert len(ivf.pending) < 256 + assert all(ivf.listed[r] for r in listed_rows) + + +class TestIvfFoldAtWrite: + def test_the_write_that_fills_a_batch_folds_it(self): + store, draw = _ivf_store() + store.build_ann_index() + builds = store.ann_stats()["searches"]["builds"] + extra = draw(600) + for i in range(255): + store.add(("new", i), "", vector=extra[i].tolist()) + assert store.ann_stats()["pending"] == 255 # one short: nothing folded yet + store.add(("new", 255), "", vector=extra[255].tolist()) + assert store.ann_stats()["pending"] == 0 # the 256th write folded them + for i in range(256, 600): + store.add(("new", i), "", vector=extra[i].tolist()) + assert store.ann_stats()["pending"] == 600 - 512 + assert store.ann_stats()["searches"]["builds"] == builds + _check_ivf_invariants(store) + + def test_cell_lists_stay_consistent_through_inserts_updates_deletes(self): + pytest.importorskip("numpy") + store, draw = _ivf_store(n=2500) + store.build_ann_index() + rng = random.Random(12) + extra = draw(2000) + nxt = 0 + live = [("doc", i) for i in range(2500)] + for step in range(1800): + r = rng.random() + if r < 0.55: + store.add(("new", nxt), "", vector=extra[nxt % 2000].tolist()) + live.append(("new", nxt)) + nxt += 1 + elif r < 0.75: + ref = rng.choice(live) # update: a row that is already listed + store.add(ref, "", vector=extra[rng.randrange(2000)].tolist()) + else: + ref = live.pop(rng.randrange(len(live))) + store.remove(ref) + if step % 150 == 0 and store._cache.ivf is not None: + _check_ivf_invariants(store) + store.similarity_search(extra[0].tolist(), 5) + if store._cache.ivf is not None: + _check_ivf_invariants(store) + # and the answers: every returned row is live, once, with its exact cosine + for q in draw(10).tolist(): + res = store.similarity_search(q, 20, threshold=-1.0) + refs = [r.node_ref for r in res] + assert len(refs) == len(set(refs)) and set(refs) <= set(live) + + def test_incremental_fold_equals_a_full_regroup(self): + pytest.importorskip("numpy") + store, draw = _ivf_store(n=2000) + store.build_ann_index() + extra = draw(600) + store.add_batch([(("new", i), "", extra[i].tolist()) for i in range(600)]) + ivf = store._cache.ivf + ivf.fold(store._cache) + spliced = [ivf.order[ivf.offsets[c]:ivf.offsets[c + 1]].tolist() for c in range(ivf.nlist)] + ivf.relist(store._cache.n) + regrouped = [ivf.order[ivf.offsets[c]:ivf.offsets[c + 1]].tolist() for c in range(ivf.nlist)] + assert spliced == regrouped + + def test_approximate_results_carry_the_exact_searchs_scores_bit_for_bit(self): + pytest.importorskip("numpy") + store, draw = _ivf_store(n=3000, ann_nprobe=5) + extra = draw(400) + store.add_batch([(("late", i), "", extra[i].tolist()) for i in range(400)]) + for q in draw(15).tolist(): + exact = {r.node_ref: r.score + for r in store.similarity_search(q, 100000, threshold=-1.0, exact=True)} + before = store.ann_stats()["searches"]["approximate"] + found = store.similarity_search(q, 10) + assert store.ann_stats()["searches"]["approximate"] == before + 1 + assert found and all(r.score == exact[r.node_ref] for r in found) + ordered = [(-r.score, r.node_ref) for r in found] + assert [r.score for r in found] == sorted((r.score for r in found), reverse=True) + assert len(set(ordered)) == len(ordered) + + def test_probing_every_cell_still_equals_the_exact_scan(self): + pytest.importorskip("numpy") + store, draw = _ivf_store(n=1500, ann_nprobe=32) # nprobe == nlist + extra = draw(700) + store.add_batch([(("late", i), "", extra[i].tolist()) for i in range(700)]) + for q in draw(5).tolist(): + got = as_tuples(store.similarity_search(q, 10)) + want = as_tuples(store.similarity_search(q, 10, exact=True)) + assert [g[0] for g in got] == [w[0] for w in want] + + +# ============================================================================= +# 4. binary snapshot +# ============================================================================= + +def _copy_golden(tmp_path): + for f in FIXTURES.iterdir(): + shutil.copy(f, tmp_path / f.name) + return tmp_path / "golden.ison" + + +def _store_state(graph): + """Everything a store exposes, for equality between two loads.""" + store = graph.embedding_store + cache = store._ensure_cache() + live = [i for i in range(cache.n) if cache.refs[i] is not None] + return { + "export": store.export_embeddings(), + "models": store.models(), + "count": store.count(), + "dimension": store.dimension, + "norms": None if cache.matrix is None else cache.norms[live].tobytes(), + "matrix": None if cache.matrix is None else cache.matrix[live].tobytes(), + "meta": dict(store.last_import_meta), + } + + +def _graph_load(path, **kw): + kw.setdefault("encoder", MockEncoder(12)) + return SemanticGraph.load_with_report(path, **kw) + + +class TestSnapshotGolden: + """The fixture written by the Rust crate (DESIGN.md section 11.2).""" + + def test_golden_snapshot_is_used_and_equals_the_json_load(self, tmp_path): + path = _copy_golden(tmp_path) + g, report = _graph_load(path) + assert isinstance(report, LoadReport) + assert (report.source, report.snapshot_skipped, report.embeddings) == ("snapshot", None, 36) + assert report.seconds >= 0 and g.load_report is report + snap_state = _store_state(g) + + snapshot_path_for(path).unlink() + j, jreport = _graph_load(path) + assert (jreport.source, jreport.snapshot_skipped, jreport.embeddings) == ("json", None, 36) + assert _store_state(j) == snap_state + # same search results, scores bit for bit + for q in ([0.5] * 12, [1.0] + [0.0] * 11, [0.0] * 12): + for k in (1, 5, 100): + a = g.embedding_store.similarity_search(q, k, threshold=-1) + b = j.embedding_store.similarity_search(q, k, threshold=-1) + assert as_tuples(a) == as_tuples(b) + assert g.embedding_store.last_import_meta["graph_digest"] == graph_digest(g) + + def test_golden_snapshot_without_numpy(self, tmp_path, monkeypatch): + monkeypatch.setattr(iv, "_np", None) + path = _copy_golden(tmp_path) + g, report = _graph_load(path) + assert report.source == "snapshot" and report.embeddings == 36 + snapshot_path_for(path).unlink() + j, _ = _graph_load(path) + assert g.embedding_store.export_embeddings() == j.embedding_store.export_embeddings() + a = g.embedding_store.similarity_search([0.5] * 12, 5, threshold=-1) + b = j.embedding_store.similarity_search([0.5] * 12, 5, threshold=-1) + assert as_tuples(a) == as_tuples(b) + + def test_golden_header_and_layout(self): + raw = (FIXTURES / "golden.ison.embeddings.snap").read_bytes() + assert raw[:8] == b"ISONVSNP" + version, flags, dim, block_rows, count = struct.unpack_from(" "a": a repeated key, with a valid checksum + dup = body.replace(struct.pack(" Date: Tue, 17 Nov 2026 08:46:14 -0600 Subject: [PATCH 5/7] C++: one chunked row store, thread-safe const search and batch search, optional binary snapshot The index no longer copies vectors; ChunkedMatrix (1024-row chunks) is the only copy and records reference rows. One out-of-line scoring routine (detail::cosineRows, noinline/noclone) is used by the scan, batch, IVF rerank and snapshot-loaded stores. Searches share a shared_mutex; similaritySearchBatch scores tiles of 16 queries per chunk across bounded worker threads. Pending rows fold at write time (256). Optional .embeddings.snap per Rust DESIGN.md section 11 with a streaming SHA-256; re-saving the Rust golden store reproduces its snapshot byte for byte. Breaking: get()/getEmbedding() return std::optional (a copy), EmbeddingRecord lost ann_slot, AnnStats::vectorBytes is 0, equal scores order by arrival, pending rows fold at the 256th write. Co-Authored-By: Claude Sonnet 5.5 --- isongraph-vector-cpp/CMakeLists.txt | 10 + isongraph-vector-cpp/bench/stage_d_bench.cpp | 231 ++ .../include/isongraph_vector.hpp | 1924 +++++++++++++---- .../include/isongraph_vector_ivf.hpp | 409 +++- .../tests/test_isongraph_vector.cpp | 1132 +++++++++- 5 files changed, 3195 insertions(+), 511 deletions(-) create mode 100644 isongraph-vector-cpp/bench/stage_d_bench.cpp diff --git a/isongraph-vector-cpp/CMakeLists.txt b/isongraph-vector-cpp/CMakeLists.txt index d73b753..7f75933 100644 --- a/isongraph-vector-cpp/CMakeLists.txt +++ b/isongraph-vector-cpp/CMakeLists.txt @@ -120,6 +120,12 @@ if(ISONGRAPH_VECTOR_BENCH) target_compile_options(ann_bench PRIVATE -march=native) endif() endif() + + # Resident memory, single/batch search and snapshot-vs-JSON timings + # (bench/stage_d_bench.cpp; see its header for the phases). + add_executable(stage_d_bench bench/stage_d_bench.cpp) + target_link_libraries(stage_d_bench PRIVATE isongraph_vector) + target_compile_definitions(stage_d_bench PRIVATE ISONGRAPH_BENCH_NEW=1) endif() # Tests. Skipped when this project is consumed via add_subdirectory. @@ -128,6 +134,10 @@ enable_testing() add_executable(test_isongraph_vector tests/test_isongraph_vector.cpp) target_link_libraries(test_isongraph_vector PRIVATE isongraph_vector) add_test(NAME test_isongraph_vector COMMAND test_isongraph_vector) +# The golden snapshot fixture written by the Rust crate lives beside the other +# ports' fixtures at the repository root. +target_compile_definitions(test_isongraph_vector PRIVATE + ISONGRAPH_VECTOR_FIXTURE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../tests/fixtures/snapshot") if(ISONGRAPH_VECTOR_SQLITE) add_executable(test_isongraph_vector_sqlite tests/test_isongraph_vector_sqlite.cpp) diff --git a/isongraph-vector-cpp/bench/stage_d_bench.cpp b/isongraph-vector-cpp/bench/stage_d_bench.cpp new file mode 100644 index 0000000..3bc2308 --- /dev/null +++ b/isongraph-vector-cpp/bench/stage_d_bench.cpp @@ -0,0 +1,231 @@ +// Memory, search and persistence benchmark for the vector store. +// +// stage_d_bench [n=100000] [dim=384] [queries=200] +// +// phases (run each in its own process so the resident-set numbers are clean): +// mem resident memory after importing n random unit vectors, and after the +// approximate index is built, against the raw vector bytes +// search single-query exact and approximate QPS, parallel batch QPS +// io JSON sidecar save/load, and the binary snapshot save/load (when this +// build has it), with file sizes +// +// Builds against either header set; compile with -DISONGRAPH_BENCH_NEW when the +// store has similaritySearchBatch() and the snapshot API. +// +// g++ -O2 -I include -I vendor/ison-graph-cpp -DISONGRAPH_BENCH_NEW bench/stage_d_bench.cpp -pthread +// +// Prints one "key=value" line per measurement so a script can collect the +// medians of several runs. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#define NOMINMAX +#include +#include +#endif + +#include "isongraph_vector.hpp" + +using namespace isongraph_vector; +using Clock = std::chrono::steady_clock; + +namespace { + +/// Resident set size of this process in bytes (0 where unknown). +size_t residentBytes() { +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS counters; + if (GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof counters)) return counters.WorkingSetSize; + return 0; +#else + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) return static_cast(std::stoull(line.substr(6))) * 1024; + } + return 0; +#endif +} + +struct Rng { + uint64_t state; + explicit Rng(uint64_t seed) : state(seed) {} + uint64_t next() { + state += 0x9E3779B97F4A7C15ull; + uint64_t z = state; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); + } + float unit() { return static_cast(next() >> 40) / 8388608.0f - 1.0f; } +}; + +std::vector unitVector(Rng& rng, size_t dim) { + std::vector v(dim); + double sq = 0; + for (auto& x : v) { x = rng.unit(); sq += double(x) * x; } + const float inv = static_cast(1.0 / std::sqrt(sq)); + for (auto& x : v) x *= inv; + return v; +} + +double secondsSince(Clock::time_point t) { + return std::chrono::duration(Clock::now() - t).count(); +} + +void fill(EmbeddingStore& store, size_t n, size_t dim) { + Rng rng(12345); + for (size_t i = 0; i < n; ++i) store.add(NodeRef("doc", std::to_string(i)), "", unitVector(rng, dim)); +} + +double mib(double bytes) { return bytes / 1048576.0; } + +void report(const char* key, double value) { std::printf("%s=%.4f\n", key, value); } + +} // namespace + +int main(int argc, char** argv) { + const std::string phase = argc > 1 ? argv[1] : "mem"; + const size_t n = argc > 2 ? std::strtoull(argv[2], nullptr, 10) : 100000; + const size_t dim = argc > 3 ? std::strtoull(argv[3], nullptr, 10) : 384; + const size_t nq = argc > 4 ? std::strtoull(argv[4], nullptr, 10) : 200; + const double raw = static_cast(n) * static_cast(dim) * 4.0; + + if (phase == "mem") { + const size_t before = residentBytes(); + EmbeddingStore store; + fill(store, n, dim); + const size_t imported = residentBytes(); + report("raw_mib", mib(raw)); + report("rss_import_mib", mib(static_cast(imported - before))); + report("rss_import_over_raw", static_cast(imported - before) / raw); + AnnOptions o; + o.minSize = 0; + store.enableAnn(o); + store.buildAnnIndex(); + const size_t indexed = residentBytes(); + report("rss_indexed_mib", mib(static_cast(indexed - before))); + report("rss_indexed_over_raw", static_cast(indexed - before) / raw); + return 0; + } + + if (phase == "search") { + EmbeddingStore store; + fill(store, n, dim); + Rng qrng(777); + std::vector> queries; + for (size_t i = 0; i < nq; ++i) queries.push_back(unitVector(qrng, dim)); + + store.similaritySearchVector(queries[0], 10, "", 0.0f); // warm up + auto t0 = Clock::now(); + for (const auto& q : queries) store.similaritySearchVector(q, 10, "", 0.0f); + report("exact_single_qps", static_cast(nq) / secondsSince(t0)); + + // parallel batch of the exact scan + const size_t threads = std::max(1, std::thread::hardware_concurrency()); +#if defined(ISONGRAPH_BENCH_NEW) + store.similaritySearchBatch(queries, 10, "", 0.0f, false, threads); // warm up + t0 = Clock::now(); + auto batch = store.similaritySearchBatch(queries, 10, "", 0.0f, false, threads); + report("exact_batch_qps", static_cast(nq) / secondsSince(t0)); +#else + // no batch API: the same work spread over threads by hand, one query per task + { + std::vector> out(nq); + auto runAll = [&]() { + std::atomic next{0}; + std::vector pool; + for (size_t t = 0; t < threads; ++t) { + pool.emplace_back([&]() { + for (;;) { + size_t i = next.fetch_add(1); + if (i >= nq) break; + out[i] = store.similaritySearchVector(queries[i], 10, "", 0.0f); + } + }); + } + for (auto& t : pool) t.join(); + }; + runAll(); + t0 = Clock::now(); + runAll(); + report("exact_batch_qps", static_cast(nq) / secondsSince(t0)); + } +#endif + + AnnOptions o; + o.minSize = 0; + store.enableAnn(o); + auto b0 = Clock::now(); + store.buildAnnIndex(); + report("ann_build_s", secondsSince(b0)); + store.similaritySearchVector(queries[0], 10, "", 0.0f); + t0 = Clock::now(); + for (const auto& q : queries) store.similaritySearchVector(q, 10, "", 0.0f); + report("ann_single_qps", static_cast(nq) / secondsSince(t0)); +#if defined(ISONGRAPH_BENCH_NEW) + t0 = Clock::now(); + auto annBatch = store.similaritySearchBatch(queries, 10, "", 0.0f, false, threads); + report("ann_batch_qps", static_cast(nq) / secondsSince(t0)); +#endif + RecallReport recall = store.recallCheck(std::vector>(queries.begin(), queries.begin() + std::min(nq, 50)), 10); + report("ann_recall10", recall.recall); + + // write cost: single adds with the index on (new rows, then updates) + Rng wrng(4242); + auto w0 = Clock::now(); + const size_t writes = 2000; + for (size_t i = 0; i < writes; ++i) store.add(NodeRef("late", std::to_string(i)), "", unitVector(wrng, dim)); + report("ann_write_us", secondsSince(w0) / static_cast(writes) * 1e6); + return 0; + } + + if (phase == "io") { + EmbeddingStore store; + fill(store, n, dim); + const auto dir = std::filesystem::temp_directory_path() / "isongraph_stage_d_bench"; + std::filesystem::create_directories(dir); + const std::string json = (dir / "bench.embeddings.json").string(); + auto t0 = Clock::now(); + store.saveEmbeddings(json); + report("json_save_s", secondsSince(t0)); + report("json_mib", mib(static_cast(std::filesystem::file_size(json)))); + { + EmbeddingStore back; + t0 = Clock::now(); + back.loadEmbeddings(json); + report("json_load_s", secondsSince(t0)); + } +#if defined(ISONGRAPH_BENCH_NEW) + const std::string snap = (dir / "bench.embeddings.snap").string(); + t0 = Clock::now(); + store.saveSnapshot(snap); + report("snap_save_s", secondsSince(t0)); + report("snap_mib", mib(static_cast(std::filesystem::file_size(snap)))); + { + EmbeddingStore back; + t0 = Clock::now(); + back.loadSnapshot(snap); + report("snap_load_s", secondsSince(t0)); + } +#endif + std::error_code ec; + std::filesystem::remove_all(dir, ec); + return 0; + } + + std::cerr << "unknown phase " << phase << "\n"; + return 2; +} diff --git a/isongraph-vector-cpp/include/isongraph_vector.hpp b/isongraph-vector-cpp/include/isongraph_vector.hpp index b21dffc..9ab3967 100644 --- a/isongraph-vector-cpp/include/isongraph_vector.hpp +++ b/isongraph-vector-cpp/include/isongraph_vector.hpp @@ -49,6 +49,16 @@ #include #include #include +#include +#include +#include +#include +#include +#if defined(_WIN32) +#include +#elif defined(__unix__) || defined(__APPLE__) +#include +#endif #include "ison_graph.hpp" #include "isongraph_vector_ivf.hpp" @@ -301,6 +311,12 @@ class MockEncoder : public EmbeddingEncoder { // Embedding Record // ============================================================================= +/** + * A stored embedding as handed out by EmbeddingStore::get(): a *copy* of the row. + * The store itself keeps each vector once, in a chunked matrix, and no record + * object exists inside it, so there is nothing for a reference to point at (see + * EmbeddingStore::vectorData for a zero-copy view). + */ struct EmbeddingRecord { std::string id; NodeRef node_ref; @@ -308,12 +324,6 @@ struct EmbeddingRecord { std::string text; /** Model that produced the vector: the encoder's modelName(), "provided" or "unknown". */ std::string model = "unknown"; - /** - * Internal: this record's row in the approximate index, valid only while - * an index exists (see EmbeddingStore::enableAnn). Not part of the data; - * never saved, copied between stores or compared. - */ - mutable uint32_t ann_slot = detail::IvfIndex::kNoSlot; }; // ============================================================================= @@ -615,36 +625,87 @@ class JsonParser { // ----------------------------------------------------------------------------- // SHA-256 (FIPS 180-4), self-contained so the graph digest needs no dependency. +// Streaming, so a multi-gigabyte snapshot can be hashed as it is read or written. // ----------------------------------------------------------------------------- -inline std::string sha256Hex(const std::string& data) { - static const uint32_t K[64] = { - 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, 0x3956c25bu, 0x59f111f1u, 0x923f82a4u, 0xab1c5ed5u, - 0xd807aa98u, 0x12835b01u, 0x243185beu, 0x550c7dc3u, 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 0xc19bf174u, - 0xe49b69c1u, 0xefbe4786u, 0x0fc19dc6u, 0x240ca1ccu, 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, - 0x983e5152u, 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, 0xc6e00bf3u, 0xd5a79147u, 0x06ca6351u, 0x14292967u, - 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 0x53380d13u, 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, - 0xa2bfe8a1u, 0xa81a664bu, 0xc24b8b70u, 0xc76c51a3u, 0xd192e819u, 0xd6990624u, 0xf40e3585u, 0x106aa070u, - 0x19a4c116u, 0x1e376c08u, 0x2748774cu, 0x34b0bcb5u, 0x391c0cb3u, 0x4ed8aa4au, 0x5b9cca4fu, 0x682e6ff3u, - 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u - }; - uint32_t h[8] = {0x6a09e667u, 0xbb67ae85u, 0x3c6ef372u, 0xa54ff53au, - 0x510e527fu, 0x9b05688cu, 0x1f83d9abu, 0x5be0cd19u}; +class Sha256 { +public: + Sha256() { reset(); } - std::string padded = data; - padded.push_back(static_cast(0x80)); - while (padded.size() % 64 != 56) padded.push_back('\0'); - const uint64_t bitLength = static_cast(data.size()) * 8u; - for (int shift = 56; shift >= 0; shift -= 8) { - padded.push_back(static_cast((bitLength >> shift) & 0xFFu)); + void reset() { + h_[0] = 0x6a09e667u; h_[1] = 0xbb67ae85u; h_[2] = 0x3c6ef372u; h_[3] = 0xa54ff53au; + h_[4] = 0x510e527fu; h_[5] = 0x9b05688cu; h_[6] = 0x1f83d9abu; h_[7] = 0x5be0cd19u; + buffered_ = 0; + total_ = 0; } - auto rotr = [](uint32_t x, unsigned n) { return (x >> n) | (x << (32u - n)); }; + void update(const void* data, size_t size) { + const unsigned char* p = static_cast(data); + total_ += size; + if (buffered_ > 0) { + const size_t take = std::min(size, size_t(64) - buffered_); + std::copy(p, p + take, buffer_ + buffered_); + buffered_ += take; + p += take; + size -= take; + if (buffered_ < 64) return; + compress(buffer_); + buffered_ = 0; + } + while (size >= 64) { + compress(p); + p += 64; + size -= 64; + } + if (size > 0) { + std::copy(p, p + size, buffer_); + buffered_ = size; + } + } - for (size_t offset = 0; offset < padded.size(); offset += 64) { + /** The digest of everything fed so far; the hasher itself is left unchanged. */ + std::array digest() const { + Sha256 copy = *this; + const uint64_t bitLength = copy.total_ * 8u; + const unsigned char one = 0x80; + copy.update(&one, 1); + const unsigned char zero = 0; + while (copy.buffered_ != 56) copy.update(&zero, 1); + unsigned char length[8]; + for (int i = 0; i < 8; ++i) length[i] = static_cast((bitLength >> (56 - 8 * i)) & 0xFFu); + copy.update(length, 8); + std::array out{}; + for (size_t i = 0; i < 8; ++i) { + out[4 * i] = static_cast(copy.h_[i] >> 24); + out[4 * i + 1] = static_cast(copy.h_[i] >> 16); + out[4 * i + 2] = static_cast(copy.h_[i] >> 8); + out[4 * i + 3] = static_cast(copy.h_[i]); + } + return out; + } + +private: + uint32_t h_[8]; + unsigned char buffer_[64]; + size_t buffered_ = 0; + uint64_t total_ = 0; + + static uint32_t rotr(uint32_t x, unsigned n) { return (x >> n) | (x << (32u - n)); } + + void compress(const unsigned char* block) { + static const uint32_t K[64] = { + 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, 0x3956c25bu, 0x59f111f1u, 0x923f82a4u, 0xab1c5ed5u, + 0xd807aa98u, 0x12835b01u, 0x243185beu, 0x550c7dc3u, 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 0xc19bf174u, + 0xe49b69c1u, 0xefbe4786u, 0x0fc19dc6u, 0x240ca1ccu, 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, + 0x983e5152u, 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, 0xc6e00bf3u, 0xd5a79147u, 0x06ca6351u, 0x14292967u, + 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 0x53380d13u, 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, + 0xa2bfe8a1u, 0xa81a664bu, 0xc24b8b70u, 0xc76c51a3u, 0xd192e819u, 0xd6990624u, 0xf40e3585u, 0x106aa070u, + 0x19a4c116u, 0x1e376c08u, 0x2748774cu, 0x34b0bcb5u, 0x391c0cb3u, 0x4ed8aa4au, 0x5b9cca4fu, 0x682e6ff3u, + 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u + }; uint32_t w[64]; for (size_t i = 0; i < 16; ++i) { - const unsigned char* p = reinterpret_cast(padded.data()) + offset + i * 4; + const unsigned char* p = block + i * 4; w[i] = (static_cast(p[0]) << 24) | (static_cast(p[1]) << 16) | (static_cast(p[2]) << 8) | static_cast(p[3]); } @@ -653,7 +714,7 @@ inline std::string sha256Hex(const std::string& data) { uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); w[i] = w[i - 16] + s0 + w[i - 7] + s1; } - uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + uint32_t a = h_[0], b = h_[1], c = h_[2], d = h_[3], e = h_[4], f = h_[5], g = h_[6], hh = h_[7]; for (size_t i = 0; i < 64; ++i) { uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); uint32_t ch = (e & f) ^ (~e & g); @@ -663,19 +724,29 @@ inline std::string sha256Hex(const std::string& data) { uint32_t t2 = S0 + maj; hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; } - h[0] += a; h[1] += b; h[2] += c; h[3] += d; - h[4] += e; h[5] += f; h[6] += g; h[7] += hh; + h_[0] += a; h_[1] += b; h_[2] += c; h_[3] += d; + h_[4] += e; h_[5] += f; h_[6] += g; h_[7] += hh; } +}; +inline std::string hexOf(const unsigned char* bytes, size_t n) { static const char* const HEX = "0123456789abcdef"; std::string out; - out.reserve(64); - for (uint32_t word : h) { - for (int shift = 28; shift >= 0; shift -= 4) out.push_back(HEX[(word >> shift) & 0xFu]); + out.reserve(n * 2); + for (size_t i = 0; i < n; ++i) { + out.push_back(HEX[(bytes[i] >> 4) & 0xFu]); + out.push_back(HEX[bytes[i] & 0xFu]); } return out; } +inline std::string sha256Hex(const std::string& data) { + Sha256 hasher; + hasher.update(data.data(), data.size()); + const auto digest = hasher.digest(); + return hexOf(digest.data(), digest.size()); +} + // ----------------------------------------------------------------------------- // Atomic file writes // ----------------------------------------------------------------------------- @@ -1071,7 +1142,7 @@ struct AnnStats { double buildSeconds = 0.0; /** Memory held by the index on top of the vectors it scans. */ size_t indexBytes = 0; - /** Memory of the contiguous copy of the vectors the index scans. */ + /** Always 0 now: the index keeps no copy of the vectors (kept so existing code still compiles). */ size_t vectorBytes = 0; size_t minSize = 0; /** Searches answered from the index, from a widened probe, by the full scan, and index builds, since enableAnn. */ @@ -1101,82 +1172,415 @@ struct RecallReport { double exactMs = 0.0; }; +// ============================================================================= +// Binary snapshot (optional, beside the JSON sidecar) +// ============================================================================= + +/** Appended to the graph file's name: `.embeddings.snap`. */ +constexpr const char* EMBEDDING_SNAPSHOT_SUFFIX = ".embeddings.snap"; +/** The only snapshot layout version this build reads and writes. */ +constexpr uint32_t SNAPSHOT_VERSION = 1; + +/** + * A cheap identity for a sidecar file: its length and a SHA-256 over that + * length, its first and last 64 KiB and 64 probes of 4 KiB spread over the + * rest. A serializer rewriting the sidecar changes it; a hand edit that keeps + * the length and touches no sampled byte does not, which is why a save that + * writes no snapshot deletes the old one. See DESIGN.md section 11.1. + */ +struct SidecarId { + uint64_t len = 0; + std::array fingerprint{}; + + bool operator==(const SidecarId& other) const { + return len == other.len && fingerprint == other.fingerprint; + } + bool operator!=(const SidecarId& other) const { return !(*this == other); } +}; + +/** What a snapshot is tied to: the graph (by digest) and the JSON sidecar saved with it. */ +struct SnapshotIdentity { + std::optional graphDigest; + std::optional sidecar; +}; + +/** The header of a snapshot, and its size. */ +struct SnapshotInfo { + uint32_t version = SNAPSHOT_VERSION; + size_t dimension = 0; + size_t count = 0; + std::optional graphDigest; + std::optional graphDigestBasis; + std::optional encoderModel; + std::optional encoderVersion; + std::optional sidecar; + /** File size in bytes. */ + uint64_t bytes = 0; +}; + +/** Where a SemanticGraph::load took its embeddings from. */ +enum class LoadSource { + Snapshot, ///< the binary snapshot + Json, ///< the JSON sidecar + NoEmbeddings ///< nothing was restored (embeddings off, or no sidecar) +}; + +/** What SemanticGraph::load did (see its `report` parameter). */ +struct LoadReport { + LoadSource source = LoadSource::NoEmbeddings; + /** + * Why a snapshot that exists was not used (the JSON sidecar was read + * instead); nullopt when it was used or there is none. + */ + std::optional snapshotSkipped; + /** Embeddings restored. */ + size_t embeddings = 0; + /** Wall time of the whole load. */ + double seconds = 0.0; +}; + namespace detail { +/** The scoring of one row. Strict IEEE float arithmetic, one fixed order; see cosineRows. */ +#if defined(__clang__) +#define ISONGRAPH_VECTOR_NOINLINE __attribute__((noinline)) +#elif defined(__GNUC__) +#define ISONGRAPH_VECTOR_NOINLINE __attribute__((noinline, noclone)) +#elif defined(_MSC_VER) +#define ISONGRAPH_VECTOR_NOINLINE __declspec(noinline) +#else +#define ISONGRAPH_VECTOR_NOINLINE +#endif + +/** + * The cosine of two n-float rows: dot / (|a| * |b|), 0 when either norm is 0. + * The three sums run in ascending order in single-precision accumulators. + * + * THE one scoring routine. The exact scan, the batch search, the approximate + * index's re-score, scoreMap and a store loaded from a snapshot all call this + * function and nothing else, so one query/row pair has one score everywhere. + * It is out of line on purpose: two inlined copies are free to be contracted + * (fused multiply-add) or vectorised differently by an optimiser, which showed + * up as last-bit differences on GCC 15. Never compile this with -ffast-math. + */ +ISONGRAPH_VECTOR_NOINLINE inline float cosineRows(const float* a, const float* b, size_t n) { + float dot = 0.0f; + float norm1 = 0.0f; + float norm2 = 0.0f; + + for (size_t i = 0; i < n; ++i) { + dot += a[i] * b[i]; + norm1 += a[i] * a[i]; + norm2 += b[i] * b[i]; + } + + norm1 = std::sqrt(norm1); + norm2 = std::sqrt(norm2); + + if (norm1 == 0.0f || norm2 == 0.0f) return 0.0f; + return dot / (norm1 * norm2); +} + +/** Per-slot data that is not the vector. */ +struct SlotMeta { + std::string key; ///< "type:id" + std::string text; + uint32_t model = 0; ///< index into Table::models +}; + +/** + * The whole store: a slot is a row number, and nothing else owns the floats. + * Slots are in arrival order; a delete tombstones (alive = 0) and compaction + * moves live rows down once a quarter of the slots are dead. + */ +struct Table { + ChunkedMatrix matrix; ///< the one copy of every vector + std::vector meta; + std::vector codes; ///< node type as an index into `types` + std::vector alive; + size_t dead = 0; + std::unordered_map lookup; ///< key -> slot + std::vector types; ///< never shrinks + std::unordered_map typeCodes; + std::vector models; ///< interned model names, never shrinks + std::unordered_map modelIds; + std::vector modelCount; ///< live rows per model + + Table() = default; + Table(const Table&) = default; + Table& operator=(const Table&) = default; + Table(Table&& other) noexcept { swapWith(other); } + Table& operator=(Table&& other) noexcept { + if (this != &other) { + Table fresh; + swapWith(fresh); + swapWith(other); + } + return *this; + } + + void swapWith(Table& other) noexcept { + std::swap(matrix, other.matrix); + meta.swap(other.meta); + codes.swap(other.codes); + alive.swap(other.alive); + std::swap(dead, other.dead); + lookup.swap(other.lookup); + types.swap(other.types); + typeCodes.swap(other.typeCodes); + models.swap(other.models); + modelIds.swap(other.modelIds); + modelCount.swap(other.modelCount); + } + + size_t slots() const { return alive.size(); } + size_t liveCount() const { return alive.size() - dead; } +}; + +/** Warnings already issued by a store (one per distinct model pair); safe to touch from const searches. */ +struct WarnState { + std::mutex mutex; + std::set seen; + + WarnState() = default; + WarnState(const WarnState& other) : seen(other.seen) {} + WarnState(WarnState&& other) noexcept : seen(std::move(other.seen)) {} + WarnState& operator=(const WarnState& other) { + if (this != &other) seen = other.seen; + return *this; + } + WarnState& operator=(WarnState&& other) noexcept { + if (this != &other) seen = std::move(other.seen); + return *this; + } +}; + /** * Everything the store keeps for approximate search. Copying drops the index - * (its slot table points into the source's records) and the counters; moving - * carries the index along, since moving the record map keeps its nodes. + * (it is rebuilt lazily from the copy's own rows) and the counters; moving + * carries the index along. + * + * Searches take `mutex` shared; building, retraining, folding and every write + * take it exclusively. The counters are relaxed atomics. */ struct AnnState { - mutable std::mutex mutex; + mutable std::shared_mutex mutex; std::optional options; std::unique_ptr index; - std::vector recs; // slot -> record, null once dropped - std::unordered_map codes; // node type -> small integer - std::vector hits; // per-query scratch - size_t approximate = 0, widened = 0, exact = 0, builds = 0; + std::atomic approximate{0}, widened{0}, exact{0}, builds{0}; AnnState() = default; AnnState(const AnnState& other) : options(other.options) {} AnnState(AnnState&& other) noexcept { takeFrom(other); } AnnState& operator=(const AnnState& other) { if (this != &other) { - std::lock_guard lock(mutex); - reset(); + std::unique_lock lock(mutex); + index.reset(); options = other.options; - approximate = widened = exact = builds = 0; + approximate = 0; widened = 0; exact = 0; builds = 0; } return *this; } AnnState& operator=(AnnState&& other) noexcept { if (this != &other) { - std::lock_guard lock(mutex); + std::unique_lock lock(mutex); takeFrom(other); } return *this; } - void reset() { - index.reset(); - recs.clear(); - recs.shrink_to_fit(); - codes.clear(); - hits.clear(); - } + void reset() { index.reset(); } private: void takeFrom(AnnState& other) { options = std::move(other.options); index = std::move(other.index); - recs = std::move(other.recs); - codes = std::move(other.codes); - hits = std::move(other.hits); - approximate = other.approximate; widened = other.widened; - exact = other.exact; builds = other.builds; + approximate = other.approximate.load(); + widened = other.widened.load(); + exact = other.exact.load(); + builds = other.builds.load(); other.options.reset(); - other.reset(); + other.index.reset(); + } +}; + +/** Is [p, p+n) well-formed UTF-8 (no overlong forms, no surrogates, nothing above U+10FFFF)? */ +inline bool validUtf8(const unsigned char* p, size_t n) { + size_t i = 0; + while (i < n) { + const unsigned char c = p[i]; + if (c < 0x80) { ++i; continue; } + size_t len; + uint32_t cp; + if (c >= 0xC2 && c <= 0xDF) { len = 2; cp = c & 0x1Fu; } + else if (c >= 0xE0 && c <= 0xEF) { len = 3; cp = c & 0x0Fu; } + else if (c >= 0xF0 && c <= 0xF4) { len = 4; cp = c & 0x07u; } + else return false; + if (i + len > n) return false; + for (size_t k = 1; k < len; ++k) { + if ((p[i + k] & 0xC0u) != 0x80u) return false; + cp = (cp << 6) | (p[i + k] & 0x3Fu); + } + if (len == 3 && (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF))) return false; + if (len == 4 && (cp < 0x10000 || cp > 0x10FFFF)) return false; + i += len; + } + return true; +} + +inline bool hostIsLittleEndian() { + const uint16_t probe = 1; + unsigned char first; + std::memcpy(&first, &probe, 1); + return first == 1; +} + +/** Write n floats as IEEE-754 little-endian bytes. */ +inline void floatsToLe(const float* src, size_t n, unsigned char* dst) { + static_assert(sizeof(float) == 4 && std::numeric_limits::is_iec559, + "snapshots store IEEE-754 binary32"); + if (hostIsLittleEndian()) { + std::memcpy(dst, src, n * 4); + return; + } + for (size_t i = 0; i < n; ++i) { + uint32_t bits; + std::memcpy(&bits, src + i, 4); + for (int b = 0; b < 4; ++b) dst[4 * i + static_cast(b)] = static_cast(bits >> (8 * b)); } +} + +/** Read n floats from IEEE-754 little-endian bytes. */ +inline void floatsFromLe(const unsigned char* src, size_t n, float* dst) { + if (hostIsLittleEndian()) { + std::memcpy(dst, src, n * 4); + return; + } + for (size_t i = 0; i < n; ++i) { + uint32_t bits = 0; + for (int b = 0; b < 4; ++b) bits |= static_cast(src[4 * i + static_cast(b)]) << (8 * b); + std::memcpy(dst + i, &bits, 4); + } +} + +inline void putU32(std::string& out, uint32_t v) { + for (int b = 0; b < 4; ++b) out.push_back(static_cast((v >> (8 * b)) & 0xFFu)); +} +inline void putU64(std::string& out, uint64_t v) { + for (int b = 0; b < 8; ++b) out.push_back(static_cast((v >> (8 * b)) & 0xFFu)); +} +inline void putStr(std::string& out, const std::string& s) { + putU32(out, static_cast(s.size())); + out += s; +} +inline void putOpt(std::string& out, const std::optional& s) { + if (s.has_value()) putStr(out, *s); else putU32(out, 0xFFFFFFFFu); +} + +/** Why a snapshot was refused (as opposed to being unreadable): falls back to the JSON sidecar. */ +class SnapshotRejected : public std::runtime_error { +public: + explicit SnapshotRejected(const std::string& what) : std::runtime_error(what) {} }; +/** Identity of the file at `path`; throws EmbeddingError when it cannot be read. */ +template +inline SidecarId sidecarIdOf(const std::string& path) { + namespace fs = std::filesystem; + std::ifstream file(path, std::ios::binary); + if (!file) throw Err("cannot open file for reading: " + path); + std::error_code ec; + const uint64_t len = static_cast(fs::file_size(fs::path(path), ec)); + if (ec) throw Err("cannot size " + path + ": " + ec.message()); + + constexpr uint64_t kEdge = 64 * 1024, kProbe = 4 * 1024, kProbes = 64; + Sha256 hasher; + unsigned char lenBytes[8]; + for (int b = 0; b < 8; ++b) lenBytes[b] = static_cast((len >> (8 * b)) & 0xFFu); + hasher.update(lenBytes, 8); + std::vector buf; + auto take = [&](uint64_t at, uint64_t n) { + n = std::min(n, len > at ? len - at : uint64_t(0)); + buf.resize(static_cast(n)); + file.clear(); + file.seekg(static_cast(at)); + file.read(buf.data(), static_cast(n)); + if (static_cast(file.gcount()) != n) throw Err("short read of " + path); + hasher.update(buf.data(), buf.size()); + }; + take(0, kEdge); + take(len > kEdge ? len - kEdge : 0, kEdge); + for (uint64_t i = 1; i <= kProbes; ++i) { + // floor(len * i / 65) without overflowing 64 bits + const uint64_t at = (len / (kProbes + 1)) * i + ((len % (kProbes + 1)) * i) / (kProbes + 1); + take(at, kProbe); + } + SidecarId id; + id.len = len; + id.fingerprint = hasher.digest(); + return id; +} + +/** fopen without the MSVC deprecation warning. */ +inline std::FILE* openFile(const std::string& path, const char* mode) { +#if defined(_MSC_VER) + std::FILE* file = nullptr; + if (fopen_s(&file, path.c_str(), mode) != 0) return nullptr; + return file; +#else + return std::fopen(path.c_str(), mode); +#endif +} + +/** Flush a stdio stream's data to the disk where the platform lets us; failure is not an error. */ +inline void syncFile(std::FILE* file) { +#if defined(_WIN32) + _commit(_fileno(file)); +#elif defined(__unix__) || defined(__APPLE__) + ::fsync(fileno(file)); +#else + (void)file; +#endif +} + } // namespace detail // ============================================================================= // Embedding Store // ============================================================================= +/** + * Vectors and their texts, searched by cosine similarity. + * + * Storage: ONE copy of every vector, in a chunked row matrix (1024 rows per + * chunk); a record is a slot number. The approximate index (enableAnn) refers + * to the same rows, it does not copy them. + * + * Thread safety: every const method may be called from any number of threads at + * once, including searches that build or maintain the approximate index (that + * work is serialised internally). Non-const methods (add, remove, clear, + * import*, enableAnn, ...) need exclusive access: do not run one while anything + * else touches the store. A search therefore sees the store before a write or + * after it, never in the middle. + */ class EmbeddingStore { private: - std::unordered_map embeddings_; + detail::Table t_; std::shared_ptr encoder_; std::optional dimension_; std::optional version_; ModelMismatchMode onModelMismatch_ = ModelMismatchMode::Warn; - mutable std::set warnedMismatch_; + mutable detail::WarnState warned_; EmbeddingMeta lastImportMeta_; mutable detail::AnnState ann_; - std::string nodeRefToKey(const NodeRef& ref) const { + using Ivf = detail::IvfIndex; + using IvfHit = detail::IvfHit; + + /** Pending rows that trigger a fold into the approximate index at write time. */ + static constexpr size_t kFoldBatch = 256; + + static std::string nodeRefToKey(const NodeRef& ref) { return ref.type + ":" + ref.id; } @@ -1209,6 +1613,203 @@ class EmbeddingStore { } } + // -- the table -------------------------------------------------------------- + + uint32_t typeCodeFor(const std::string& type) { + auto it = t_.typeCodes.find(type); + if (it != t_.typeCodes.end()) return it->second; + const uint32_t code = static_cast(t_.types.size()); + t_.types.push_back(type); + t_.typeCodes.emplace(type, code); + return code; + } + + uint32_t modelIdFor(const std::string& model) { + auto it = t_.modelIds.find(model); + if (it != t_.modelIds.end()) return it->second; + const uint32_t id = static_cast(t_.models.size()); + t_.models.push_back(model); + t_.modelCount.push_back(0); + t_.modelIds.emplace(model, id); + return id; + } + + NodeRef refOf(uint32_t slot) const { + const std::string& key = t_.meta[slot].key; + const std::string& type = t_.types[t_.codes[slot]]; + return NodeRef(type, key.substr(type.size() + 1)); + } + + std::string_view typeOf(uint32_t slot) const { return t_.types[t_.codes[slot]]; } + std::string_view idOf(uint32_t slot) const { + const std::string& key = t_.meta[slot].key; + return std::string_view(key).substr(t_.types[t_.codes[slot]].size() + 1); + } + + SimilarityResult resultOf(uint32_t slot, float score) const { + return SimilarityResult{refOf(slot), score, t_.meta[slot].text}; + } + + const float* rowOf(uint32_t slot) const { return t_.matrix.row(slot); } + + /** Live slots in the order the JSON sidecar uses: node type, then id (byte order). */ + std::vector liveSlotsByTypeAndId() const { + std::vector slots; + slots.reserve(t_.liveCount()); + for (uint32_t s = 0; s < t_.slots(); ++s) { + if (t_.alive[s]) slots.push_back(s); + } + std::sort(slots.begin(), slots.end(), [this](uint32_t a, uint32_t b) { + const std::string_view ta = typeOf(a), tb = typeOf(b); + if (ta != tb) return ta < tb; + return idOf(a) < idOf(b); + }); + return slots; + } + + /** + * Insert or replace one row: the only way a vector enters the store. Returns + * true when a new slot was added, false when an existing row was overwritten + * in place. `v` has `n` floats, which must equal the store's dimension. + */ + bool putRow(const NodeRef& ref, const float* v, size_t n, const std::string& text, + const std::string& model) { + std::string key = nodeRefToKey(ref); + const uint32_t code = typeCodeFor(ref.type); + const uint32_t modelId = modelIdFor(model); + auto found = t_.lookup.find(key); + uint32_t slot; + bool added; + if (found == t_.lookup.end()) { + if (t_.slots() >= std::numeric_limits::max() - 1) { + throw EmbeddingError("the store cannot hold more than 2^32-2 embeddings"); + } + if (t_.matrix.rows() == 0 && t_.matrix.dim() != n) t_.matrix.reset(n); + const size_t before = t_.slots(); + try { + t_.meta.push_back(detail::SlotMeta{key, text, modelId}); + t_.codes.push_back(code); + t_.alive.push_back(1); + slot = static_cast(t_.matrix.append(v)); + t_.lookup.emplace(std::move(key), slot); + } catch (...) { + t_.meta.resize(before); + t_.codes.resize(before); + t_.alive.resize(before); + if (t_.matrix.rows() > before) t_.matrix.truncate(before); + throw; + } + ++t_.modelCount[modelId]; + added = true; + } else { + slot = found->second; + t_.matrix.set(slot, v); + detail::SlotMeta& meta = t_.meta[slot]; + --t_.modelCount[meta.model]; + meta.text = text; + meta.model = modelId; + ++t_.modelCount[modelId]; + t_.codes[slot] = code; + added = false; + } + annAfterPut(slot, added); + return added; + } + + /** Move live rows down over the tombstones (order preserving) and renumber everything slot-indexed. */ + void compact() { + std::unique_lock lock(ann_.mutex); + const size_t n = t_.slots(); + std::vector remap(n, Ivf::kNoSlot); + size_t kept = 0; + for (size_t i = 0; i < n; ++i) { + if (!t_.alive[i]) continue; + remap[i] = static_cast(kept); + if (kept != i) { + t_.matrix.moveRow(i, kept); + t_.meta[kept] = std::move(t_.meta[i]); + t_.codes[kept] = t_.codes[i]; + t_.lookup[t_.meta[kept].key] = static_cast(kept); + } + ++kept; + } + t_.matrix.truncate(kept); + t_.meta.resize(kept); + t_.codes.resize(kept); + t_.alive.assign(kept, 1); + t_.dead = 0; + t_.meta.shrink_to_fit(); + t_.codes.shrink_to_fit(); + t_.alive.shrink_to_fit(); + if (ann_.index) ann_.index->remap(remap, kept); + } + + // -- scoring ---------------------------------------------------------------- + + /** + * Score the live rows [lo, hi) against `q` (dimension n) and keep the best in + * `heap`: the best `k` by (score desc, slot asc) - a min-heap whose root is + * the worst kept - or, with `all`, every row that passes (the caller sorts). + * Rows of another type than `code` (unless kAnyCode) and scores below + * `threshold` (NaN included) are skipped. + */ + void scanRows(const float* q, size_t n, size_t lo, size_t hi, uint32_t code, float threshold, + size_t k, bool all, std::vector& heap) const { + // [lo, hi) is one chunk (callers pass chunk bounds), so its rows are contiguous + const float* base = t_.matrix.chunkData(lo / detail::kChunkRows); + for (size_t slot = lo; slot < hi; ++slot) { + if (!t_.alive[slot]) continue; + if (code != Ivf::kAnyCode && t_.codes[slot] != code) continue; + const float score = detail::cosineRows(q, base + (slot - lo) * n, n); + if (!(score >= threshold)) continue; + const IvfHit hit{score, static_cast(slot)}; + if (all) { + heap.push_back(hit); + } else if (heap.size() < k) { + heap.push_back(hit); + std::push_heap(heap.begin(), heap.end(), detail::hitBefore); + } else if (detail::hitBefore(hit, heap.front())) { + std::pop_heap(heap.begin(), heap.end(), detail::hitBefore); + heap.back() = hit; + std::push_heap(heap.begin(), heap.end(), detail::hitBefore); + } + } + } + + /** Type filter as a code: false when the type does not exist at all (then nothing matches). */ + bool typeFilter(const std::string& nodeType, uint32_t& code) const { + code = Ivf::kAnyCode; + if (nodeType.empty()) return true; + auto it = t_.typeCodes.find(nodeType); + if (it == t_.typeCodes.end()) return false; + code = it->second; + return true; + } + + std::vector materialize(std::vector& hits, size_t topK) const { + std::sort(hits.begin(), hits.end(), detail::hitBefore); + if (hits.size() > topK) hits.resize(topK); + std::vector results; + results.reserve(hits.size()); + for (const IvfHit& hit : hits) results.push_back(resultOf(hit.slot, hit.score)); + return results; + } + + std::vector exactSearch(const std::vector& query, size_t topK, + const std::string& nodeType, float threshold) const { + uint32_t code; + if (topK == 0 || !typeFilter(nodeType, code)) return {}; + const bool all = topK >= t_.liveCount(); + std::vector heap; + if (!all) heap.reserve(topK); + for (size_t c = 0; c < t_.matrix.chunks(); ++c) { + const size_t lo = c * detail::kChunkRows; + scanRows(query.data(), query.size(), lo, lo + t_.matrix.rowsInChunk(c), code, threshold, + topK, all, heap); + } + return materialize(heap, topK); + } + /** Complain when a text query's encoder is not the stored vectors' model. */ void checkModelCompat() const { if (onModelMismatch_ == ModelMismatchMode::Ignore) return; @@ -1216,9 +1817,10 @@ class EmbeddingStore { if (!current.has_value()) return; std::set foreign; // sorted, which also orders the message - for (const auto& [key, record] : embeddings_) { - if (!isUnknownModel(record.model) && record.model != *current) { - foreign.insert(record.model); + for (size_t id = 0; id < t_.models.size(); ++id) { + const std::string& model = t_.models[id]; + if (t_.modelCount[id] > 0 && !isUnknownModel(model) && model != *current) { + foreign.insert(model); } } if (foreign.empty()) return; @@ -1238,51 +1840,15 @@ class EmbeddingStore { } std::string key = *current + "\n"; for (const std::string& model : foreign) key += model + "\n"; - if (warnedMismatch_.insert(key).second) { - emitWarning(message); - } - } - - // One out-of-line copy on purpose: the exact scan, the approximate path's - // re-score and the tests all compare these scores for bit equality, and - // two inlined copies are free to be contracted or vectorised differently - // by an optimiser (observed as last-bit differences on GCC 15). -#if defined(__clang__) - __attribute__((noinline)) -#elif defined(__GNUC__) - __attribute__((noinline, noclone)) -#elif defined(_MSC_VER) - __declspec(noinline) -#endif - float cosineSimilarity(const std::vector& v1, const std::vector& v2) const { - float dot = 0.0f; - float norm1 = 0.0f; - float norm2 = 0.0f; - - for (size_t i = 0; i < v1.size() && i < v2.size(); ++i) { - dot += v1[i] * v2[i]; - norm1 += v1[i] * v1[i]; - norm2 += v2[i] * v2[i]; + bool first; + { + std::lock_guard lock(warned_.mutex); + first = warned_.seen.insert(key).second; } - - norm1 = std::sqrt(norm1); - norm2 = std::sqrt(norm2); - - if (norm1 == 0.0f || norm2 == 0.0f) return 0.0f; - return dot / (norm1 * norm2); + if (first) emitWarning(message); } - // -- approximate index upkeep (all of it under ann_.mutex) ----------------- - - using Ivf = detail::IvfIndex; - - uint32_t annCodeLocked(const std::string& type) const { - auto it = ann_.codes.find(type); - if (it != ann_.codes.end()) return it->second; - uint32_t code = static_cast(ann_.codes.size()); - ann_.codes.emplace(type, code); - return code; - } + // -- approximate index upkeep --------------------------------------------------- size_t annThreads() const { return ann_.options ? ann_.options->threads : 1; } @@ -1307,115 +1873,101 @@ class EmbeddingStore { return lists * 2 >= ix.nlist ? 0 : lists; } - void annDropIndexLocked() const { ann_.reset(); } + bool annTooSmallLocked() const { + return t_.liveCount() < std::max(2, ann_.options->minSize) || !dimension_.has_value(); + } + + /** Does the index need building, retraining or dropping before it can answer? */ + bool annNeedsMaintenanceLocked() const { + if (!ann_.options) return false; + if (annTooSmallLocked()) return ann_.index != nullptr; + if (!ann_.index) return true; + return static_cast(ann_.index->churn) > 0.5 * static_cast(ann_.index->trainedN); + } void annBuildLocked() const { - std::vector recs; - recs.reserve(embeddings_.size()); - for (const auto& entry : embeddings_) recs.push_back(&entry.second); - // Slots are numbered by key, not by hash-table order, so the same data - // gives the same index however (and on whichever standard library) it - // was inserted. - std::sort(recs.begin(), recs.end(), - [](const EmbeddingRecord* a, const EmbeddingRecord* b) { return a->id < b->id; }); - - const size_t n = recs.size(); - const size_t dim = *dimension_; - std::vector data(n * dim); - std::vector codes(n); - ann_.codes.clear(); - for (size_t i = 0; i < n; ++i) { - const EmbeddingRecord& r = *recs[i]; - if (r.vector.size() != dim) { - throw EmbeddingError("cannot index " + r.id + ": it has " + - std::to_string(r.vector.size()) + " dimensions, store holds " + - std::to_string(dim)); - } - std::copy(r.vector.begin(), r.vector.end(), data.begin() + static_cast(i * dim)); - codes[i] = annCodeLocked(r.node_ref.type); - r.ann_slot = static_cast(i); - } - size_t nlist = ann_.options->nlist ? ann_.options->nlist : AnnOptions::autoNlist(n); - ann_.index = std::make_unique(Ivf::build(dim, std::move(data), std::move(codes), nlist, + std::vector order; + order.reserve(t_.liveCount()); + for (uint32_t s = 0; s < t_.slots(); ++s) { + if (t_.alive[s]) order.push_back(s); + } + // Training sees the rows sorted by key, not in arrival (or hash-table) + // order, so the same data gives the same index however it was inserted. + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return t_.meta[a].key < t_.meta[b].key; + }); + const size_t nlist = ann_.options->nlist ? ann_.options->nlist : AnnOptions::autoNlist(order.size()); + ann_.index = std::make_unique(Ivf::build(t_.matrix, t_.alive, order, nlist, ann_.options->seed, annThreads())); - ann_.recs = std::move(recs); - ++ann_.builds; + ann_.builds.fetch_add(1, std::memory_order_relaxed); } /** - * The index for the store as it is now, built or maintained as the policy - * says; null means "answer exactly". Policy (as in Python): + * Bring the index to the state the policy asks for (caller holds the lock + * exclusively). Policy (as in Python): * - built lazily by the first eligible search, never persisted; - * - new and updated rows are scored exactly until max(512, 2%) of the - * indexed rows are pending, then folded into their nearest cell; + * - new and updated rows are scored exactly until kFoldBatch of them are + * pending, and are then folded into their nearest cell by the write that + * reached that number (annAfterPut); * - deletes tombstone; a quarter dead and the arrays are compacted; * - once writes + deletes since training exceed 50% of the trained size * the index is retrained from the current data; * - a store under minSize, a clear() or a dimension change drops it. */ - Ivf* annIndexLocked() const { - if (!ann_.options) return nullptr; - const size_t live = embeddings_.size(); - if (live < std::max(2, ann_.options->minSize) || !dimension_.has_value()) { - annDropIndexLocked(); - return nullptr; + void annMaintainLocked() const { + if (!ann_.options) return; + if (annTooSmallLocked()) { + ann_.reset(); + return; } if (ann_.index && static_cast(ann_.index->churn) > 0.5 * static_cast(ann_.index->trainedN)) { - annDropIndexLocked(); - } - if (!ann_.index) { - annBuildLocked(); - } else if (static_cast(ann_.index->pendingLive) >= - std::max(512.0, 0.02 * static_cast(ann_.index->trainedN))) { - ann_.index->fold(annThreads()); - } - return ann_.index.get(); - } - - /** A record was written (inserted or replaced). */ - void annAfterPut(const EmbeddingRecord& record) { - std::lock_guard lock(ann_.mutex); - if (!ann_.index) return; - uint32_t code = annCodeLocked(record.node_ref.type); - if (record.ann_slot != Ivf::kNoSlot) { - ann_.index->overwrite(record.ann_slot, record.vector.data(), code); - ann_.recs[record.ann_slot] = &record; - } else { - record.ann_slot = ann_.index->append(record.vector.data(), code); - ann_.recs.push_back(&record); + ann_.reset(); } + if (!ann_.index) annBuildLocked(); } - /** A record is about to be erased. */ - void annBeforeErase(const EmbeddingRecord& record) { - std::lock_guard lock(ann_.mutex); - if (!ann_.index || record.ann_slot == Ivf::kNoSlot) return; - ann_.index->drop(record.ann_slot); - ann_.recs[record.ann_slot] = nullptr; - if (ann_.index->needsCompaction()) { - std::vector remap = ann_.index->compact(); - std::vector kept(ann_.index->slots(), nullptr); - for (size_t old = 0; old < remap.size(); ++old) { - if (remap[old] == Ivf::kNoSlot) continue; - kept[remap[old]] = ann_.recs[old]; - if (kept[remap[old]] != nullptr) kept[remap[old]]->ann_slot = remap[old]; + /** + * The index, ready to search, with a shared lock held on `lock`; null (lock + * released) means "answer exactly". Maintenance, when due, runs under the + * exclusive lock first, so any number of searches share one lazy build. + */ + const Ivf* annAcquire(std::shared_lock& lock) const { + for (;;) { + lock = std::shared_lock(ann_.mutex); + if (!ann_.options) { + lock.unlock(); + return nullptr; + } + if (!annNeedsMaintenanceLocked()) { + if (!ann_.index) { + lock.unlock(); + return nullptr; + } + return ann_.index.get(); } - ann_.recs = std::move(kept); + lock.unlock(); + std::unique_lock exclusive(ann_.mutex); + annMaintainLocked(); } } - /** Insert or replace the record for `key`, keeping the approximate index in step. */ - void putRecord(const std::string& key, EmbeddingRecord&& record) { - auto it = embeddings_.find(key); - uint32_t slot = Ivf::kNoSlot; - if (it == embeddings_.end()) { - it = embeddings_.emplace(key, std::move(record)).first; + /** A row was written (inserted or replaced in place). */ + void annAfterPut(uint32_t slot, bool added) { + std::unique_lock lock(ann_.mutex); + if (!ann_.index) return; + const float* v = t_.matrix.row(slot); + if (added) { + ann_.index->append(v); } else { - slot = it->second.ann_slot; - it->second = std::move(record); + ann_.index->overwrite(slot, v); } - it->second.ann_slot = slot; - annAfterPut(it->second); + if (ann_.index->pendingLive >= kFoldBatch) ann_.index->fold(t_.matrix, annThreads()); + } + + /** A row is about to be tombstoned. */ + void annBeforeErase(uint32_t slot) { + std::unique_lock lock(ann_.mutex); + if (ann_.index) ann_.index->drop(slot); } /** @@ -1430,47 +1982,50 @@ class EmbeddingStore { std::optional> annSearch( const std::vector& query, size_t topK, const std::string& nodeType, float threshold) const { - std::lock_guard lock(ann_.mutex); auto fallBack = [&]() -> std::optional> { - ++ann_.exact; + ann_.exact.fetch_add(1, std::memory_order_relaxed); return std::nullopt; }; - Ivf* ix = annIndexLocked(); + std::shared_lock lock; + const Ivf* ix = annAcquire(lock); if (ix == nullptr) return fallBack(); const float norm = std::sqrt(detail::simd::dot(query.data(), query.data(), query.size())); if (!(norm > 0.0f)) return fallBack(); // a zero query has no direction to rank cells by if (topK >= ix->live()) return fallBack(); // nothing to narrow down - uint32_t code = Ivf::kAnyCode; - if (!nodeType.empty()) { - auto found = ann_.codes.find(nodeType); - if (found == ann_.codes.end()) return std::vector(); // no such type at all - code = found->second; - } + uint32_t code; + if (!typeFilter(nodeType, code)) return std::vector(); // no such type at all // The fast scores differ from the exact ones in the last bits: admit // rows that close to the threshold and re-check them below. const float slack = threshold - 1e-5f; + std::vector hits; for (size_t scale = 1;; scale *= 4) { size_t lists = annListsLocked(*ix, scale); if (lists == 0) return fallBack(); - ix->search(query.data(), 1.0f / norm, code, slack, lists, topK, ann_.hits); - if (ann_.hits.size() < topK) continue; - std::vector results; - results.reserve(ann_.hits.size()); - for (const detail::IvfHit& hit : ann_.hits) { - const EmbeddingRecord& r = *ann_.recs[hit.slot]; - float score = cosineSimilarity(query, r.vector); - if (score >= threshold) results.push_back(SimilarityResult{r.node_ref, score, r.text}); + ix->search(t_.matrix, t_.codes.data(), query.data(), 1.0f / norm, code, slack, lists, topK, hits); + if (hits.size() < topK) continue; + std::vector exact; + exact.reserve(hits.size()); + for (const IvfHit& hit : hits) { + float score = detail::cosineRows(query.data(), rowOf(hit.slot), query.size()); + if (score >= threshold) exact.push_back(IvfHit{score, hit.slot}); } - if (results.size() >= topK) { - std::stable_sort(results.begin(), results.end()); - ++ann_.approximate; - if (scale > 1) ++ann_.widened; - return results; + if (exact.size() >= topK) { + ann_.approximate.fetch_add(1, std::memory_order_relaxed); + if (scale > 1) ann_.widened.fetch_add(1, std::memory_order_relaxed); + return materialize(exact, topK); } } } + /** Threads for a batch: what was asked (0 = one per hardware thread), at most 256 and at most the tasks. */ + static size_t clampWorkers(size_t requested, size_t tasks) { + size_t workers = requested == 0 ? detail::autoThreads() : requested; + workers = std::min(workers, 256); + workers = std::min(workers, tasks); + return std::max(workers, 1); + } + public: /** * @param encoder encoder for text; may be null when only vectors are added @@ -1514,7 +2069,9 @@ class EmbeddingStore { */ std::map models() const { std::map counts; - for (const auto& [key, record] : embeddings_) ++counts[record.model]; + for (size_t id = 0; id < t_.models.size(); ++id) { + if (t_.modelCount[id] > 0) counts[t_.models[id]] = t_.modelCount[id]; + } return counts; } @@ -1543,11 +2100,13 @@ class EmbeddingStore { if (!dimension.has_value() && encoder_) dimension = encoder_->dimension(); std::vector out; - for (const auto& [key, record] : embeddings_) { - bool byModel = model.has_value() && record.model != *model && - (includeUnknown || !isUnknownModel(record.model)); - bool byDim = dimension.has_value() && record.vector.size() != *dimension; - if (byModel || byDim) out.push_back(record.node_ref); + for (uint32_t slot = 0; slot < t_.slots(); ++slot) { + if (!t_.alive[slot]) continue; + const std::string& recordModel = t_.models[t_.meta[slot].model]; + bool byModel = model.has_value() && recordModel != *model && + (includeUnknown || !isUnknownModel(recordModel)); + bool byDim = dimension.has_value() && t_.matrix.dim() != *dimension; + if (byModel || byDim) out.push_back(refOf(slot)); } std::sort(out.begin(), out.end()); return out; @@ -1571,18 +2130,19 @@ class EmbeddingStore { std::string add(const NodeRef& node_ref, const std::string& text, const std::vector& vector = {}) { - std::vector vec = vector; - if (vec.empty()) { + std::vector encoded; + const std::vector* vec = &vector; + if (vector.empty()) { if (!encoder_) { throw NoEncoderError("No encoder provided and no vector given"); } - vec = encoder_->encode(text); + encoded = encoder_->encode(text); + vec = &encoded; } - checkDimension(vec, "vector"); + checkDimension(*vec, "vector"); - std::string key = nodeRefToKey(node_ref); - putRecord(key, EmbeddingRecord{key, node_ref, std::move(vec), text, currentModelStamp()}); - return key; + putRow(node_ref, vec->data(), vec->size(), text, currentModelStamp()); + return nodeRefToKey(node_ref); } /** Add many embeddings, encoding their texts in one batched call. */ @@ -1610,25 +2170,54 @@ class EmbeddingStore { return entries.size(); } - const EmbeddingRecord* get(const NodeRef& node_ref) const { - std::string key = nodeRefToKey(node_ref); - auto it = embeddings_.find(key); - if (it != embeddings_.end()) { - return &it->second; - } - return nullptr; + /** + * The stored embedding for `node_ref`, or nullopt. The record is a COPY of + * the row (the store keeps no record objects); see vectorData() to read the + * floats without copying. + */ + std::optional get(const NodeRef& node_ref) const { + auto it = t_.lookup.find(nodeRefToKey(node_ref)); + if (it == t_.lookup.end()) return std::nullopt; + const uint32_t slot = it->second; + const float* row = rowOf(slot); + return EmbeddingRecord{t_.meta[slot].key, refOf(slot), + std::vector(row, row + t_.matrix.dim()), + t_.meta[slot].text, t_.models[t_.meta[slot].model]}; + } + + /** Is there an embedding for `node_ref`? */ + bool contains(const NodeRef& node_ref) const { + return t_.lookup.find(nodeRefToKey(node_ref)) != t_.lookup.end(); + } + + /** + * A view of the stored vector (dimension() floats), or nullptr. Valid only + * until the next write to the store (add, remove, clear, import, ...), which + * may overwrite or move the row; copy it if you need it longer. + */ + const float* vectorData(const NodeRef& node_ref) const { + auto it = t_.lookup.find(nodeRefToKey(node_ref)); + return it == t_.lookup.end() ? nullptr : rowOf(it->second); } bool remove(const NodeRef& node_ref) { std::string key = nodeRefToKey(node_ref); - auto it = embeddings_.find(key); - if (it == embeddings_.end()) return false; - annBeforeErase(it->second); - embeddings_.erase(it); + auto it = t_.lookup.find(key); + if (it == t_.lookup.end()) return false; + const uint32_t slot = it->second; + annBeforeErase(slot); + --t_.modelCount[t_.meta[slot].model]; + t_.lookup.erase(it); + t_.alive[slot] = 0; + t_.meta[slot] = detail::SlotMeta{}; + ++t_.dead; + if (t_.dead >= 64 && t_.dead * 4 > t_.slots()) compact(); return true; } /** + * Top-k cosine matches of a text query (needs an encoder). + * * @param exact skip the approximate index and scan every vector (no effect * unless enableAnn() was called) */ @@ -1646,7 +2235,8 @@ class EmbeddingStore { /** * Cosine top-k of a query vector. With enableAnn() a bounded topK on a * large enough store is answered from the approximate index (see there); - * `exact` forces the full scan. + * `exact` forces the full scan. Equal scores are ordered by slot (arrival + * order), so an answer is reproducible. */ std::vector similaritySearchVector( const std::vector& queryVector, @@ -1659,44 +2249,134 @@ class EmbeddingStore { if (topK > 0) { // A store without enableAnn() only reads its (empty) options under a - // brief lock here; the scan below is what it always was. + // brief shared lock here; the scan below is what it always was. bool enabled; { - std::lock_guard lock(ann_.mutex); + std::shared_lock lock(ann_.mutex); enabled = ann_.options.has_value(); - if (enabled && exact) ++ann_.exact; } + if (enabled && exact) ann_.exact.fetch_add(1, std::memory_order_relaxed); if (enabled && !exact) { if (auto answer = annSearch(queryVector, topK, nodeType, threshold)) { return std::move(*answer); } } } + return exactSearch(queryVector, topK, nodeType, threshold); + } - std::vector results; + /** + * Many queries at once, in parallel; entry i of the result is exactly what + * similaritySearchVector(queries[i], ...) returns - the same ids, the same + * scores bit for bit, in the same order - whatever the number of threads. + * + * The exact scan is shared across queries block by block (each 1024-row + * chunk is scored against a tile of 16 queries while it is in cache); with + * the approximate index on (and `exact` false) each query is answered by + * the index on its own thread. The calling thread is one of the workers; + * the others are started and joined inside this call, so nothing outlives + * it, and an exception thrown by any worker is rethrown here after all have + * stopped. Every query is validated before any work starts. + * + * @param threads upper bound on threads used (0 = hardware concurrency; + * never more than 256, or than there is work for) + */ + std::vector> similaritySearchBatch( + const std::vector>& queries, + size_t topK = 10, + const std::string& nodeType = "", + float threshold = 0.0f, + bool exact = false, + size_t threads = 0 + ) const { + for (const auto& query : queries) checkQuery(query); + std::vector> out(queries.size()); + if (queries.empty() || topK == 0) return out; + + const bool annOn = annEnabled(); + const bool everyRow = topK >= t_.liveCount(); + + if ((annOn && !exact) || everyRow) { + // One query per task, each through the single-query path. + std::atomic next{0}; + std::atomic stop{false}; + detail::runWorkers(clampWorkers(threads, queries.size()), [&](size_t) { + for (;;) { + const size_t i = next.fetch_add(1); + if (i >= queries.size() || stop.load()) break; + try { + out[i] = similaritySearchVector(queries[i], topK, nodeType, threshold, exact); + } catch (...) { + stop = true; + throw; + } + } + }); + return out; + } - for (const auto& [key, record] : embeddings_) { - if (!nodeType.empty() && record.node_ref.type != nodeType) { - continue; + // (the per-query path above counts its exact scans itself) + if (annOn && exact) ann_.exact.fetch_add(queries.size(), std::memory_order_relaxed); + uint32_t code; + if (!typeFilter(nodeType, code)) return out; + + constexpr size_t kTile = 16; + const size_t nq = queries.size(); + const size_t tiles = (nq + kTile - 1) / kTile; + const size_t chunks = t_.matrix.chunks(); + const size_t tasks = chunks * tiles; + const size_t workers = clampWorkers(threads, tasks); + const size_t dim = t_.matrix.dim(); + + std::vector>> perWorker(workers); + std::atomic next{0}; + std::atomic stop{false}; + detail::runWorkers(workers, [&](size_t w) { + std::vector> heaps(nq); + try { + for (;;) { + const size_t task = next.fetch_add(1); + if (task >= tasks || stop.load()) break; + // chunk-major: a chunk is scored against all tiles while it is cached + const size_t chunk = task / tiles, tile = task % tiles; + const size_t lo = chunk * detail::kChunkRows; + const size_t hi = lo + t_.matrix.rowsInChunk(chunk); + const size_t qa = tile * kTile, qb = std::min(nq, qa + kTile); + for (size_t qi = qa; qi < qb; ++qi) { + scanRows(queries[qi].data(), dim, lo, hi, code, threshold, topK, false, heaps[qi]); + } + } + } catch (...) { + stop = true; + throw; } + perWorker[w] = std::move(heaps); + }); - float score = cosineSimilarity(queryVector, record.vector); - if (score >= threshold) { - results.push_back(SimilarityResult{record.node_ref, score, record.text}); + for (size_t qi = 0; qi < nq; ++qi) { + std::vector merged; + for (const auto& heaps : perWorker) { + if (heaps.empty()) continue; + merged.insert(merged.end(), heaps[qi].begin(), heaps[qi].end()); } + out[qi] = materialize(merged, topK); } + return out; + } - // Selecting the top k rather than ordering every candidate: this is - // O(n log k) where a full sort is O(n log n), and returns the same - // rows in the same order. - if (results.size() > topK) { - std::partial_sort(results.begin(), results.begin() + topK, results.end()); - results.resize(topK); - } else { - std::sort(results.begin(), results.end()); - } - - return results; + /** similaritySearchBatch() for text queries (encoded one by one, in order, before any search starts). */ + std::vector> similaritySearchTextBatch( + const std::vector& queries, + size_t topK = 10, + const std::string& nodeType = "", + float threshold = 0.0f, + bool exact = false, + size_t threads = 0 + ) const { + std::vector> vectors; + vectors.reserve(queries.size()); + for (const auto& query : queries) vectors.push_back(encodeQuery(query)); + return similaritySearchBatch(vectors, topK, nodeType, threshold, exact, threads); } /** @@ -1710,9 +2390,13 @@ class EmbeddingStore { checkQuery(queryVector); std::map scores; - for (const auto& [key, record] : embeddings_) { - if (!nodeType.empty() && record.node_ref.type != nodeType) continue; - scores[key] = cosineSimilarity(queryVector, record.vector); + uint32_t code; + if (!typeFilter(nodeType, code)) return scores; + for (uint32_t slot = 0; slot < t_.slots(); ++slot) { + if (!t_.alive[slot]) continue; + if (code != Ivf::kAnyCode && t_.codes[slot] != code) continue; + scores[t_.meta[slot].key] = + detail::cosineRows(queryVector.data(), rowOf(slot), queryVector.size()); } return scores; } @@ -1739,32 +2423,33 @@ class EmbeddingStore { * unbounded topK (one that asks for every row), a zero query vector, or a * store under minSize use the exact scan. * - * The index keeps its own contiguous copy of the vectors (see - * AnnStats::vectorBytes), so an indexed store holds them twice. - * Concurrent searches are safe; they take turns on the index. Calling this - * again replaces the options and drops the current index. + * The index holds NO copy of the vectors: it refers to the store's one + * matrix (AnnStats::vectorBytes is 0). Pending writes are folded into their + * cells by the write that makes 256 of them. Concurrent searches are safe + * and run in parallel; a lazy build or retrain makes them wait for it. + * Calling this again replaces the options and drops the current index. */ void enableAnn(AnnOptions options = AnnOptions()) { - std::lock_guard lock(ann_.mutex); + std::unique_lock lock(ann_.mutex); ann_.reset(); ann_.options = options; } /** Turn approximate search off and free the index. */ void disableAnn() { - std::lock_guard lock(ann_.mutex); + std::unique_lock lock(ann_.mutex); ann_.reset(); ann_.options.reset(); } bool annEnabled() const { - std::lock_guard lock(ann_.mutex); + std::shared_lock lock(ann_.mutex); return ann_.options.has_value(); } /** Cells scanned per query from now on (0 = automatic). Takes effect immediately; no rebuild. */ void setAnnNprobe(size_t nprobe) { - std::lock_guard lock(ann_.mutex); + std::unique_lock lock(ann_.mutex); if (!ann_.options) throw EmbeddingError("this store has no approximate index; call enableAnn() first"); ann_.options->nprobe = nprobe; } @@ -1778,25 +2463,25 @@ class EmbeddingStore { */ AnnStats buildAnnIndex(bool force = false) { { - std::lock_guard lock(ann_.mutex); + std::unique_lock lock(ann_.mutex); if (!ann_.options) throw EmbeddingError("this store has no approximate index; call enableAnn() first"); if (force) ann_.reset(); - Ivf* ix = annIndexLocked(); - if (ix != nullptr && ix->pendingLive > 0) ix->fold(annThreads()); + annMaintainLocked(); + if (ann_.index && ann_.index->pendingLive > 0) ann_.index->fold(t_.matrix, annThreads()); } return annStats(); } /** What the approximate index looks like right now. Never builds anything. */ AnnStats annStats() const { - std::lock_guard lock(ann_.mutex); + std::shared_lock lock(ann_.mutex); AnnStats stats; stats.enabled = ann_.options.has_value(); stats.minSize = ann_.options ? ann_.options->minSize : 0; - stats.approximateSearches = ann_.approximate; - stats.widenedSearches = ann_.widened; - stats.exactSearches = ann_.exact; - stats.builds = ann_.builds; + stats.approximateSearches = ann_.approximate.load(); + stats.widenedSearches = ann_.widened.load(); + stats.exactSearches = ann_.exact.load(); + stats.builds = ann_.builds.load(); const Ivf* ix = ann_.index.get(); stats.active = ix != nullptr; if (ix != nullptr) { @@ -1807,7 +2492,7 @@ class EmbeddingStore { stats.churn = ix->churn; stats.buildSeconds = ix->buildSeconds; stats.indexBytes = ix->overheadBytes(); - stats.vectorBytes = ix->vectorBytes(); + stats.vectorBytes = 0; } else if (ann_.options) { stats.nprobe = ann_.options->nprobe; } @@ -1890,40 +2575,84 @@ class EmbeddingStore { bool annVisit(const std::vector& query, const std::string& nodeType, float threshold, size_t scale, const std::function& visit) const { checkQuery(query); - std::vector order; + std::vector order; { - std::lock_guard lock(ann_.mutex); - Ivf* ix = annIndexLocked(); + std::shared_lock lock; + const Ivf* ix = annAcquire(lock); if (ix == nullptr) return false; const float norm = std::sqrt(detail::simd::dot(query.data(), query.data(), query.size())); if (!(norm > 0.0f)) return false; - uint32_t code = Ivf::kAnyCode; - if (!nodeType.empty()) { - auto found = ann_.codes.find(nodeType); - if (found == ann_.codes.end()) return true; // nothing of that type exists - code = found->second; - } + uint32_t code; + if (!typeFilter(nodeType, code)) return true; // nothing of that type exists size_t lists = annListsLocked(*ix, scale); if (lists == 0) return false; - ix->candidates(query.data(), 1.0f / norm, code, threshold - 1e-5f, lists, ann_.hits); - order.reserve(ann_.hits.size()); - for (const detail::IvfHit& hit : ann_.hits) order.push_back(ann_.recs[hit.slot]); - ++ann_.approximate; - if (scale > 1) ++ann_.widened; + std::vector hits; + ix->candidates(t_.matrix, t_.codes.data(), query.data(), 1.0f / norm, code, + threshold - 1e-5f, lists, hits); + order.reserve(hits.size()); + for (const IvfHit& hit : hits) order.push_back(hit.slot); + ann_.approximate.fetch_add(1, std::memory_order_relaxed); + if (scale > 1) ann_.widened.fetch_add(1, std::memory_order_relaxed); } // Outside the lock: `visit` is user code and may search this store again. - for (const EmbeddingRecord* r : order) { - float score = cosineSimilarity(query, r->vector); + for (uint32_t slot : order) { + float score = detail::cosineRows(query.data(), rowOf(slot), query.size()); if (!(score >= threshold)) continue; - if (!visit(SimilarityResult{r->node_ref, score, r->text})) break; + if (!visit(resultOf(slot, score))) break; } return true; } size_t count() const { - return embeddings_.size(); + return t_.lookup.size(); + } + + /** + * Check the store's internal invariants (DESIGN.md section 7): parallel + * arrays all hold one entry per slot, the matrix holds exactly that many + * rows, `dead` counts the tombstones, the key table has one entry per live + * slot and each entry's key is `type:id` of its slot, per-model counts equal + * the rows carrying each model, and an approximate index (if any) has one + * slot per table slot. Throws EmbeddingError naming the first violation. + * For tests and diagnostics; O(n). + */ + void verifyInvariants() const { + auto fail = [](const std::string& what) { throw EmbeddingError("invariant violated: " + what); }; + const size_t n = t_.slots(); + if (t_.meta.size() != n || t_.codes.size() != n) fail("parallel arrays differ in length"); + if (t_.matrix.rows() != n) fail("matrix rows != slots"); + size_t dead = 0; + std::vector perModel(t_.models.size(), 0); + for (size_t slot = 0; slot < n; ++slot) { + if (!t_.alive[slot]) { ++dead; continue; } + const detail::SlotMeta& meta = t_.meta[slot]; + auto it = t_.lookup.find(meta.key); + if (it == t_.lookup.end() || it->second != slot) fail("key table does not map " + meta.key + " to its slot"); + if (t_.codes[slot] >= t_.types.size()) fail("type code out of range"); + const std::string& type = t_.types[t_.codes[slot]]; + if (meta.key.size() < type.size() + 1 || meta.key.compare(0, type.size(), type) != 0 || + meta.key[type.size()] != ':') { + fail("key " + meta.key + " is not type:id of its slot"); + } + if (meta.model >= perModel.size()) fail("model index out of range"); + ++perModel[meta.model]; + } + if (dead != t_.dead) fail("dead count is wrong"); + if (t_.lookup.size() != n - dead) fail("key table size != live rows"); + for (size_t m = 0; m < perModel.size(); ++m) { + if (perModel[m] != t_.modelCount[m]) fail("model count of '" + t_.models[m] + "' is wrong"); + } + if (n > 0 && dimension_.has_value() && t_.matrix.dim() != *dimension_) fail("matrix width != dimension"); + std::shared_lock lock(ann_.mutex); + if (ann_.index) { + if (ann_.index->slots() != n) fail("approximate index has a different slot count"); + if (ann_.index->live() != n - dead) fail("approximate index has a different live count"); + } } + /** Bytes held by the vector matrix (the one copy of every vector, tombstoned rows included). */ + size_t vectorBytes() const { return t_.matrix.bytes(); } + /** * Clear all embeddings. * @@ -1935,10 +2664,16 @@ class EmbeddingStore { */ void clear(bool resetDimension = false) { { - std::lock_guard lock(ann_.mutex); + std::unique_lock lock(ann_.mutex); ann_.reset(); } - embeddings_.clear(); + t_.matrix.reset(resetDimension ? 0 : t_.matrix.dim()); + std::vector().swap(t_.meta); + std::vector().swap(t_.codes); + std::vector().swap(t_.alive); + t_.dead = 0; + std::unordered_map().swap(t_.lookup); + std::fill(t_.modelCount.begin(), t_.modelCount.end(), size_t(0)); if (resetDimension) dimension_.reset(); } @@ -1953,24 +2688,22 @@ class EmbeddingStore { const std::optional& graphDigest = std::nullopt) const { ExportedEmbeddings payload; payload.dimension = dimension_; - payload.embeddings.reserve(embeddings_.size()); + const std::vector slots = liveSlotsByTypeAndId(); + payload.embeddings.reserve(slots.size()); - for (const auto& [key, record] : embeddings_) { + for (uint32_t slot : slots) { + const float* row = rowOf(slot); payload.embeddings.push_back(ExportedEmbedding{ - record.node_ref.type, - record.node_ref.id, + std::string(typeOf(slot)), + std::string(idOf(slot)), // C++ node ids are always strings; the other ports use this to // tell a numeric id from a numeric-looking string one. "str", - record.text, - record.vector, - record.model + t_.meta[slot].text, + std::vector(row, row + t_.matrix.dim()), + t_.models[t_.meta[slot].model] }); } - std::sort(payload.embeddings.begin(), payload.embeddings.end(), - [](const ExportedEmbedding& a, const ExportedEmbedding& b) { - return std::tie(a.type, a.id) < std::tie(b.type, b.id); - }); payload.count = payload.embeddings.size(); payload.meta.encoderModel = currentModel(); @@ -2005,9 +2738,8 @@ class EmbeddingStore { size_t written = 0; for (const auto& item : payload.embeddings) { checkDimension(item.vector, "embedding for :" + item.type + ":" + item.id); - std::string key = nodeRefToKey(NodeRef(item.type, item.id)); - putRecord(key, EmbeddingRecord{key, NodeRef(item.type, item.id), item.vector, - item.text, item.model}); + putRow(NodeRef(item.type, item.id), item.vector.data(), item.vector.size(), item.text, + item.model); ++written; } return written; @@ -2196,7 +2928,7 @@ class EmbeddingStore { size_t saveEmbeddings(const std::string& path, const std::optional& graphDigest = std::nullopt) const { detail::atomicWriteText(path, toJson(graphDigest)); - return embeddings_.size(); + return count(); } /** Read a payload file written by saveEmbeddings (or any port's equivalent). */ @@ -2215,7 +2947,375 @@ class EmbeddingStore { return content.str(); } + // -- binary snapshot ------------------------------------------------------ + + /** + * Identity of the file at `path` (see SidecarId); throws EmbeddingError when + * it cannot be read. + */ + static SidecarId sidecarIdOf(const std::string& path) { + return detail::sidecarIdOf(path); + } + + /** + * Write a binary snapshot of every embedding to `path` (DESIGN.md section + * 11.2): a temp file beside it is written, flushed to disk and renamed over + * `path`, so a failure or crash leaves the previous file untouched and no + * temp file behind. Rows go in the order the JSON sidecar uses (node type, + * then id), so loading the snapshot gives the same store, slot for slot, as + * loading the sidecar. All integers are little-endian whatever the host. + * + * `identity` ties the snapshot to the graph and sidecar it was saved with; + * SemanticGraph::load refuses a snapshot whose identity differs. + */ + SnapshotInfo saveSnapshot(const std::string& path, const SnapshotIdentity& identity = {}) const { + namespace fs = std::filesystem; + static std::atomic counter{0}; + fs::path target(path); + fs::path dir = target.parent_path(); + std::random_device rd; + std::string tmpName = "." + target.filename().string() + "." + + std::to_string(static_cast( + std::chrono::steady_clock::now().time_since_epoch().count())) + "." + + std::to_string(rd()) + "." + std::to_string(counter++) + ".tmp"; + fs::path tmp = dir.empty() ? fs::path(tmpName) : dir / tmpName; + + std::FILE* file = nullptr; + try { + file = detail::openFile(tmp.string(), "wb"); + if (file == nullptr) throw EmbeddingError("cannot open snapshot for writing: " + path); + std::setvbuf(file, nullptr, _IOFBF, 4u << 20); + SnapshotInfo info = writeSnapshot(file, identity); + if (std::fflush(file) != 0) throw EmbeddingError("failed writing snapshot: " + path); + detail::syncFile(file); + const int closed = std::fclose(file); + file = nullptr; + if (closed != 0) throw EmbeddingError("failed writing snapshot: " + path); + std::error_code ec; + fs::rename(tmp, target, ec); + if (ec) throw EmbeddingError("cannot replace " + path + ": " + ec.message()); + return info; + } catch (...) { + if (file != nullptr) std::fclose(file); + std::error_code ignored; + fs::remove(tmp, ignored); + throw; + } + } + + /** + * Replace the store's embeddings with those of the snapshot at `path`. + * Nothing changes unless the whole file verifies (structure and checksum): + * it is read into a staging store, and a failure throws EmbeddingError with + * the store untouched. The encoder, version, model-mismatch mode and ANN + * settings are kept (an existing ANN index is dropped and rebuilt lazily). + * Does not check identity; SemanticGraph::load does that. + */ + SnapshotInfo loadSnapshot(const std::string& path) { + try { + EmbeddingStore stage; + SnapshotInfo info = readSnapshot(path, nullptr, stage); + adopt(std::move(stage), info); + return info; + } catch (const detail::SnapshotRejected& rejected) { + throw EmbeddingError(std::string("invalid snapshot: ") + rejected.what()); + } + } + + /** + * Use the snapshot at `snap` for the graph whose digest is `graphDigest` and + * whose JSON sidecar is at `sidecar`, if it provably belongs to them (same + * digest and basis, same sidecar length and fingerprint) and verifies. + * Returns the reason when it was not used (the store is then untouched, and + * the caller falls back to the sidecar), or nullopt after adopting it; + * `info` receives the header on success. + */ + std::optional tryLoadSnapshotFor(const std::string& snap, const std::string& sidecar, + const std::string& graphDigest, + SnapshotInfo* info = nullptr) { + try { + SidecarId current; + try { + current = sidecarIdOf(sidecar); + } catch (const EmbeddingError& e) { + return std::string("cannot read the sidecar ") + sidecar + ": " + e.what(); + } + EmbeddingStore stage; + SnapshotInfo header = readSnapshot(snap, [&](const SnapshotInfo& h) -> std::optional { + if (!h.graphDigest.has_value() || *h.graphDigest != graphDigest || + !h.graphDigestBasis.has_value() || *h.graphDigestBasis != std::string(GRAPH_DIGEST_BASIS)) { + return std::string("it was saved for a different graph (digest differs)"); + } + if (!h.sidecar.has_value()) return std::string("it records no sidecar identity"); + if (*h.sidecar != current) { + return std::string("the JSON sidecar changed since the snapshot was written (stale snapshot)"); + } + return std::nullopt; + }, stage); + adopt(std::move(stage), header); + if (info != nullptr) *info = header; + return std::nullopt; + } catch (const detail::SnapshotRejected& rejected) { + return std::string(rejected.what()); + } catch (const EmbeddingError& e) { + return std::string(e.what()); + } + } + private: + /** Install a verified staging store's rows in this store. */ + void adopt(EmbeddingStore&& stage, const SnapshotInfo& info) { + { + std::unique_lock lock(ann_.mutex); + ann_.reset(); + } + t_ = std::move(stage.t_); + dimension_ = stage.dimension_; + lastImportMeta_ = EmbeddingMeta{info.encoderModel, info.encoderVersion, info.graphDigest, + info.graphDigestBasis}; + } + + /** Add one snapshot row to a staging store; false for a repeated key. */ + bool putSnapshotRow(const std::string& type, const std::string& id, const std::string& text, + const std::string& model, const float* v, size_t n) { + if (t_.lookup.find(type + ":" + id) != t_.lookup.end()) return false; + if (!dimension_.has_value()) dimension_ = n; + putRow(NodeRef(type, id), v, n, text, model); + return true; + } + + SnapshotInfo writeSnapshot(std::FILE* file, const SnapshotIdentity& identity) const { + constexpr uint32_t kBlockRows = 1024; + const std::vector rows = liveSlotsByTypeAndId(); + const size_t count = rows.size(); + const size_t dim = count > 0 ? t_.matrix.dim() : dimension_.value_or(0); + + // models in order of first use + std::vector models; + std::unordered_map modelIndex; + for (uint32_t slot : rows) { + const uint32_t id = t_.meta[slot].model; + if (modelIndex.emplace(id, static_cast(models.size())).second) { + models.push_back(t_.models[id]); + } + } + + SnapshotInfo info; + info.version = SNAPSHOT_VERSION; + info.dimension = dim; + info.count = count; + info.graphDigest = identity.graphDigest; + if (identity.graphDigest.has_value()) info.graphDigestBasis = std::string(GRAPH_DIGEST_BASIS); + info.encoderModel = currentModel(); + info.encoderVersion = version(); + info.sidecar = identity.sidecar; + + std::string head = "ISONVSNP"; + detail::putU32(head, SNAPSHOT_VERSION); + detail::putU32(head, identity.sidecar.has_value() ? 1u : 0u); + detail::putU32(head, static_cast(dim)); + detail::putU32(head, kBlockRows); + detail::putU64(head, static_cast(count)); + detail::putOpt(head, info.graphDigest); + detail::putOpt(head, info.graphDigestBasis); + detail::putOpt(head, info.encoderModel); + detail::putOpt(head, info.encoderVersion); + if (identity.sidecar.has_value()) { + detail::putU64(head, identity.sidecar->len); + head.append(reinterpret_cast(identity.sidecar->fingerprint.data()), 32); + } + detail::putU32(head, static_cast(models.size())); + for (const std::string& m : models) detail::putStr(head, m); + + detail::Sha256 hasher; + uint64_t written = 0; + auto emit = [&](const void* data, size_t n) { + if (n == 0) return; + if (std::fwrite(data, 1, n, file) != n) throw EmbeddingError("failed writing snapshot"); + hasher.update(data, n); + written += n; + }; + emit(head.data(), head.size()); + + std::string meta; + std::vector floats; + for (size_t start = 0; start < count; start += kBlockRows) { + const size_t end = std::min(count, start + kBlockRows); + meta.clear(); + for (size_t i = start; i < end; ++i) { + const uint32_t slot = rows[i]; + detail::putStr(meta, std::string(typeOf(slot))); + detail::putStr(meta, std::string(idOf(slot))); + detail::putStr(meta, t_.meta[slot].text); + detail::putU32(meta, modelIndex.at(t_.meta[slot].model)); + } + emit(meta.data(), meta.size()); + floats.resize((end - start) * dim * 4); + for (size_t i = start; i < end; ++i) { + detail::floatsToLe(rowOf(rows[i]), dim, floats.data() + (i - start) * dim * 4); + } + emit(floats.data(), floats.size()); + } + const auto digest = hasher.digest(); + if (std::fwrite(digest.data(), 1, digest.size(), file) != digest.size()) { + throw EmbeddingError("failed writing snapshot"); + } + info.bytes = written + digest.size(); + return info; + } + + /** + * Read and fully verify the snapshot at `path` into `stage` (an empty store), + * after `accept` (if any) has approved its header. Throws SnapshotRejected for + * a file that is not a valid snapshot of the right identity, EmbeddingError + * when it cannot be read at all. + */ + static SnapshotInfo readSnapshot( + const std::string& path, + const std::function(const SnapshotInfo&)>& accept, + EmbeddingStore& stage) { + namespace fs = std::filesystem; + constexpr uint64_t kTrailer = 32; + std::error_code ec; + const uint64_t fileLen = static_cast(fs::file_size(fs::path(path), ec)); + if (ec) throw EmbeddingError("cannot open snapshot " + path + ": " + ec.message()); + std::FILE* file = detail::openFile(path, "rb"); + if (file == nullptr) throw EmbeddingError("cannot open snapshot " + path); + struct Closer { + std::FILE* f; + ~Closer() { if (f != nullptr) std::fclose(f); } + } closer{file}; + std::setvbuf(file, nullptr, _IOFBF, 4u << 20); + if (fileLen < kTrailer + 8) throw detail::SnapshotRejected("file too short"); + + detail::Sha256 hasher; + uint64_t consumed = 0; + const uint64_t body = fileLen - kTrailer; + auto remaining = [&]() { return body - consumed; }; + auto exact = [&](void* buf, size_t n) { + if (n > remaining()) throw detail::SnapshotRejected("truncated"); + if (n > 0 && std::fread(buf, 1, n, file) != n) throw detail::SnapshotRejected("truncated"); + hasher.update(buf, n); + consumed += n; + }; + auto u32 = [&]() { + unsigned char b[4]; + exact(b, 4); + return static_cast(b[0]) | (static_cast(b[1]) << 8) | + (static_cast(b[2]) << 16) | (static_cast(b[3]) << 24); + }; + auto u64 = [&]() { + unsigned char b[8]; + exact(b, 8); + uint64_t v = 0; + for (int i = 7; i >= 0; --i) v = (v << 8) | b[i]; + return v; + }; + // A length-prefixed string; `optional` lets the length be 0xFFFFFFFF (absent). + auto str = [&](bool optional) -> std::optional { + const uint32_t len = u32(); + if (len == 0xFFFFFFFFu) { + if (optional) return std::nullopt; + throw detail::SnapshotRejected("a required string is marked absent"); + } + if (len > remaining()) throw detail::SnapshotRejected("a string is longer than the file"); + std::string s(len, '\0'); + exact(s.empty() ? nullptr : &s[0], len); + if (!detail::validUtf8(reinterpret_cast(s.data()), s.size())) { + throw detail::SnapshotRejected("a string is not UTF-8"); + } + return s; + }; + + unsigned char magic[8]; + exact(magic, 8); + if (std::memcmp(magic, "ISONVSNP", 8) != 0) throw detail::SnapshotRejected("not a snapshot (bad magic)"); + const uint32_t version = u32(); + if (version != SNAPSHOT_VERSION) { + throw detail::SnapshotRejected("unsupported snapshot version " + std::to_string(version) + + " (this build reads " + std::to_string(SNAPSHOT_VERSION) + ")"); + } + const uint32_t flags = u32(); + if ((flags & ~1u) != 0) throw detail::SnapshotRejected("unknown flags " + std::to_string(flags)); + const uint64_t dim = u32(); + const uint64_t blockRows = u32(); + const uint64_t count = u64(); + if (blockRows == 0 || blockRows > (uint64_t(1) << 20)) { + throw detail::SnapshotRejected("implausible block size " + std::to_string(blockRows)); + } + if (count > 0xFFFFFFFFull || (count > 0 && dim == 0)) { + throw detail::SnapshotRejected("implausible shape " + std::to_string(count) + " x " + std::to_string(dim)); + } + // count * dim * 4 > remaining, without overflowing 64 bits + if (count > 0 && dim > (remaining() / 4) / count) { + throw detail::SnapshotRejected("fewer bytes than the rows need"); + } + + SnapshotInfo info; + info.version = version; + info.dimension = static_cast(dim); + info.count = static_cast(count); + info.graphDigest = str(true); + info.graphDigestBasis = str(true); + info.encoderModel = str(true); + info.encoderVersion = str(true); + if (flags & 1u) { + SidecarId id; + id.len = u64(); + exact(id.fingerprint.data(), 32); + info.sidecar = id; + } + const uint32_t modelCount = u32(); + if (modelCount > remaining() / 4) throw detail::SnapshotRejected("more models than the file can hold"); + std::vector models; + models.reserve(modelCount); + for (uint32_t i = 0; i < modelCount; ++i) models.push_back(*str(false)); + info.bytes = fileLen; + if (accept) { + if (auto why = accept(info)) throw detail::SnapshotRejected(*why); + } + + struct RowMeta { + std::string type, id, text; + uint32_t model; + }; + std::vector bytes; + std::vector row(count > 0 ? static_cast(dim) : size_t(0)); + uint64_t left = count; + while (left > 0) { + const size_t rows = static_cast(std::min(left, blockRows)); + std::vector meta; + meta.reserve(rows); + for (size_t i = 0; i < rows; ++i) { + RowMeta m; + m.type = *str(false); + m.id = *str(false); + m.text = *str(false); + m.model = u32(); + if (m.model >= models.size()) throw detail::SnapshotRejected("a row names a model that is not listed"); + meta.push_back(std::move(m)); + } + const uint64_t need = static_cast(rows) * dim * 4; + if (need > remaining()) throw detail::SnapshotRejected("truncated"); + bytes.resize(static_cast(need)); + exact(bytes.data(), bytes.size()); + for (size_t i = 0; i < rows; ++i) { + detail::floatsFromLe(bytes.data() + i * static_cast(dim) * 4, static_cast(dim), row.data()); + if (!stage.putSnapshotRow(meta[i].type, meta[i].id, meta[i].text, models[meta[i].model], + row.data(), row.size())) { + throw detail::SnapshotRejected("the key " + meta[i].type + ":" + meta[i].id + " appears twice"); + } + } + left -= rows; + } + if (remaining() != 0) throw detail::SnapshotRejected("bytes left over after the last block"); + const auto expected = hasher.digest(); + unsigned char stored[32]; + if (std::fread(stored, 1, 32, file) != 32) throw detail::SnapshotRejected("truncated"); + if (std::memcmp(stored, expected.data(), 32) != 0) throw detail::SnapshotRejected("checksum mismatch"); + return info; + } + static std::string jsonString(const std::string& value) { static const char* const HEX = "0123456789abcdef"; std::string out = "\""; @@ -2668,7 +3768,8 @@ class SemanticGraph { return embedding_store_.addBatch(items); } - const EmbeddingRecord* getEmbedding(const NodeRef& nodeRef) const { + /** The node's stored embedding as a copy (see EmbeddingStore::get), or nullopt. */ + std::optional getEmbedding(const NodeRef& nodeRef) const { return embedding_store_.get(nodeRef); } @@ -2807,8 +3908,8 @@ class SemanticGraph { const std::string& nodeType = "", float threshold = 0.0f ) const { - const EmbeddingRecord* record = embedding_store_.get(nodeRef); - if (record == nullptr) return {}; + std::optional record = embedding_store_.get(nodeRef); + if (!record.has_value()) return {}; auto results = embedding_store_.similaritySearchVector( record->vector, topK + 1, nodeType, threshold); @@ -3217,8 +4318,8 @@ class SemanticGraph { const Node& node = graph_.getNode(ref.type, ref.id); sub.graph_.addNode(node.type, node.id, node.properties); - const EmbeddingRecord* record = embedding_store_.get(ref); - if (record != nullptr) { + std::optional record = embedding_store_.get(ref); + if (record.has_value()) { sub.embedding_store_.add(ref, record->text, record->vector); } } @@ -3243,7 +4344,7 @@ class SemanticGraph { graph_.forEachNode([&](const Node& node) { NodeRef ref(node.type, node.id); - if (skipExisting && embedding_store_.get(ref) != nullptr) return; + if (skipExisting && embedding_store_.contains(ref)) return; std::string text = textFn(node); if (!text.empty()) { batch.emplace_back(ref, text); @@ -3302,8 +4403,8 @@ class SemanticGraph { std::vector> pairs; for (const NodeRef& ref : stale) { - const EmbeddingRecord* record = embedding_store_.get(ref); - if (record == nullptr) continue; + std::optional record = embedding_store_.get(ref); + if (!record.has_value()) continue; std::string text = record->text; if (text.empty()) { if (graph_.hasNode(ref)) { @@ -3339,6 +4440,11 @@ class SemanticGraph { return path + EMBEDDING_SIDECAR_SUFFIX; } + /** Snapshot path next to a graph saved at `path` (`.embeddings.snap`). */ + static std::string snapshotPathFor(const std::string& path) { + return path + EMBEDDING_SNAPSHOT_SUFFIX; + } + /** * Save the graph to a file, and its embeddings alongside it. * @@ -3361,11 +4467,22 @@ class SemanticGraph { * With `embeddings` false an existing sidecar is left in place (and will * fail the digest check on load, as it no longer matches). * + * With `snapshot` true a binary snapshot of the embeddings is also written + * to `.embeddings.snap` (after the sidecar, which stays the portable + * copy; the graph file, the sidecar bytes and the digest are identical with + * or without it). load() uses it when it provably belongs to the graph and + * the sidecar on disk, and reads the JSON otherwise. If the snapshot cannot + * be written the graph and sidecar are saved anyway, a stale snapshot is + * removed, and the error is thrown. A save without `snapshot` deletes any + * existing snapshot first (the sidecar is about to change). Ignored when + * `embeddings` is false. + * * @param format "auto" (by extension), "ison" or "isonl" * @param embeddings write the sidecar (false for a graph-only save) + * @param snapshot also write the binary snapshot */ void save(const std::string& path, const std::string& format = "auto", - bool embeddings = true) const { + bool embeddings = true, bool snapshot = false) const { std::string actualFormat = format; if (format == "auto") { actualFormat = (path.size() >= 6 && path.compare(path.size() - 6, 6, ".isonl") == 0) @@ -3378,8 +4495,27 @@ class SemanticGraph { if (embeddings) { ison::Document reread = isonl ? ison::loads_isonl(text) : ison::loads(text); - embedding_store_.saveEmbeddings(embeddingsPathFor(path), - detail::digestOfDocument(reread)); + const std::string digest = detail::digestOfDocument(reread); + const std::string sidecar = embeddingsPathFor(path); + const std::string snap = snapshotPathFor(path); + if (!snapshot) { + // the sidecar is about to change: a snapshot of the old one is stale + std::error_code ignored; + std::filesystem::remove(snap, ignored); + } + embedding_store_.saveEmbeddings(sidecar, digest); + if (snapshot) { + try { + SnapshotIdentity identity; + identity.graphDigest = digest; + identity.sidecar = EmbeddingStore::sidecarIdOf(sidecar); + embedding_store_.saveSnapshot(snap, identity); + } catch (...) { + std::error_code ignored; + std::filesystem::remove(snap, ignored); + throw; + } + } } } @@ -3412,6 +4548,16 @@ class SemanticGraph { * @param ann opt in to approximate search (see EmbeddingStore::enableAnn). * The index is never saved; it is rebuilt lazily from the loaded * vectors on the first search. + * @param report if not null, filled with where the embeddings came from. + * When a binary snapshot (`.embeddings.snap`, written by + * save(..., snapshot=true)) sits beside the sidecar it is used if its + * header names this graph's digest and the sidecar file as it is now + * and its checksum verifies; the result equals loading the JSON. + * Otherwise the JSON sidecar is read exactly as before and + * `report->snapshotSkipped` says why the snapshot was not used. A + * snapshot is only ever looked at when the sidecar exists and + * `embeddings` is true; it does not change what `onMismatch` means + * for the JSON path. */ static SemanticGraph load( const std::string& path, @@ -3423,8 +4569,18 @@ class SemanticGraph { GraphMismatchMode onMismatch = GraphMismatchMode::Error, std::optional version = std::nullopt, ModelMismatchMode onModelMismatch = ModelMismatchMode::Warn, - std::optional ann = std::nullopt + std::optional ann = std::nullopt, + LoadReport* report = nullptr ) { + const auto started = std::chrono::steady_clock::now(); + LoadReport local; + auto finish = [&](SemanticGraph&& graph, LoadSource source, size_t restored) { + local.source = source; + local.embeddings = restored; + local.seconds = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + if (report != nullptr) *report = local; + return std::move(graph); + }; // Read and parse the text here rather than through ISONGraph::load: the // digest has to be taken from the document (it records which ids were // integers and which cells were null, and the graph does not). @@ -3459,57 +4615,83 @@ class SemanticGraph { onModelMismatch, std::move(version), std::move(ann)); result.graph_ = std::move(base); - if (embeddings) { - std::string sidecar = embeddingsPathFor(path); + if (!embeddings) return finish(std::move(result), LoadSource::NoEmbeddings, 0); + + std::string sidecar = embeddingsPathFor(path); + { std::ifstream probe(sidecar, std::ios::binary); - if (probe.is_open()) { - probe.close(); + if (!probe.is_open()) return finish(std::move(result), LoadSource::NoEmbeddings, 0); + } - ExportedEmbeddings payload; - try { - payload = EmbeddingStore::parseJson(EmbeddingStore::readFile(sidecar)); - } catch (const EmbeddingError& error) { - throw EmbeddingError("unreadable embeddings sidecar " + sidecar + ": " + - error.what()); - } + std::optional actualDigest; + auto graphDigestNow = [&]() -> const std::string& { + if (!actualDigest.has_value()) actualDigest = detail::digestOfDocument(doc); + return *actualDigest; + }; + auto warnOnVersionSkew = [&]() { + const auto& storedVersion = result.embedding_store_.lastImportMeta().encoderVersion; + auto currentVersion = result.embedding_store_.version(); + if (storedVersion.has_value() && currentVersion.has_value() && + *storedVersion != *currentVersion) { + emitWarning("embeddings were made by encoder version '" + *storedVersion + + "', but the encoder in use is version '" + *currentVersion + "'"); + } + }; - std::optional problem; - if (payload.meta.graphDigest.has_value() && - payload.meta.graphDigestBasis.value_or(GRAPH_DIGEST_BASIS) == - std::string(GRAPH_DIGEST_BASIS)) { - std::string actual = detail::digestOfDocument(doc); - if (*payload.meta.graphDigest != actual) { - problem = "embeddings sidecar " + sidecar + " does not belong to graph " + - path + ": it was written for graph digest " + - *payload.meta.graphDigest + ", but the graph reads as " + - actual + " (the sidecar is from a different save, or the " - "graph was edited since). Pass GraphMismatchMode::Reembed to " - "rebuild the vectors or GraphMismatchMode::Warn to use them " - "anyway."; - } + // The binary snapshot is a cache of the sidecar: take it only when it + // provably belongs to this graph and this sidecar, else fall through. + const std::string snap = snapshotPathFor(path); + { + std::error_code ec; + if (std::filesystem::is_regular_file(snap, ec)) { + SnapshotInfo info; + std::optional why = + result.embedding_store_.tryLoadSnapshotFor(snap, sidecar, graphDigestNow(), &info); + if (!why.has_value()) { + warnOnVersionSkew(); + return finish(std::move(result), LoadSource::Snapshot, info.count); } + local.snapshotSkipped = *why; + } + } - if (problem.has_value() && onMismatch == GraphMismatchMode::Error) { - throw GraphEmbeddingMismatchError(*problem); - } - if (problem.has_value() && onMismatch == GraphMismatchMode::Reembed) { - result.embedAllNodes(); - } else { - if (problem.has_value()) emitWarning(*problem); - result.embedding_store_.importEmbeddings(payload); - - const auto& storedVersion = result.embedding_store_.lastImportMeta().encoderVersion; - auto currentVersion = result.embedding_store_.version(); - if (storedVersion.has_value() && currentVersion.has_value() && - *storedVersion != *currentVersion) { - emitWarning("embeddings were made by encoder version '" + *storedVersion + - "', but the encoder in use is version '" + *currentVersion + "'"); - } - } + ExportedEmbeddings payload; + try { + payload = EmbeddingStore::parseJson(EmbeddingStore::readFile(sidecar)); + } catch (const EmbeddingError& error) { + throw EmbeddingError("unreadable embeddings sidecar " + sidecar + ": " + + error.what()); + } + + std::optional problem; + if (payload.meta.graphDigest.has_value() && + payload.meta.graphDigestBasis.value_or(GRAPH_DIGEST_BASIS) == + std::string(GRAPH_DIGEST_BASIS)) { + const std::string& actual = graphDigestNow(); + if (*payload.meta.graphDigest != actual) { + problem = "embeddings sidecar " + sidecar + " does not belong to graph " + + path + ": it was written for graph digest " + + *payload.meta.graphDigest + ", but the graph reads as " + + actual + " (the sidecar is from a different save, or the " + "graph was edited since). Pass GraphMismatchMode::Reembed to " + "rebuild the vectors or GraphMismatchMode::Warn to use them " + "anyway."; } } - return result; + if (problem.has_value() && onMismatch == GraphMismatchMode::Error) { + throw GraphEmbeddingMismatchError(*problem); + } + if (problem.has_value() && onMismatch == GraphMismatchMode::Reembed) { + result.embedAllNodes(); + } else { + if (problem.has_value()) emitWarning(*problem); + result.embedding_store_.importEmbeddings(payload); + warnOnVersionSkew(); + } + + const size_t restored = result.embedding_store_.count(); + return finish(std::move(result), LoadSource::Json, restored); } }; diff --git a/isongraph-vector-cpp/include/isongraph_vector_ivf.hpp b/isongraph-vector-cpp/include/isongraph_vector_ivf.hpp index bbaf9fb..de723fd 100644 --- a/isongraph-vector-cpp/include/isongraph_vector_ivf.hpp +++ b/isongraph-vector-cpp/include/isongraph_vector_ivf.hpp @@ -7,8 +7,9 @@ * isongraph_vector.hpp owns the slot <-> record mapping and the policy * (when to build, fold, retrain); this file is the arithmetic. * - * Layout (all contiguous, nothing allocated per query): - * data slots x dim floats, the vectors as given (not normalised) + * The index holds NO copy of the vectors. The rows live once, in the store's + * ChunkedMatrix (below); every method that scores or trains takes that matrix + * and the per-slot type codes by reference. Layout of what the index itself keeps: * invNorm 1 / |row| per slot (0 for a zero row, which then scores 0) * assign cell of each slot, -1 = in no cell yet (new/updated) or dead * order the slots of every cell, grouped by cell @@ -22,7 +23,9 @@ * threads used: every row's cell is independent of the others, and a * centroid sums its rows in ascending row order. * - * Not thread-safe by itself; EmbeddingStore serialises access. + * Thread safety: search() and candidates() are const and use only per-call + * scratch, so any number may run at once; every other method needs exclusive + * access, which EmbeddingStore provides with a reader/writer lock. */ #ifndef ISONGRAPH_VECTOR_IVF_HPP @@ -36,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -220,6 +224,39 @@ inline size_t autoThreads() { return n == 0 ? 1 : n; } +/** + * Run fn(worker) once on each of `workers` threads (worker 0 is the caller's), + * and return when all are done. The first exception any of them throws is + * rethrown on the caller's thread after every worker has been joined, so a + * failure neither leaks a thread nor terminates the process. If the platform + * cannot start a thread the remaining workers simply do not exist: callers + * must hand out work from a shared counter, so the workers that did start + * finish all of it. + */ +template +void runWorkers(size_t workers, F&& fn) { + if (workers == 0) workers = 1; + std::exception_ptr failure; + std::atomic failed{false}; + auto guarded = [&](size_t w) { + try { + fn(w); + } catch (...) { + if (!failed.exchange(true)) failure = std::current_exception(); + } + }; + std::vector pool; + pool.reserve(workers - 1); + try { + for (size_t w = 1; w < workers; ++w) pool.emplace_back(guarded, w); + } catch (const std::system_error&) { + // fewer threads than asked for; see above + } + guarded(0); + for (auto& t : pool) t.join(); + if (failure) std::rethrow_exception(failure); +} + /** One search hit: higher score first, lower slot first on a tie. */ struct IvfHit { float score; @@ -230,6 +267,149 @@ inline bool hitBefore(const IvfHit& a, const IvfHit& b) { return a.score > b.score || (a.score == b.score && a.slot < b.slot); } +// ============================================================================= +// The vector matrix: one copy of every vector +// ============================================================================= + +/** Rows per chunk of the matrix (and per scan block). */ +constexpr size_t kChunkRows = 1024; + +/** + * Row-major float matrix stored in chunks of kChunkRows rows. Growth never + * copies stored rows (a new chunk is allocated; only the first chunk, which + * starts small, is regrown while it is still below kChunkRows rows), a deletion + * is a row move done by the owner during compaction, and the rows of one chunk + * are contiguous, so a scan block never straddles chunks. Chunks are allocated + * uninitialised, so memory is only touched when a row is written. + * + * Row pointers are stable until the next write to the matrix. + */ +class ChunkedMatrix { +public: + ChunkedMatrix() = default; + ChunkedMatrix(ChunkedMatrix&& other) noexcept { takeFrom(other); } + ChunkedMatrix& operator=(ChunkedMatrix&& other) noexcept { + if (this != &other) takeFrom(other); + return *this; + } + ChunkedMatrix(const ChunkedMatrix& other) { copyFrom(other); } + ChunkedMatrix& operator=(const ChunkedMatrix& other) { + if (this != &other) { + chunks_.clear(); + copyFrom(other); + } + return *this; + } + + size_t dim() const { return dim_; } + size_t rows() const { return rows_; } + size_t chunks() const { return chunks_.size(); } + /** Rows held by chunk `c` (kChunkRows except for the last). */ + size_t rowsInChunk(size_t c) const { + return c + 1 < chunks_.size() ? kChunkRows : rows_ - c * kChunkRows; + } + /** Bytes allocated for rows (not counting the table of chunk pointers). */ + size_t bytes() const { + size_t total = 0; + for (size_t c = 0; c < chunks_.size(); ++c) { + total += (c == 0 ? firstCap_ : kChunkRows) * dim_ * sizeof(float); + } + return total; + } + + /** Forget everything and set the row width. */ + void reset(size_t dim) { + chunks_.clear(); + rows_ = 0; + firstCap_ = 0; + dim_ = dim; + } + + const float* row(size_t slot) const { + return chunks_[slot / kChunkRows].get() + (slot % kChunkRows) * dim_; + } + float* row(size_t slot) { + return chunks_[slot / kChunkRows].get() + (slot % kChunkRows) * dim_; + } + const float* chunkData(size_t c) const { return chunks_[c].get(); } + + /** Add a row at the end; returns its slot. */ + size_t append(const float* v) { + const size_t slot = rows_; + if (slot % kChunkRows == 0) { + if (slot == 0) { + growFirst(4); + } else { + chunks_.emplace_back(new float[kChunkRows * dim_]); + } + } else if (slot < kChunkRows && slot == firstCap_) { + growFirst(std::min(kChunkRows, firstCap_ * 2)); + } + ++rows_; + std::copy(v, v + dim_, row(slot)); + return slot; + } + + void set(size_t slot, const float* v) { std::copy(v, v + dim_, row(slot)); } + + /** Copy row `from` over row `to` (from > to). */ + void moveRow(size_t from, size_t to) { + const float* src = row(from); + std::copy(src, src + dim_, row(to)); + } + + /** Keep the first `keep` rows and free the chunks no longer needed. */ + void truncate(size_t keep) { + rows_ = keep; + const size_t needed = (keep + kChunkRows - 1) / kChunkRows; + while (chunks_.size() > needed) chunks_.pop_back(); + if (keep == 0) firstCap_ = 0; + } + +private: + std::vector> chunks_; + size_t dim_ = 0; + size_t rows_ = 0; + size_t firstCap_ = 0; // rows the first chunk has room for (<= kChunkRows) + + void takeFrom(ChunkedMatrix& other) noexcept { + chunks_ = std::move(other.chunks_); + dim_ = other.dim_; + rows_ = other.rows_; + firstCap_ = other.firstCap_; + other.chunks_.clear(); + other.rows_ = 0; + other.firstCap_ = 0; + } + + void growFirst(size_t wantRows) { + wantRows = std::min(wantRows, kChunkRows); + std::unique_ptr next(new float[wantRows * dim_]); + if (!chunks_.empty()) { + const size_t used = std::min(rows_, kChunkRows); + std::copy(chunks_[0].get(), chunks_[0].get() + used * dim_, next.get()); + chunks_[0] = std::move(next); + } else { + chunks_.push_back(std::move(next)); + } + firstCap_ = wantRows; + } + + void copyFrom(const ChunkedMatrix& other) { + dim_ = other.dim_; + rows_ = other.rows_; + firstCap_ = other.firstCap_; + chunks_.clear(); + chunks_.reserve(other.chunks_.size()); + for (size_t c = 0; c < other.chunks_.size(); ++c) { + const size_t cap = c == 0 ? other.firstCap_ : kChunkRows; + std::unique_ptr mine(new float[cap * dim_]); + std::copy(other.chunks_[c].get(), other.chunks_[c].get() + other.rowsInChunk(c) * dim_, mine.get()); + chunks_.push_back(std::move(mine)); + } + } +}; + // ============================================================================= // The index // ============================================================================= @@ -250,60 +430,63 @@ class IvfIndex { size_t dead() const { return dead_; } /** - * Train on `n` rows laid out contiguously in `data` (slot i = row i, all - * alive), then put every row in its nearest cell. `codes` gives each row - * a small integer type code for filtered searches. + * Train on the live rows of `m` and put every one in its nearest cell. + * Slot i of the index is row i of the matrix; `alive` says which slots hold a + * row. `trainOrder` lists the live slots in the order training should see + * them (the store sorts them by key, so the same data gives the same index + * however it was inserted). */ - static IvfIndex build(size_t dim, std::vector&& data, std::vector&& codes, - size_t nlistWanted, uint64_t seed, size_t threads, int iterations = 8) { + static IvfIndex build(const ChunkedMatrix& m, const std::vector& alive, + const std::vector& trainOrder, size_t nlistWanted, + uint64_t seed, size_t threads, int iterations = 8) { auto started = std::chrono::steady_clock::now(); IvfIndex index; - index.dim = dim; - index.data_ = std::move(data); - index.codes_ = std::move(codes); - const size_t n = index.codes_.size(); - index.alive_.assign(n, 1); + index.dim = m.dim(); + const size_t n = m.rows(); + index.alive_ = alive; + index.dead_ = 0; + for (uint8_t a : index.alive_) index.dead_ += a ? 0 : 1; index.pend_.assign(n, 0); index.assign_.assign(n, -1); - index.invNorm_.resize(n); + index.invNorm_.assign(n, 0.0f); if (threads == 0) threads = autoThreads(); + const size_t dim = index.dim; parallelFor((n + 8191) / 8192, threads, [&](size_t block) { size_t lo = block * 8192, hi = std::min(n, lo + 8192); - for (size_t i = lo; i < hi; ++i) index.invNorm_[i] = inverseNorm(index.row(i), dim); + for (size_t i = lo; i < hi; ++i) { + if (index.alive_[i]) index.invNorm_[i] = inverseNorm(m.row(i), dim); + } }); - index.nlist = std::max(1, std::min(nlistWanted, n)); - index.trainedN = n; - index.train(seed, threads, iterations); - index.assignAll(threads); + const size_t liveRows = trainOrder.size(); + index.nlist = std::max(1, std::min(nlistWanted, liveRows)); + index.trainedN = liveRows; + index.train(m, trainOrder, seed, threads, iterations); + index.assignAll(m, threads); index.relist(); index.buildSeconds = std::chrono::duration( std::chrono::steady_clock::now() - started).count(); return index; } - // -- writes ----------------------------------------------------------------- + // -- writes (the owner has already written the row to the matrix) ------------- - /** A new row; it is scored exactly by every query until folded into a cell. */ - uint32_t append(const float* v, uint32_t code) { + /** A new row at the next slot; it is scored exactly by every query until folded into a cell. */ + uint32_t append(const float* v) { uint32_t slot = static_cast(alive_.size()); - data_.insert(data_.end(), v, v + dim); invNorm_.push_back(inverseNorm(v, dim)); assign_.push_back(-1); alive_.push_back(1); pend_.push_back(1); - codes_.push_back(code); pending_.push_back(slot); ++pendingLive; ++churn; return slot; } - /** Replace a row in place: it leaves its cell and is pending again. */ - void overwrite(uint32_t slot, const float* v, uint32_t code) { - std::copy(v, v + dim, data_.begin() + static_cast(slot) * dim); + /** A row was replaced in place: it leaves its cell and is pending again. */ + void overwrite(uint32_t slot, const float* v) { invNorm_[slot] = inverseNorm(v, dim); - codes_[slot] = code; assign_[slot] = -1; if (!pend_[slot]) { pend_[slot] = 1; @@ -326,41 +509,30 @@ class IvfIndex { ++churn; } - bool needsCompaction() const { return dead_ >= 64 && dead_ * 4 > alive_.size(); } - /** - * Squeeze out the tombstones, keeping live rows in slot order and their - * cells. Returns, for every old slot, its new slot (kNoSlot when dead); - * the owner uses it to renumber its own slot tables. + * The owner squeezed the tombstones out of its matrix, keeping live rows in + * slot order: `remapTable[old]` is the new slot (kNoSlot for a dead one) and + * `kept` the new slot count. Renumber everything slot-indexed to match. */ - std::vector compact() { + void remap(const std::vector& remapTable, size_t kept) { const size_t n = alive_.size(); - std::vector remap(n, kNoSlot); - size_t kept = 0; for (size_t i = 0; i < n; ++i) { - if (!alive_[i]) continue; - remap[i] = static_cast(kept); - if (kept != i) { - std::copy(row(i), row(i) + dim, data_.begin() + static_cast(kept) * dim); - invNorm_[kept] = invNorm_[i]; - assign_[kept] = assign_[i]; - pend_[kept] = pend_[i]; - codes_[kept] = codes_[i]; - } - ++kept; + const uint32_t to = remapTable[i]; + if (to == kNoSlot || to == i) continue; + invNorm_[to] = invNorm_[i]; + assign_[to] = assign_[i]; + pend_[to] = pend_[i]; } - data_.resize(kept * dim); invNorm_.resize(kept); assign_.resize(kept); pend_.resize(kept); - codes_.resize(kept); alive_.assign(kept, 1); dead_ = 0; std::vector pending; pending.reserve(pendingLive); for (uint32_t old : pending_) { - if (old < n && remap[old] != kNoSlot && pend_[remap[old]]) { - pending.push_back(remap[old]); + if (old < n && remapTable[old] != kNoSlot && pend_[remapTable[old]]) { + pending.push_back(remapTable[old]); } } // a slot can be listed twice only if it was dropped and re-listed, which @@ -369,11 +541,10 @@ class IvfIndex { pending.erase(std::unique(pending.begin(), pending.end()), pending.end()); pending_ = std::move(pending); relist(); - return remap; } /** Put every pending row in its nearest cell and regroup. */ - void fold(size_t threads) { + void fold(const ChunkedMatrix& m, size_t threads) { std::vector ids; ids.reserve(pendingLive); for (uint32_t slot : pending_) { @@ -384,7 +555,8 @@ class IvfIndex { if (!ids.empty()) { std::vector gathered(ids.size() * dim); for (size_t i = 0; i < ids.size(); ++i) { - std::copy(row(ids[i]), row(ids[i]) + dim, gathered.begin() + static_cast(i) * dim); + const float* src = m.row(ids[i]); + std::copy(src, src + dim, gathered.begin() + static_cast(i) * static_cast(dim)); } std::vector cells(ids.size()); if (threads == 0) threads = autoThreads(); @@ -404,45 +576,43 @@ class IvfIndex { /** * Score the rows of the `lists` cells whose centroids are nearest `q`, plus * every pending row, and keep the best `k` that score at least `threshold` - * (and have type `code`, unless it is kAnyCode). `out` is filled best - * first. Scores are cosine similarities computed from the cached inverse - * norms. + * (and have type `code` in `codes`, unless it is kAnyCode). `out` is filled + * best first. Scores are cosine similarities computed from the cached + * inverse norms. const, and all scratch is local to the call: any number of + * searches may run at once. */ static constexpr uint32_t kAnyCode = std::numeric_limits::max(); - void search(const float* q, float invQ, uint32_t code, float threshold, size_t lists, - size_t k, std::vector& out) { + void search(const ChunkedMatrix& m, const uint32_t* codes, const float* q, float invQ, + uint32_t code, float threshold, size_t lists, size_t k, + std::vector& out) const { out.clear(); if (k == 0 || live() == 0) return; - chooseCells(q, lists); - heap_.clear(); - heap_.reserve(k); - scan(q, invQ, code, threshold, k, /*collectAll=*/false); - out.assign(heap_.begin(), heap_.end()); + std::vector chosen; + chooseCells(q, lists, chosen); + scan(m, codes, q, invQ, code, threshold, k, /*collectAll=*/false, chosen, out); std::sort(out.begin(), out.end(), hitBefore); } /** Like search() but returns every candidate that passes the filters, best first. */ - void candidates(const float* q, float invQ, uint32_t code, float threshold, size_t lists, - std::vector& out) { + void candidates(const ChunkedMatrix& m, const uint32_t* codes, const float* q, float invQ, + uint32_t code, float threshold, size_t lists, + std::vector& out) const { out.clear(); if (live() == 0) return; - chooseCells(q, lists); - heap_.clear(); - scan(q, invQ, code, threshold, 0, /*collectAll=*/true); - out.assign(heap_.begin(), heap_.end()); + std::vector chosen; + chooseCells(q, lists, chosen); + scan(m, codes, q, invQ, code, threshold, 0, /*collectAll=*/true, chosen, out); std::sort(out.begin(), out.end(), hitBefore); } - /** Bytes held by the index apart from the vectors themselves. */ + /** Bytes held by the index. It holds no copy of the vectors. */ size_t overheadBytes() const { return centroids_.capacity() * sizeof(float) + assign_.capacity() * sizeof(int32_t) + order_.capacity() * sizeof(uint32_t) + offsets_.capacity() * sizeof(uint32_t) + - invNorm_.capacity() * sizeof(float) + codes_.capacity() * sizeof(uint32_t) + + invNorm_.capacity() * sizeof(float) + alive_.capacity() + pend_.capacity() + pending_.capacity() * sizeof(uint32_t); } - /** Bytes of the contiguous vector copy the index scans. */ - size_t vectorBytes() const { return data_.capacity() * sizeof(float); } static float inverseNorm(const float* v, size_t n) { float s = simd::dot(v, v, n); @@ -450,26 +620,16 @@ class IvfIndex { } private: - std::vector data_; std::vector invNorm_; std::vector assign_; std::vector alive_; std::vector pend_; - std::vector codes_; std::vector pending_; std::vector order_; std::vector offsets_; std::vector centroids_; size_t dead_ = 0; - // per-query scratch, sized once - std::vector cellScore_; - std::vector cellIdx_; - std::vector chosen_; - std::vector heap_; - - const float* row(size_t slot) const { return data_.data() + slot * dim; } - // -- training ----------------------------------------------------------------- static uint64_t below(std::mt19937_64& rng, uint64_t bound) { return rng() % bound; } @@ -495,6 +655,16 @@ class IvfIndex { }); } + /** nearest() over the first `count` rows of the matrix (256-row blocks never cross a chunk). */ + void nearestMatrix(const ChunkedMatrix& m, size_t count, int32_t* out, size_t threads) const { + constexpr size_t kChunk = 256; + static_assert(kChunkRows % kChunk == 0, "a 256-row block must lie inside one chunk"); + parallelFor((count + kChunk - 1) / kChunk, threads, [&](size_t chunk) { + size_t lo = chunk * kChunk; + nearestRange(m.row(lo), std::min(kChunk, count - lo), out + lo); + }); + } + /** Blocked nearest-centroid kernel: 4 rows x 2 centroids per step, centroids in L2-sized panels. */ void nearestRange(const float* rows, size_t count, int32_t* out) const { constexpr size_t kRowBlock = 32; @@ -541,8 +711,9 @@ class IvfIndex { const float* centroid(size_t c) const { return centroids_.data() + c * dim; } float* centroid(size_t c) { return centroids_.data() + c * dim; } - void train(uint64_t seed, size_t threads, int iterations) { - const size_t total = alive_.size(); + void train(const ChunkedMatrix& m, const std::vector& trainOrder, uint64_t seed, + size_t threads, int iterations) { + const size_t total = trainOrder.size(); std::mt19937_64 rng(seed); const size_t sampleN = std::min(total, 40 * nlist); @@ -560,8 +731,9 @@ class IvfIndex { parallelFor((sampleN + 4095) / 4096, threads, [&](size_t block) { size_t lo = block * 4096, hi = std::min(sampleN, lo + 4096); for (size_t i = lo; i < hi; ++i) { - const float* src = row(sample[i]); - float inv = invNorm_[sample[i]]; + const uint32_t slot = trainOrder[sample[i]]; + const float* src = m.row(slot); + float inv = invNorm_[slot]; float* dst = x.data() + i * dim; for (size_t d = 0; d < dim; ++d) dst[d] = src[d] * inv; } @@ -622,9 +794,9 @@ class IvfIndex { } } - void assignAll(size_t threads) { + void assignAll(const ChunkedMatrix& m, size_t threads) { const size_t n = alive_.size(); - nearest(data_.data(), n, assign_.data(), threads); + nearestMatrix(m, n, assign_.data(), threads); for (size_t i = 0; i < n; ++i) { if (!alive_[i]) assign_[i] = -1; } @@ -650,60 +822,59 @@ class IvfIndex { // -- query internals --------------------------------------------------------- - void chooseCells(const float* q, size_t lists) { - if (cellScore_.size() != nlist) { - cellScore_.assign(nlist, 0.0f); - cellIdx_.resize(nlist); - } - chosen_.clear(); + void chooseCells(const float* q, size_t lists, std::vector& chosen) const { + chosen.clear(); if (lists >= nlist) { - for (size_t c = 0; c < nlist; ++c) chosen_.push_back(static_cast(c)); + for (size_t c = 0; c < nlist; ++c) chosen.push_back(static_cast(c)); return; } + std::vector cellScore(nlist); + std::vector cellIdx(nlist); for (size_t c = 0; c < nlist; ++c) { - cellScore_[c] = simd::dot(q, centroid(c), dim); - cellIdx_[c] = static_cast(c); + cellScore[c] = simd::dot(q, centroid(c), dim); + cellIdx[c] = static_cast(c); } - auto better = [this](uint32_t a, uint32_t b) { - return cellScore_[a] > cellScore_[b] || (cellScore_[a] == cellScore_[b] && a < b); + auto better = [&cellScore](uint32_t a, uint32_t b) { + return cellScore[a] > cellScore[b] || (cellScore[a] == cellScore[b] && a < b); }; - std::nth_element(cellIdx_.begin(), cellIdx_.begin() + static_cast(lists - 1), - cellIdx_.end(), better); - chosen_.assign(cellIdx_.begin(), cellIdx_.begin() + static_cast(lists)); - std::sort(chosen_.begin(), chosen_.end()); + std::nth_element(cellIdx.begin(), cellIdx.begin() + static_cast(lists - 1), + cellIdx.end(), better); + chosen.assign(cellIdx.begin(), cellIdx.begin() + static_cast(lists)); + std::sort(chosen.begin(), chosen.end()); } - void scan(const float* q, float invQ, uint32_t code, float threshold, size_t k, bool collectAll) { - const float* base = data_.data(); + void scan(const ChunkedMatrix& m, const uint32_t* codes, const float* q, float invQ, + uint32_t code, float threshold, size_t k, bool collectAll, + const std::vector& chosen, std::vector& heap) const { auto take = [&](uint32_t slot) { - float s = simd::dot(base + static_cast(slot) * dim, q, dim) * invNorm_[slot] * invQ; + float s = simd::dot(m.row(slot), q, dim) * invNorm_[slot] * invQ; if (!(s >= threshold)) return; IvfHit hit{s, slot}; if (collectAll) { - heap_.push_back(hit); - } else if (heap_.size() < k) { - heap_.push_back(hit); - std::push_heap(heap_.begin(), heap_.end(), hitBefore); - } else if (hitBefore(hit, heap_.front())) { - std::pop_heap(heap_.begin(), heap_.end(), hitBefore); - heap_.back() = hit; - std::push_heap(heap_.begin(), heap_.end(), hitBefore); + heap.push_back(hit); + } else if (heap.size() < k) { + heap.push_back(hit); + std::push_heap(heap.begin(), heap.end(), hitBefore); + } else if (hitBefore(hit, heap.front())) { + std::pop_heap(heap.begin(), heap.end(), hitBefore); + heap.back() = hit; + std::push_heap(heap.begin(), heap.end(), hitBefore); } }; - for (uint32_t c : chosen_) { + for (uint32_t c : chosen) { const uint32_t lo = offsets_[c], hi = offsets_[c + 1]; for (uint32_t p = lo; p < hi; ++p) { - if (p + 2 < hi) simd::prefetch(base + static_cast(order_[p + 2]) * dim); + if (p + 2 < hi) simd::prefetch(m.row(order_[p + 2])); uint32_t slot = order_[p]; if (assign_[slot] != static_cast(c)) continue; // moved or deleted since - if (code != kAnyCode && codes_[slot] != code) continue; + if (code != kAnyCode && codes[slot] != code) continue; take(slot); } } for (uint32_t slot : pending_) { if (!alive_[slot] || !pend_[slot]) continue; - if (code != kAnyCode && codes_[slot] != code) continue; + if (code != kAnyCode && codes[slot] != code) continue; take(slot); } } diff --git a/isongraph-vector-cpp/tests/test_isongraph_vector.cpp b/isongraph-vector-cpp/tests/test_isongraph_vector.cpp index e20a12b..17cd690 100644 --- a/isongraph-vector-cpp/tests/test_isongraph_vector.cpp +++ b/isongraph-vector-cpp/tests/test_isongraph_vector.cpp @@ -20,6 +20,9 @@ #include #include #include +#include +#include +#include #include "../include/isongraph_vector.hpp" #include "../include/isongraph_vector_http.hpp" @@ -77,15 +80,16 @@ int tests_passed = 0; throw std::runtime_error("Assertion failed: " #a " <= " #b); \ } -#define ASSERT_NULL(ptr) \ - if ((ptr) != nullptr) { \ - throw std::runtime_error("Assertion failed: " #ptr " is not null"); \ - } +// EmbeddingStore::get() hands out a std::optional copy (it used to hand out a +// pointer into the store), so "null" means "no value" for either. +template +bool isNullValue(const T& p) { return p == nullptr; } +template +bool isNullValue(const std::optional& o) { return !o.has_value(); } -#define ASSERT_NOT_NULL(ptr) \ - if ((ptr) == nullptr) { \ - throw std::runtime_error("Assertion failed: " #ptr " is null"); \ - } +#define ASSERT_NULL(ptr) if (!isNullValue(ptr)) { throw std::runtime_error("Assertion failed: " #ptr " is not null"); } + +#define ASSERT_NOT_NULL(ptr) if (isNullValue(ptr)) { throw std::runtime_error("Assertion failed: " #ptr " is null"); } // ============================================================================= // MockEncoder Tests @@ -672,8 +676,8 @@ TEST(add_node_auto_embeds_from_properties) { graph.addNode("person", "1", {{"name", "Alice"}, {"description", "engineer"}}); ASSERT_EQ(graph.embeddingStore().count(), 1u); - const auto* record = graph.getEmbedding(NodeRef("person", "1")); - ASSERT_TRUE(record != nullptr); + const auto record = graph.getEmbedding(NodeRef("person", "1")); + ASSERT_TRUE(record.has_value()); ASSERT_EQ(record->text, "Alice engineer"); } @@ -814,7 +818,7 @@ TEST(imports_a_payload_written_by_the_python_port) { // and the vectors match what this port encodes for the same text auto expected = encoder->encode("Alice"); - const auto& stored = store.get(NodeRef("person", "1"))->vector; + const auto stored = store.get(NodeRef("person", "1"))->vector; // a copy: get() returns a copy of the row ASSERT_EQ(stored.size(), expected.size()); for (size_t i = 0; i < expected.size(); ++i) { ASSERT_NEAR(stored[i], expected[i], 1e-6f); @@ -835,8 +839,8 @@ TEST(embed_text_is_identical_across_ports) { {"text", ison::Value(true)}, }); - const auto* record = graph.getEmbedding(NodeRef("doc", "1")); - ASSERT_TRUE(record != nullptr); + const auto record = graph.getEmbedding(NodeRef("doc", "1")); + ASSERT_TRUE(record.has_value()); ASSERT_EQ(record->text, "Report 1.5 true"); } @@ -940,7 +944,7 @@ TEST(update_node_re_embeds_when_an_embed_field_changes) { const Node& node = graph.updateNode("person", "1", {{"name", "Bob"}}); ASSERT_EQ(node.properties.at("name").as_string(), "Bob"); - const auto* record = graph.getEmbedding(NodeRef("person", "1")); + const auto record = graph.getEmbedding(NodeRef("person", "1")); ASSERT_NOT_NULL(record); ASSERT_EQ(record->text, "Bob"); ASSERT_TRUE(record->vector != before); @@ -970,7 +974,7 @@ TEST(update_node_leaves_the_vector_when_no_embed_field_changed) { graph.addNode("person", "1", {{"name", "Alice"}}); graph.updateNode("person", "1", {{"age", ison::Value(int64_t(30))}}); - const auto* record = graph.getEmbedding(NodeRef("person", "1")); + const auto record = graph.getEmbedding(NodeRef("person", "1")); ASSERT_NOT_NULL(record); ASSERT_EQ(record->text, "Alice"); ASSERT_EQ(graph.graph().getNode("person", "1").properties.at("age").as_int(), int64_t(30)); @@ -1116,7 +1120,7 @@ TEST(sidecar_json_survives_quotes_braces_and_non_ascii_text) { EmbeddingStore other(encoder); ASSERT_EQ(other.fromJson(store.toJson()), 1u); - const auto* record = other.get(NodeRef("doc", "r\xc3\xa9sum\xc3\xa9")); + const auto record = other.get(NodeRef("doc", "r\xc3\xa9sum\xc3\xa9")); ASSERT_NOT_NULL(record); ASSERT_EQ(record->text, text); } @@ -1129,7 +1133,7 @@ TEST(reads_a_python_sidecar_with_ascii_escaped_unicode) { R"PAYLOAD({"format": "ison-embeddings", "version": 1, "dimension": 2, "count": 1, "embeddings": [{"type": "doc", "id": "r\u00e9sum\u00e9", "id_type": "str", "text": "caf\u00e9 } \ud83d\ude00 end", "model": "mock-2", "vector": [0.25, -0.5]}]})PAYLOAD"; EmbeddingStore store(std::make_shared(2)); ASSERT_EQ(store.fromJson(payload), 1u); - const auto* record = store.get(NodeRef("doc", "r\xc3\xa9sum\xc3\xa9")); + const auto record = store.get(NodeRef("doc", "r\xc3\xa9sum\xc3\xa9")); ASSERT_NOT_NULL(record); ASSERT_EQ(record->text, "caf\xc3\xa9 } \xf0\x9f\x98\x80 end"); ASSERT_EQ(record->vector.size(), 2u); @@ -1765,7 +1769,7 @@ TEST(lifecycle_reembed_keeps_custom_embed_text) { graph.addNodeWithEmbed("person", "1", {{"name", "A"}}, "a very specific blurb"); graph.setEncoder(retagged(8, "v2")); graph.reembedStale(); - const auto* record = graph.getEmbedding(NodeRef("person", "1")); + const auto record = graph.getEmbedding(NodeRef("person", "1")); ASSERT_EQ(record->text, "a very specific blurb"); ASSERT_EQ(record->model, "v2"); } @@ -2055,7 +2059,7 @@ TEST(consistency_reembed_mode_rebuilds_from_the_edited_graph) { auto loaded = SemanticGraph::load(path, encoder, "auto", true, true, {"name", "description", "content", "text"}, GraphMismatchMode::Reembed); - const auto* record = loaded.getEmbedding(NodeRef("person", "2")); + const auto record = loaded.getEmbedding(NodeRef("person", "2")); ASSERT_EQ(record->text, "Robert"); auto expected = encoder->encode("Robert"); for (size_t i = 0; i < expected.size(); ++i) ASSERT_NEAR(record->vector[i], expected[i], 1e-6f); @@ -3243,8 +3247,13 @@ TEST(ann_pending_rows_fold_and_heavy_churn_retrains) { auto extra = gen.drawMany(700); for (size_t i = 0; i < extra.size(); ++i) store.add(NodeRef("late", std::to_string(i)), "", extra[i]); - ASSERT_EQ(store.annStats().pending, 700u); - store.similaritySearchVector(extra[0], 5); // >= 512 pending: folded + // pending rows are folded into their cells by the write that makes 256 of + // them (at write time, not at the next search): 700 = 2 x 256 + 188 + ASSERT_EQ(store.annStats().pending, 188u); + store.similaritySearchVector(extra[0], 5); // a search does not fold or rebuild + ASSERT_EQ(store.annStats().pending, 188u); + ASSERT_EQ(store.annStats().builds, 1u); + store.buildAnnIndex(); // an explicit build folds the rest ASSERT_EQ(store.annStats().pending, 0u); ASSERT_EQ(store.annStats().builds, 1u); @@ -3500,6 +3509,1087 @@ TEST(ann_concurrent_searches_are_safe) { ASSERT_EQ(store.annStats().builds, 1u); // four threads, one lazy build } +// ============================================================================= +// Stage D: one copy of every vector, batch search, binary snapshot +// ============================================================================= + +namespace { + +uint32_t floatBits(float f) { + uint32_t u; + std::memcpy(&u, &f, sizeof u); + return u; +} + +/// Same ids, same texts and the same scores bit for bit, in the same order. +bool sameHits(const std::vector& a, const std::vector& b) { + if (a.size() != b.size()) return false; + for (size_t i = 0; i < a.size(); ++i) { + if (!(a[i].node_ref == b[i].node_ref) || a[i].text != b[i].text || + floatBits(a[i].score) != floatBits(b[i].score)) { + return false; + } + } + return true; +} + +struct SdRng { + uint64_t state; + explicit SdRng(uint64_t seed) : state(seed) {} + uint64_t next() { + state += 0x9E3779B97F4A7C15ull; + uint64_t z = state; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); + } + float unit() { return static_cast(next() >> 40) / 8388608.0f - 1.0f; } + size_t below(size_t n) { return static_cast(next() % n); } + std::vector vec(size_t dim) { + std::vector v(dim); + for (auto& x : v) x = unit(); + return v; + } +}; + +/// A fresh directory under the system temp directory, removed on destruction. +class SdTempDir { + std::filesystem::path dir_; +public: + explicit SdTempDir(const std::string& tag) { + static std::atomic counter{0}; + std::random_device rd; + dir_ = std::filesystem::temp_directory_path() / + ("isongraph_vector_cpp_" + tag + "_" + std::to_string(rd()) + "_" + std::to_string(counter++)); + std::filesystem::create_directories(dir_); + } + ~SdTempDir() { + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + SdTempDir(const SdTempDir&) = delete; + SdTempDir& operator=(const SdTempDir&) = delete; + std::string file(const std::string& name) const { return (dir_ / name).string(); } + const std::filesystem::path& path() const { return dir_; } +}; + +std::string sdSlurp(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("cannot read " + path); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +void sdSpit(const std::string& path, const std::string& bytes) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(bytes.data(), static_cast(bytes.size())); +} + +std::string sdHex(const std::array& d) { return detail::hexOf(d.data(), d.size()); } + +/// Where the Rust crate's golden snapshot fixture lives: the CMake-provided +/// path, else next to this source file, else found by walking up from the +/// working directory (so a hand-run binary finds it too). +std::filesystem::path sdFixtureDir() { + std::vector candidates; +#if defined(ISONGRAPH_VECTOR_FIXTURE_DIR) + candidates.emplace_back(ISONGRAPH_VECTOR_FIXTURE_DIR); +#endif + candidates.push_back(std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / "tests" / "fixtures" / "snapshot"); + std::error_code ec; + std::filesystem::path up = std::filesystem::current_path(ec); + for (int i = 0; i < 6 && !up.empty(); ++i) { + candidates.push_back(up / "tests" / "fixtures" / "snapshot"); + up = up.parent_path(); + } + for (const auto& c : candidates) { + if (std::filesystem::exists(c / "golden.ison", ec)) return c; + } + return candidates.front(); +} + +/// Copy of the golden fixture (graph, JSON sidecar, snapshot) in a temp dir. +void sdCopyGolden(const SdTempDir& dir, bool withSnapshot = true) { + const auto src = sdFixtureDir(); + if (!std::filesystem::exists(src / "golden.ison")) { + throw std::runtime_error("golden snapshot fixture not found at " + src.string()); + } + sdSpit(dir.file("golden.ison"), sdSlurp((src / "golden.ison").string())); + sdSpit(dir.file("golden.ison.embeddings.json"), sdSlurp((src / "golden.ison.embeddings.json").string())); + if (withSnapshot) sdSpit(dir.file("golden.ison.embeddings.snap"), sdSlurp((src / "golden.ison.embeddings.snap").string())); +} + +/// Every key, text, model and vector (bit for bit) of two stores, and some searches, agree. +void sdExpectSameStore(const EmbeddingStore& a, const EmbeddingStore& b) { + ASSERT_EQ(a.count(), b.count()); + ASSERT_TRUE(a.dimension() == b.dimension()); + ASSERT_TRUE(a.models() == b.models()); + auto pa = a.exportEmbeddings(); + auto pb = b.exportEmbeddings(); + ASSERT_EQ(pa.embeddings.size(), pb.embeddings.size()); + for (size_t i = 0; i < pa.embeddings.size(); ++i) { + const auto& x = pa.embeddings[i]; + const auto& y = pb.embeddings[i]; + ASSERT_EQ(x.type, y.type); + ASSERT_EQ(x.id, y.id); + ASSERT_EQ(x.text, y.text); + ASSERT_EQ(x.model, y.model); + ASSERT_EQ(x.vector.size(), y.vector.size()); + for (size_t j = 0; j < x.vector.size(); ++j) ASSERT_EQ(floatBits(x.vector[j]), floatBits(y.vector[j])); + } + if (a.dimension().has_value() && a.count() > 0) { + SdRng rng(99); + for (int q = 0; q < 12; ++q) { + auto query = rng.vec(*a.dimension()); + ASSERT_TRUE(sameHits(a.similaritySearchVector(query, 10, "", -1.0f), b.similaritySearchVector(query, 10, "", -1.0f))); + ASSERT_TRUE(sameHits(a.similaritySearchVector(query, 5, "doc", 0.0f), b.similaritySearchVector(query, 5, "doc", 0.0f))); + } + } +} + +} // namespace + +// ---- SHA-256 ----------------------------------------------------------------- + +TEST(sha256_matches_the_nist_vectors_and_streams) { + ASSERT_EQ(detail::sha256Hex("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + ASSERT_EQ(detail::sha256Hex(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + ASSERT_EQ(detail::sha256Hex("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + + // a million 'a', fed in awkward chunk sizes, equals one call + const std::string million(1000000, 'a'); + detail::Sha256 streamed; + size_t at = 0, step = 1; + while (at < million.size()) { + size_t n = std::min(step, million.size() - at); + streamed.update(million.data() + at, n); + at += n; + step = step * 3 + 1; + if (step > 70000) step = 1; + } + const auto digest = streamed.digest(); + ASSERT_EQ(sdHex(digest), "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0"); + ASSERT_EQ(detail::sha256Hex(million), sdHex(digest)); + + // digest() leaves the hasher usable + detail::Sha256 h; + h.update("ab", 2); + (void)h.digest(); + h.update("c", 1); + ASSERT_EQ(sdHex(h.digest()), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); +} + +// ---- one copy of every vector -------------------------------------------------- + +TEST(store_keeps_one_copy_of_every_vector) { + constexpr size_t N = 5000, DIM = 64; + SdRng rng(1); + EmbeddingStore store; + for (size_t i = 0; i < N; ++i) store.add(NodeRef("doc", std::to_string(i)), "t", rng.vec(DIM)); + const size_t raw = N * DIM * sizeof(float); + ASSERT_GE(store.vectorBytes(), raw); + // the last chunk is allocated whole (1024 rows); nothing else is stored twice + ASSERT_TRUE(store.vectorBytes() <= raw + 1024 * DIM * sizeof(float)); + store.verifyInvariants(); + + // with the approximate index on, the vectors are still held once + AnnOptions o; + o.nlist = 32; o.nprobe = 4; o.minSize = 0; + store.enableAnn(o); + AnnStats built = store.buildAnnIndex(); + ASSERT_TRUE(built.active); + ASSERT_EQ(built.vectorBytes, 0u); + ASSERT_TRUE(built.indexBytes < raw / 4); + ASSERT_TRUE(store.vectorBytes() <= raw + 1024 * DIM * sizeof(float)); + store.verifyInvariants(); + + // a small store does not reserve a whole chunk + EmbeddingStore tiny; + for (size_t i = 0; i < 10; ++i) tiny.add(NodeRef("d", std::to_string(i)), "", rng.vec(DIM)); + ASSERT_TRUE(tiny.vectorBytes() <= 16 * DIM * sizeof(float)); +} + +TEST(get_returns_a_copy_and_vector_data_a_view) { + EmbeddingStore store; + store.add(NodeRef("a", "1"), "one", {1.0f, 2.0f, 3.0f}); + store.add(NodeRef("a", "2"), "two", {4.0f, 5.0f, 6.0f}); + auto record = store.get(NodeRef("a", "1")); + ASSERT_TRUE(record.has_value()); + ASSERT_EQ(record->id, "a:1"); + ASSERT_EQ(record->model, "provided"); + store.add(NodeRef("a", "1"), "uno", {7.0f, 8.0f, 9.0f}); // overwrite in place + ASSERT_EQ(record->vector[0], 1.0f); // the copy did not change + ASSERT_EQ(record->text, "one"); + ASSERT_EQ(store.get(NodeRef("a", "1"))->vector[0], 7.0f); + const float* view = store.vectorData(NodeRef("a", "2")); + ASSERT_TRUE(view != nullptr); + ASSERT_EQ(view[2], 6.0f); + ASSERT_TRUE(store.vectorData(NodeRef("a", "nope")) == nullptr); + ASSERT_TRUE(store.contains(NodeRef("a", "2"))); + ASSERT_FALSE(store.contains(NodeRef("a", "3"))); +} + +TEST(store_matches_a_model_through_random_writes_deletes_and_compaction) { + constexpr size_t DIM = 12; + SdRng rng(7); + EmbeddingStore store; + std::vector order; // live keys in slot (arrival) order + std::map> truth; + auto drop = [&](const std::string& key) { + order.erase(std::find(order.begin(), order.end(), key)); + truth.erase(key); + }; + for (int op = 0; op < 9000; ++op) { + const size_t what = rng.below(10); + const NodeRef ref(rng.below(2) ? "x" : "y", std::to_string(rng.below(1500))); + const std::string full = ref.type + ":" + ref.id; + if (what < 6) { + std::vector v = rng.below(40) == 0 ? std::vector(DIM, 0.0f) : rng.vec(DIM); + store.add(ref, "text " + full, v); + if (!truth.count(full)) order.push_back(full); + truth[full] = v; + } else if (what < 8) { + const bool had = truth.count(full) > 0; + ASSERT_EQ(store.remove(ref), had); + if (had) drop(full); + } else if (!order.empty()) { + // a burst of deletes, enough to cross the compaction threshold now and then + for (int k = 0; k < 40 && !order.empty(); ++k) { + const std::string victim = order[rng.below(order.size())]; + const size_t colon = victim.find(':'); + ASSERT_TRUE(store.remove(NodeRef(victim.substr(0, colon), victim.substr(colon + 1)))); + drop(victim); + } + } + if (op % 450 == 0) { + store.verifyInvariants(); + ASSERT_EQ(store.count(), truth.size()); + auto query = rng.vec(DIM); + // reference: score every live row in arrival order, best first, ties by arrival + std::vector> scored; + for (size_t i = 0; i < order.size(); ++i) { + scored.emplace_back(detail::cosineRows(query.data(), truth[order[i]].data(), DIM), i); + } + std::stable_sort(scored.begin(), scored.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + auto got = store.similaritySearchVector(query, 15, "", -2.0f); + ASSERT_EQ(got.size(), std::min(15, order.size())); + for (size_t i = 0; i < got.size(); ++i) { + ASSERT_EQ(floatBits(got[i].score), floatBits(scored[i].first)); + ASSERT_EQ(got[i].node_ref.type + ":" + got[i].node_ref.id, order[scored[i].second]); + } + } + } + store.verifyInvariants(); + for (const auto& [key, v] : truth) { + const size_t colon = key.find(':'); + auto record = store.get(NodeRef(key.substr(0, colon), key.substr(colon + 1))); + ASSERT_TRUE(record.has_value()); + ASSERT_TRUE(record->vector == v); + ASSERT_EQ(record->text, "text " + key); + } +} + +TEST(equal_scores_are_ordered_by_arrival) { + EmbeddingStore store; + for (int i = 0; i < 30; ++i) store.add(NodeRef("d", std::to_string(i)), "", {1.0f, 0.0f}); + store.add(NodeRef("d", "other"), "", {0.0f, 1.0f}); + auto hits = store.similaritySearchVector({1.0f, 0.0f}, 8, "", 0.5f); + ASSERT_EQ(hits.size(), 8u); + for (int i = 0; i < 8; ++i) ASSERT_EQ(hits[static_cast(i)].node_ref.id, std::to_string(i)); + // delete enough to compact; the survivors keep their order + for (int i = 0; i < 30; i += 2) store.remove(NodeRef("d", std::to_string(i))); + hits = store.similaritySearchVector({1.0f, 0.0f}, 4, "", 0.5f); + ASSERT_EQ(hits[0].node_ref.id, "1"); + ASSERT_EQ(hits[1].node_ref.id, "3"); + ASSERT_EQ(hits[3].node_ref.id, "7"); +} + +// ---- batch search --------------------------------------------------------------- + +namespace { + +/// A store with three chunks of rows, two types, zero rows, exact duplicates, +/// updates and deletes (so tombstones and compaction are in play). +EmbeddingStore sdBatchStore(size_t dim, bool withAnn) { + SdRng rng(21); + EmbeddingStore store; + if (withAnn) { + AnnOptions o; + o.nlist = 24; o.nprobe = 6; o.minSize = 0; o.seed = 3; + store.enableAnn(o); + } + const size_t n = 2600; + std::vector> data; + for (size_t i = 0; i < n; ++i) { + std::vector v = rng.vec(dim); + if (i % 97 == 5) v.assign(dim, 0.0f); + if (i % 61 == 3 && i > 10) v = data[i - 3]; // exact duplicate: an equal score + data.push_back(v); + store.add(NodeRef(i % 4 == 0 ? "a" : "b", std::to_string(i)), "t" + std::to_string(i), v); + } + for (size_t i = 1; i < n; i += 13) store.add(NodeRef(i % 4 == 0 ? "a" : "b", std::to_string(i)), "updated", rng.vec(dim)); + for (size_t i = 2; i < n; i += 9) store.remove(NodeRef(i % 4 == 0 ? "a" : "b", std::to_string(i))); + return store; +} + +} // namespace + +TEST(batch_search_is_bit_identical_to_single_search) { + constexpr size_t DIM = 24; + EmbeddingStore store = sdBatchStore(DIM, false); + SdRng rng(5); + std::vector> queries; + for (int q = 0; q < 37; ++q) queries.push_back(rng.vec(DIM)); // 37 = two full tiles and a partial one + queries.push_back(std::vector(DIM, 0.0f)); // a zero query scores 0 everywhere + queries.push_back(store.get(NodeRef("b", "1"))->vector); // an exact stored vector + for (size_t k : {size_t(1), size_t(10), size_t(500), size_t(100000)}) { + for (const char* type : {"", "a", "nope"}) { + for (float threshold : {-1.0f, 0.2f}) { + std::vector> single; + for (const auto& q : queries) single.push_back(store.similaritySearchVector(q, k, type, threshold)); + for (size_t threads : {size_t(1), size_t(2), size_t(3), size_t(8), size_t(64), size_t(0)}) { + auto batch = store.similaritySearchBatch(queries, k, type, threshold, false, threads); + ASSERT_EQ(batch.size(), single.size()); + for (size_t i = 0; i < queries.size(); ++i) ASSERT_TRUE(sameHits(batch[i], single[i])); + } + } + } + } + store.verifyInvariants(); +} + +TEST(batch_search_with_the_approximate_index_equals_single_search) { + constexpr size_t DIM = 24; + EmbeddingStore store = sdBatchStore(DIM, true); + SdRng rng(6); + std::vector> queries; + for (int q = 0; q < 40; ++q) queries.push_back(rng.vec(DIM)); + for (bool exact : {false, true}) { + std::vector> single; + for (const auto& q : queries) single.push_back(store.similaritySearchVector(q, 10, "", -1.0f, exact)); + for (size_t threads : {size_t(1), size_t(4), size_t(16)}) { + auto batch = store.similaritySearchBatch(queries, 10, "", -1.0f, exact, threads); + for (size_t i = 0; i < queries.size(); ++i) ASSERT_TRUE(sameHits(batch[i], single[i])); + } + } + ASSERT_TRUE(store.annStats().active); + store.verifyInvariants(); +} + +TEST(one_scoring_routine_gives_one_score_everywhere) { + constexpr size_t DIM = 24; + EmbeddingStore store = sdBatchStore(DIM, true); + SdRng rng(8); + std::vector> queries; + for (int q = 0; q < 20; ++q) queries.push_back(rng.vec(DIM)); + auto batch = store.similaritySearchBatch(queries, 25, "", -2.0f, true, 4); + for (size_t qi = 0; qi < queries.size(); ++qi) { + const auto& q = queries[qi]; + auto single = store.similaritySearchVector(q, 25, "", -2.0f, true); + auto viaIndex = store.similaritySearchVector(q, 25, "", -2.0f); // approximate path: re-scored exactly + auto map = store.scoreMap(q); // the generic one-pass path + ASSERT_TRUE(sameHits(batch[qi], single)); + for (const auto& hit : viaIndex) { + ASSERT_EQ(floatBits(map.at(hit.node_ref.type + ":" + hit.node_ref.id)), floatBits(hit.score)); + } + for (const auto& hit : single) { + const std::string key = hit.node_ref.type + ":" + hit.node_ref.id; + ASSERT_EQ(floatBits(map.at(key)), floatBits(hit.score)); + const float* row = store.vectorData(hit.node_ref); + ASSERT_EQ(floatBits(detail::cosineRows(q.data(), row, DIM)), floatBits(hit.score)); + } + } +} + +TEST(batch_search_validates_and_handles_edge_cases) { + EmbeddingStore store; + for (int i = 0; i < 50; ++i) store.add(NodeRef("d", std::to_string(i)), "", {static_cast(i + 1), 1.0f, 0.5f}); + ASSERT_TRUE(store.similaritySearchBatch({}, 5).empty()); + auto none = store.similaritySearchBatch({{1.0f, 1.0f, 1.0f}, {2.0f, 1.0f, 0.0f}}, 0); + ASSERT_EQ(none.size(), 2u); + ASSERT_TRUE(none[0].empty() && none[1].empty()); + // one wrong-width query fails the whole batch before any search starts + ASSERT_THROWS(store.similaritySearchBatch({{1.0f, 1.0f, 1.0f}, {1.0f, 1.0f}}, 5), DimensionMismatchError); + ASSERT_THROWS(store.similaritySearchBatch({{}}, 5), DimensionMismatchError); + // an empty store answers every query with nothing + EmbeddingStore empty; + auto r = empty.similaritySearchBatch({{1.0f, 2.0f}}, 3); + ASSERT_EQ(r.size(), 1u); + ASSERT_TRUE(r[0].empty()); + // text queries + auto encoder = std::make_shared(16); + EmbeddingStore texts(encoder); + for (int i = 0; i < 40; ++i) texts.add(NodeRef("d", std::to_string(i)), "doc " + std::to_string(i)); + auto byText = texts.similaritySearchTextBatch({"doc 3", "doc 17", "other"}, 5, "", -1.0f, false, 2); + ASSERT_EQ(byText.size(), 3u); + ASSERT_TRUE(sameHits(byText[1], texts.similaritySearch("doc 17", 5, "", -1.0f))); + ASSERT_EQ(byText[0][0].node_ref.id, "3"); +} + +TEST(run_workers_rethrows_the_first_failure_after_joining_every_thread) { + std::atomic finished{0}; + bool threw = false; + try { + detail::runWorkers(6, [&](size_t w) { + if (w == 2) throw std::runtime_error("worker failed"); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + ++finished; + }); + } catch (const std::runtime_error& e) { + threw = std::string(e.what()) == "worker failed"; + } + ASSERT_TRUE(threw); + ASSERT_EQ(finished.load(), 5); // every other worker ran to completion before the throw + int ran = 0; + detail::runWorkers(1, [&](size_t) { ++ran; }); + ASSERT_EQ(ran, 1); +} + +TEST(concurrent_const_searches_and_batches_agree_with_a_serial_run) { + constexpr size_t DIM = 24; + EmbeddingStore store = sdBatchStore(DIM, true); + SdRng rng(11); + std::vector> queries; + for (int q = 0; q < 24; ++q) queries.push_back(rng.vec(DIM)); + std::vector> expect, expectExact; + for (const auto& q : queries) { + expect.push_back(store.similaritySearchVector(q, 7, "", -1.0f)); + expectExact.push_back(store.similaritySearchVector(q, 7, "", -1.0f, true)); + } + std::atomic bad{0}; + auto worker = [&](int id) { + for (int round = 0; round < 6; ++round) { + if ((id + round) % 3 == 0) { + auto batch = store.similaritySearchBatch(queries, 7, "", -1.0f, round % 2 == 0, 3); + const auto& want = (round % 2 == 0) ? expectExact : expect; + for (size_t i = 0; i < queries.size(); ++i) bad += sameHits(batch[i], want[i]) ? 0 : 1; + } else { + for (size_t i = 0; i < queries.size(); ++i) { + bad += sameHits(store.similaritySearchVector(queries[i], 7, "", -1.0f), expect[i]) ? 0 : 1; + bad += sameHits(store.similaritySearchVector(queries[i], 7, "", -1.0f, true), expectExact[i]) ? 0 : 1; + } + } + } + }; + std::vector pool; + for (int t = 0; t < 6; ++t) pool.emplace_back(worker, t); + for (auto& t : pool) t.join(); + ASSERT_EQ(bad.load(), 0); +} + +TEST(pending_rows_fold_at_write_time_and_writes_stay_visible) { + ClusteredData gen(30, 24, 31); + auto data = gen.drawMany(3000); + EmbeddingStore store; + store.enableAnn(annOptions(32, 4, 0)); + fillStore(store, data); + store.buildAnnIndex(); + auto fresh = gen.drawMany(600); + size_t folds = 0, lastPending = 0; + for (size_t i = 0; i < fresh.size(); ++i) { + store.add(NodeRef("new", std::to_string(i)), "", fresh[i]); + const size_t pending = store.annStats().pending; + ASSERT_TRUE(pending < 256); // never reaches the fold batch between writes + if (pending < lastPending) ++folds; + lastPending = pending; + if (i % 97 == 0) { // visible at once, folded or not + auto top = store.similaritySearchVector(fresh[i], 1); + ASSERT_EQ(top[0].node_ref.type, "new"); + } + } + ASSERT_EQ(folds, 2u); + ASSERT_EQ(store.annStats().builds, 1u); + store.verifyInvariants(); +} + +// ---- snapshot: layout, identity, fallback ----------------------------------------- + +TEST(snapshot_smallest_example_is_byte_exact_per_the_spec) { + EmbeddingStore store; + store.add(NodeRef("t", "a"), "x", {1.0f, -2.0f}); + SdTempDir dir("snapmin"); + const std::string path = dir.file("min.snap"); + SnapshotInfo info = store.saveSnapshot(path); + ASSERT_EQ(info.bytes, 123u); + ASSERT_EQ(info.count, 1u); + ASSERT_EQ(info.dimension, 2u); + const std::string bytes = sdSlurp(path); + ASSERT_EQ(bytes.size(), 123u); + std::string expected = "ISONVSNP"; + detail::putU32(expected, 1); // version + detail::putU32(expected, 0); // flags + detail::putU32(expected, 2); // dim + detail::putU32(expected, 1024); // block rows + detail::putU64(expected, 1); // count + for (int i = 0; i < 4; ++i) detail::putU32(expected, 0xFFFFFFFFu); + detail::putU32(expected, 1); + detail::putStr(expected, "provided"); + detail::putStr(expected, "t"); + detail::putStr(expected, "a"); + detail::putStr(expected, "x"); + detail::putU32(expected, 0); + expected += std::string("\x00\x00\x80\x3f\x00\x00\x00\xc0", 8); + ASSERT_EQ(expected.size(), 91u); + ASSERT_EQ(bytes.substr(0, 91), expected); + // the trailer is the SHA-256 of those 91 bytes (value computed independently with Python's hashlib) + ASSERT_EQ(detail::hexOf(reinterpret_cast(bytes.data()) + 91, 32), + "a9c5f4584064d7e0f5bc89bc6173fe062db3439d65a500f01179d4e56ee69df2"); + + EmbeddingStore back; + SnapshotInfo loaded = back.loadSnapshot(path); + ASSERT_EQ(loaded.count, 1u); + ASSERT_EQ(back.get(NodeRef("t", "a"))->vector[1], -2.0f); + ASSERT_EQ(back.get(NodeRef("t", "a"))->model, "provided"); + ASSERT_EQ(back.get(NodeRef("t", "a"))->text, "x"); +} + +TEST(float_bytes_are_ieee_754_little_endian) { + const float v[3] = {1.0f, -2.0f, 0.15625f}; + unsigned char out[12]; + detail::floatsToLe(v, 3, out); + const unsigned char want[12] = {0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x20, 0x3e}; + ASSERT_TRUE(std::memcmp(out, want, 12) == 0); + float back[3]; + detail::floatsFromLe(want, 3, back); + ASSERT_TRUE(back[0] == 1.0f && back[1] == -2.0f && back[2] == 0.15625f); +} + +TEST(sidecar_fingerprint_matches_an_independent_implementation) { + SdTempDir dir("fp"); + // value of the fingerprint for byte i = (i * 31 + 7) & 255, computed with Python's hashlib + const std::pair cases[] = { + {0, "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc"}, + {10, "c69cf557f7d1ea5716f1050a37be05c27bce7eb531cdd411b64343b4e8229cb5"}, + {100000, "41bc372b8f7d3ece400b9ab6ebedf055df23935990d8b9ae5527de1c467fafdf"}, + {300000, "689a231bf532d7bc20d8b6a2d04ce8325433314759f00bbebb4754c8958ae920"}, + }; + for (const auto& [len, hex] : cases) { + std::string bytes(len, '\0'); + for (size_t i = 0; i < len; ++i) bytes[i] = static_cast((i * 31 + 7) & 255); + sdSpit(dir.file("f.bin"), bytes); + SidecarId id = EmbeddingStore::sidecarIdOf(dir.file("f.bin")); + ASSERT_EQ(id.len, len); + ASSERT_EQ(sdHex(id.fingerprint), hex); + } + ASSERT_THROWS(EmbeddingStore::sidecarIdOf(dir.file("missing.bin")), EmbeddingError); +} + +namespace { + +struct SnapRow { + std::string type, id, text; + uint32_t model = 0; + std::vector v; +}; + +/// Builds snapshot bytes by hand (with a correct trailer unless asked otherwise), +/// so the rejection rules can be tested one at a time. +struct SnapSpec { + std::string magic = "ISONVSNP"; + uint32_t version = 1, flags = 0, dim = 2, blockRows = 1024; + std::optional count; // default: rows.size() + std::optional digest, basis, encModel, encVersion; + std::optional sidecar; + std::vector models{"m"}; + std::vector rows; + std::string beforeTrailer; // junk between the last block and the trailer + bool badTrailer = false; + + std::string build() const { + std::string out = magic; + detail::putU32(out, version); + detail::putU32(out, flags); + detail::putU32(out, dim); + detail::putU32(out, blockRows); + detail::putU64(out, count.value_or(rows.size())); + detail::putOpt(out, digest); + detail::putOpt(out, basis); + detail::putOpt(out, encModel); + detail::putOpt(out, encVersion); + if ((flags & 1u) && sidecar) { + detail::putU64(out, sidecar->len); + out.append(reinterpret_cast(sidecar->fingerprint.data()), 32); + } + detail::putU32(out, static_cast(models.size())); + for (const auto& m : models) detail::putStr(out, m); + const size_t step = blockRows == 0 ? std::max(rows.size(), 1) : blockRows; + for (size_t start = 0; start < rows.size(); start += step) { + const size_t end = std::min(rows.size(), start + step); + for (size_t i = start; i < end; ++i) { + detail::putStr(out, rows[i].type); + detail::putStr(out, rows[i].id); + detail::putStr(out, rows[i].text); + detail::putU32(out, rows[i].model); + } + for (size_t i = start; i < end; ++i) { + std::string f(rows[i].v.size() * 4, '\0'); + detail::floatsToLe(rows[i].v.data(), rows[i].v.size(), reinterpret_cast(&f[0])); + out += f; + } + } + out += beforeTrailer; + return withTrailer(out, badTrailer); + } + + static std::string withTrailer(const std::string& body, bool bad = false) { + auto digest = detail::Sha256(); + digest.update(body.data(), body.size()); + auto d = digest.digest(); + if (bad) d[5] ^= 0x40; + return body + std::string(reinterpret_cast(d.data()), 32); + } +}; + +SnapSpec sdGoodSpec() { + SnapSpec s; + s.digest = "sha256:abc"; + s.basis = GRAPH_DIGEST_BASIS; + s.models = {"m0", "m1"}; + s.rows = {SnapRow{"doc", "1", "first", 0, {1.0f, 2.0f}}, + SnapRow{"doc", "2", "second \xc3\xa9 \xf0\x9f\x98\x80", 1, {-0.5f, 4.0f}}, + SnapRow{"note", "x:y", "", 0, {0.0f, 0.0f}}}; + return s; +} + +/// A store that must come through a refused load unchanged. +EmbeddingStore sdBaseStore() { + EmbeddingStore base; + base.add(NodeRef("keep", "1"), "kept text", {3.0f, 4.0f}); + base.add(NodeRef("keep", "2"), "kept too", {-1.0f, 0.5f}); + return base; +} + +void sdExpectRefused(const SdTempDir& dir, const std::string& bytes, const std::string& why) { + sdSpit(dir.file("bad.snap"), bytes); + EmbeddingStore store = sdBaseStore(); + bool threw = false; + try { + store.loadSnapshot(dir.file("bad.snap")); + } catch (const EmbeddingError&) { + threw = true; + } + if (!threw) throw std::runtime_error("a snapshot with " + why + " was accepted"); + sdExpectSameStore(store, sdBaseStore()); // untouched + store.verifyInvariants(); +} + +} // namespace + +TEST(snapshot_reader_accepts_a_hand_built_valid_file) { + SdTempDir dir("snapok"); + SnapSpec spec = sdGoodSpec(); + sdSpit(dir.file("ok.snap"), spec.build()); + EmbeddingStore store = sdBaseStore(); + SnapshotInfo info = store.loadSnapshot(dir.file("ok.snap")); + ASSERT_EQ(info.count, 3u); + ASSERT_EQ(store.count(), 3u); // replaced, not merged + ASSERT_FALSE(store.contains(NodeRef("keep", "1"))); + ASSERT_EQ(store.get(NodeRef("doc", "2"))->model, "m1"); + ASSERT_EQ(store.get(NodeRef("doc", "2"))->text, "second \xc3\xa9 \xf0\x9f\x98\x80"); + ASSERT_EQ(store.get(NodeRef("note", "x:y"))->model, "m0"); + ASSERT_EQ(store.lastImportMeta().graphDigest.value(), "sha256:abc"); + store.verifyInvariants(); + // the same rows over several blocks + spec.blockRows = 2; + sdSpit(dir.file("ok2.snap"), spec.build()); + EmbeddingStore blocks; + blocks.loadSnapshot(dir.file("ok2.snap")); + sdExpectSameStore(store, blocks); + // no rows at all is a valid snapshot + SnapSpec none; + none.dim = 0; + sdSpit(dir.file("none.snap"), none.build()); + EmbeddingStore emptied = sdBaseStore(); + emptied.loadSnapshot(dir.file("none.snap")); + ASSERT_EQ(emptied.count(), 0u); +} + +TEST(snapshot_reader_applies_every_rejection_rule) { + SdTempDir dir("snaprej"); + { SnapSpec s = sdGoodSpec(); s.magic = "ISONVSNQ"; sdExpectRefused(dir, s.build(), "a bad magic"); } + { SnapSpec s = sdGoodSpec(); s.version = 2; sdExpectRefused(dir, s.build(), "version 2"); } + { SnapSpec s = sdGoodSpec(); s.flags = 2; sdExpectRefused(dir, s.build(), "an unknown flag"); } + { SnapSpec s = sdGoodSpec(); s.flags = 3; s.sidecar = SidecarId{}; sdExpectRefused(dir, s.build(), "unknown flag bit 1 alongside bit 0"); } + { SnapSpec s = sdGoodSpec(); s.blockRows = 0; sdExpectRefused(dir, s.build(), "block_rows 0"); } + { SnapSpec s = sdGoodSpec(); s.blockRows = (1u << 20) + 1; sdExpectRefused(dir, s.build(), "block_rows above 2^20"); } + { SnapSpec s = sdGoodSpec(); s.dim = 0; sdExpectRefused(dir, s.build(), "dim 0 and rows"); } + { SnapSpec s = sdGoodSpec(); s.count = 1000000; sdExpectRefused(dir, s.build(), "a count the file cannot hold"); } + { SnapSpec s = sdGoodSpec(); s.count = 0x100000000ull; sdExpectRefused(dir, s.build(), "a count above 2^32-1"); } + { SnapSpec s = sdGoodSpec(); s.count = 4; sdExpectRefused(dir, s.build(), "a count larger than the rows present"); } + { SnapSpec s = sdGoodSpec(); s.count = 2; sdExpectRefused(dir, s.build(), "bytes left after the last counted block"); } + { SnapSpec s = sdGoodSpec(); s.dim = 4000000; sdExpectRefused(dir, s.build(), "a dimension the file cannot hold"); } + { SnapSpec s = sdGoodSpec(); s.rows[1].model = 2; sdExpectRefused(dir, s.build(), "a model index past the model list"); } + { SnapSpec s = sdGoodSpec(); s.rows[2] = s.rows[0]; sdExpectRefused(dir, s.build(), "a repeated key"); } + { + // ("a", "b:c") and ("a:b", "c") are the same key "a:b:c" + SnapSpec s = sdGoodSpec(); + s.rows[0] = SnapRow{"a", "b:c", "", 0, {1.0f, 0.0f}}; + s.rows[2] = SnapRow{"a:b", "c", "", 0, {0.0f, 1.0f}}; + sdExpectRefused(dir, s.build(), "a repeated key made of different type/id splits"); + } + { SnapSpec s = sdGoodSpec(); s.beforeTrailer = "xx"; sdExpectRefused(dir, s.build(), "bytes between the last block and the trailer"); } + { SnapSpec s = sdGoodSpec(); s.badTrailer = true; sdExpectRefused(dir, s.build(), "a wrong checksum"); } + { SnapSpec s = sdGoodSpec(); s.rows[0].text = "bad \xff utf8"; sdExpectRefused(dir, s.build(), "invalid UTF-8 (a lone 0xff)"); } + { SnapSpec s = sdGoodSpec(); s.rows[0].text = "overlong \xc0\x80"; sdExpectRefused(dir, s.build(), "an overlong UTF-8 form"); } + { SnapSpec s = sdGoodSpec(); s.rows[0].text = "surrogate \xed\xa0\x80"; sdExpectRefused(dir, s.build(), "a UTF-8 encoded surrogate"); } + { SnapSpec s = sdGoodSpec(); s.models = {"\xff"}; s.rows[1].model = 0; sdExpectRefused(dir, s.build(), "an invalid UTF-8 model name"); } + + // string lengths: past the end of the file, and "absent" where a string is required + { + std::string good = sdGoodSpec().build(); + std::string body = good.substr(0, good.size() - 32); + const size_t digestLen = std::string("sha256:abc").size(); + // header: 8 + 4*4 + 8 = 32, then the digest string + std::string longLen = body; + for (int i = 0; i < 4; ++i) longLen[32 + static_cast(i)] = static_cast(i == 3 ? 0x7f : 0xff); + sdExpectRefused(dir, SnapSpec::withTrailer(longLen), "a string longer than the file"); + // first row's type length: after digest(4+10) basis(4+19) enc model(4) enc version(4) n_models(4) m0(4+2) m1(4+2) + const size_t firstRow = 32 + 4 + digestLen + 4 + std::string(GRAPH_DIGEST_BASIS).size() + 4 + 4 + 4 + 6 + 6; + std::string absent = body; + for (int i = 0; i < 4; ++i) absent[firstRow + static_cast(i)] = static_cast(0xff); + sdExpectRefused(dir, SnapSpec::withTrailer(absent), "an absent string where one is required"); + } +} + +TEST(snapshot_reader_rejects_every_truncation_and_every_flipped_byte) { + SdTempDir dir("snapfuzz"); + const std::string good = sdGoodSpec().build(); + { + EmbeddingStore ok; + sdSpit(dir.file("good.snap"), good); + ok.loadSnapshot(dir.file("good.snap")); + ASSERT_EQ(ok.count(), 3u); + } + for (size_t len = 0; len < good.size(); ++len) { + sdExpectRefused(dir, good.substr(0, len), "a file cut to " + std::to_string(len) + " bytes"); + } + for (size_t at = 0; at < good.size(); ++at) { + std::string flipped = good; + flipped[at] = static_cast(flipped[at] ^ 0x01); + sdExpectRefused(dir, flipped, "byte " + std::to_string(at) + " flipped"); + } + sdExpectRefused(dir, good + "x", "a byte after the trailer"); + { + EmbeddingStore store = sdBaseStore(); + ASSERT_THROWS(store.loadSnapshot(dir.file("does-not-exist.snap")), EmbeddingError); + sdExpectSameStore(store, sdBaseStore()); + } +} + +// ---- snapshot: the Rust golden fixture -------------------------------------------- + +TEST(golden_snapshot_is_used_and_equals_the_json_only_load) { + SdTempDir withSnap("goldsnap"), jsonOnly("goldjson"); + sdCopyGolden(withSnap, true); + sdCopyGolden(jsonOnly, false); + auto encoder = std::make_shared(12); + + LoadReport snapReport, jsonReport; + SemanticGraph fromSnap = SemanticGraph::load(withSnap.file("golden.ison"), encoder, "auto", true, true, + {"name"}, GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Ignore, std::nullopt, &snapReport); + SemanticGraph fromJson = SemanticGraph::load(jsonOnly.file("golden.ison"), encoder, "auto", true, true, + {"name"}, GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Ignore, std::nullopt, &jsonReport); + ASSERT_TRUE(snapReport.source == LoadSource::Snapshot); + ASSERT_FALSE(snapReport.snapshotSkipped.has_value()); + ASSERT_TRUE(jsonReport.source == LoadSource::Json); + ASSERT_FALSE(jsonReport.snapshotSkipped.has_value()); + ASSERT_EQ(snapReport.embeddings, 36u); + ASSERT_EQ(jsonReport.embeddings, 36u); + ASSERT_TRUE(snapReport.seconds >= 0.0); + + sdExpectSameStore(fromSnap.embeddingStore(), fromJson.embeddingStore()); + ASSERT_EQ(fromSnap.embeddingStore().count(), 36u); + ASSERT_EQ(*fromSnap.embeddingStore().dimension(), 12u); + ASSERT_EQ(fromSnap.embeddingStore().models().size(), 2u); + ASSERT_EQ(fromSnap.graph().nodeCount(), fromJson.graph().nodeCount()); + const auto& a = fromSnap.embeddingStore().lastImportMeta(); + const auto& b = fromJson.embeddingStore().lastImportMeta(); + ASSERT_TRUE(a.encoderModel == b.encoderModel); + ASSERT_TRUE(a.encoderVersion == b.encoderVersion); + ASSERT_TRUE(a.graphDigest == b.graphDigest); + ASSERT_TRUE(a.graphDigestBasis == b.graphDigestBasis); + ASSERT_EQ(a.graphDigest.value_or(""), "sha256:a33458853f63a977962395b84c42fb347ed0f6ed66bc00eabb92ef69bb83026c"); + fromSnap.embeddingStore().verifyInvariants(); + + // the fixture has an exact duplicate vector (a tie) and a zero vector: same order from both + auto dup = fromJson.embeddingStore().get(NodeRef("a:b", "00002")); + ASSERT_TRUE(dup.has_value()); + auto tied = fromSnap.embeddingStore().similaritySearchVector(dup->vector, 3, "", 0.5f); + ASSERT_TRUE(sameHits(tied, fromJson.embeddingStore().similaritySearchVector(dup->vector, 3, "", 0.5f))); + ASSERT_EQ(floatBits(tied[0].score), floatBits(tied[1].score)); +} + +TEST(golden_snapshot_is_reproduced_byte_for_byte) { + SdTempDir dir("goldre"); + sdCopyGolden(dir, true); + auto encoder = std::make_shared(12); + encoder->setModelName("second-model"); // the Rust fixture's encoder at save time + SemanticGraph graph = SemanticGraph::load(dir.file("golden.ison"), encoder); + SnapshotIdentity identity; + identity.graphDigest = "sha256:a33458853f63a977962395b84c42fb347ed0f6ed66bc00eabb92ef69bb83026c"; + identity.sidecar = EmbeddingStore::sidecarIdOf(dir.file("golden.ison.embeddings.json")); + graph.embeddingStore().saveSnapshot(dir.file("again.snap"), identity); + const std::string golden = sdSlurp((sdFixtureDir() / "golden.ison.embeddings.snap").string()); + const std::string ours = sdSlurp(dir.file("again.snap")); + ASSERT_EQ(ours.size(), golden.size()); + ASSERT_TRUE(ours == golden); +} + +TEST(snapshot_that_does_not_fit_falls_back_to_the_json_sidecar) { + auto encoder = std::make_shared(12); + struct Case { + const char* name; + std::function damage; + const char* reason; // substring of snapshotSkipped + }; + const std::string snapName = "golden.ison.embeddings.snap"; + auto patchSnap = [&](const SdTempDir& d, const std::function& f) { + std::string bytes = sdSlurp(d.file(snapName)); + f(bytes); + sdSpit(d.file(snapName), bytes); + }; + std::vector cases; + cases.push_back({"corrupt vector byte", [&](const SdTempDir& d) { + patchSnap(d, [](std::string& b) { b[b.size() - 200] = static_cast(b[b.size() - 200] ^ 0x10); }); }, "checksum"}); + cases.push_back({"truncated", [&](const SdTempDir& d) { + patchSnap(d, [](std::string& b) { b.resize(b.size() - 100); }); }, ""}); + cases.push_back({"trailing garbage", [&](const SdTempDir& d) { + patchSnap(d, [](std::string& b) { b += "garbage"; }); }, ""}); + cases.push_back({"not a snapshot", [&](const SdTempDir& d) { + patchSnap(d, [](std::string& b) { b = std::string(5000, 'z'); }); }, "magic"}); + cases.push_back({"empty file", [&](const SdTempDir& d) { + patchSnap(d, [](std::string& b) { b.clear(); }); }, ""}); + cases.push_back({"stale: sidecar rewritten (same length)", [&](const SdTempDir& d) { + std::string json = sdSlurp(d.file("golden.ison.embeddings.json")); + const size_t at = json.find("0.8050792"); + json[at + 8] = '3'; // 0.8050793: one digit, same length + sdSpit(d.file("golden.ison.embeddings.json"), json); }, "stale"}); + cases.push_back({"stale: sidecar longer", [&](const SdTempDir& d) { + std::string json = sdSlurp(d.file("golden.ison.embeddings.json")); + sdSpit(d.file("golden.ison.embeddings.json"), json + " "); }, "stale"}); + cases.push_back({"wrong graph digest in the header", [&](const SdTempDir& d) { + // a valid snapshot of the same rows, tied to another graph + EmbeddingStore tmp; + tmp.loadEmbeddings(d.file("golden.ison.embeddings.json")); + SnapshotIdentity id; + id.graphDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + id.sidecar = EmbeddingStore::sidecarIdOf(d.file("golden.ison.embeddings.json")); + tmp.saveSnapshot(d.file(snapName), id); }, "different graph"}); + cases.push_back({"header has no sidecar identity", [&](const SdTempDir& d) { + EmbeddingStore tmp; + tmp.loadEmbeddings(d.file("golden.ison.embeddings.json")); + SnapshotIdentity id; + id.graphDigest = "sha256:a33458853f63a977962395b84c42fb347ed0f6ed66bc00eabb92ef69bb83026c"; + tmp.saveSnapshot(d.file(snapName), id); }, "no sidecar identity"}); + cases.push_back({"header has no graph digest", [&](const SdTempDir& d) { + EmbeddingStore tmp; + tmp.loadEmbeddings(d.file("golden.ison.embeddings.json")); + SnapshotIdentity id; + id.sidecar = EmbeddingStore::sidecarIdOf(d.file("golden.ison.embeddings.json")); + tmp.saveSnapshot(d.file(snapName), id); }, "different graph"}); + + SdTempDir reference("goldref"); + sdCopyGolden(reference, false); + SemanticGraph expected = SemanticGraph::load(reference.file("golden.ison"), encoder); + + for (const auto& c : cases) { + SdTempDir dir("goldbad"); + sdCopyGolden(dir, true); + c.damage(dir); + LoadReport report; + SemanticGraph loaded = SemanticGraph::load(dir.file("golden.ison"), encoder, "auto", true, true, + {"name"}, GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Ignore, std::nullopt, &report); + if (report.source != LoadSource::Json) throw std::runtime_error(std::string(c.name) + ": snapshot was used"); + if (!report.snapshotSkipped.has_value()) throw std::runtime_error(std::string(c.name) + ": no reason given"); + if (report.snapshotSkipped->find(c.reason) == std::string::npos) { + throw std::runtime_error(std::string(c.name) + ": reason was '" + *report.snapshotSkipped + "'"); + } + ASSERT_EQ(report.embeddings, 36u); + loaded.embeddingStore().verifyInvariants(); + // what was loaded is what the JSON says (a rewritten sidecar shows through) + SemanticGraph fromJson = [&] { + std::error_code ec; + std::filesystem::remove(dir.file(snapName), ec); + return SemanticGraph::load(dir.file("golden.ison"), encoder); + }(); + sdExpectSameStore(loaded.embeddingStore(), fromJson.embeddingStore()); + } + ASSERT_EQ(expected.embeddingStore().count(), 36u); +} + +TEST(snapshot_is_never_used_without_its_sidecar_and_never_with_embeddings_off) { + auto encoder = std::make_shared(12); + SdTempDir dir("goldnone"); + sdCopyGolden(dir, true); + LoadReport report; + SemanticGraph off = SemanticGraph::load(dir.file("golden.ison"), encoder, "auto", /*embeddings=*/false, true, + {"name"}, GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Ignore, std::nullopt, &report); + ASSERT_TRUE(report.source == LoadSource::NoEmbeddings); + ASSERT_EQ(off.embeddingStore().count(), 0u); + + std::filesystem::remove(dir.file("golden.ison.embeddings.json")); + SemanticGraph missing = SemanticGraph::load(dir.file("golden.ison"), encoder, "auto", true, true, + {"name"}, GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Ignore, std::nullopt, &report); + ASSERT_TRUE(report.source == LoadSource::NoEmbeddings); + ASSERT_FALSE(report.snapshotSkipped.has_value()); + ASSERT_EQ(missing.embeddingStore().count(), 0u); +} + +TEST(snapshot_of_an_edited_graph_is_refused_and_the_json_mismatch_policy_applies) { + auto encoder = std::make_shared(12); + SdTempDir dir("goldedit"); + sdCopyGolden(dir, true); + std::string graph = sdSlurp(dir.file("golden.ison")); + const size_t at = graph.find("Alice"); + graph.replace(at, 5, "Alicia"); + sdSpit(dir.file("golden.ison"), graph); + + LoadReport report; + ASSERT_THROWS(SemanticGraph::load(dir.file("golden.ison"), encoder), GraphEmbeddingMismatchError); + + std::vector warnings; + setWarningHandler([&](const std::string& m) { warnings.push_back(m); }); + SemanticGraph warned = SemanticGraph::load(dir.file("golden.ison"), encoder, "auto", true, true, {"name"}, + GraphMismatchMode::Warn, std::nullopt, ModelMismatchMode::Ignore, + std::nullopt, &report); + setWarningHandler(nullptr); + ASSERT_TRUE(report.source == LoadSource::Json); + ASSERT_TRUE(report.snapshotSkipped.has_value()); + ASSERT_TRUE(report.snapshotSkipped->find("different graph") != std::string::npos); + ASSERT_EQ(warned.embeddingStore().count(), 36u); + ASSERT_EQ(warnings.size(), 1u); +} + +// ---- snapshot: our own save -> load ------------------------------------------------ + +TEST(save_with_snapshot_round_trips_and_the_other_files_are_unchanged) { + SdTempDir dir("snaprt"), plain("snapplain"); + auto encoder = std::make_shared(16); + auto build = [&]() { + SemanticGraph g("persist", encoder); + for (int i = 0; i < 300; ++i) { + g.addNode(i % 2 ? "doc" : "note", std::to_string(i), + {{"name", "item " + std::to_string(i)}, {"text", "line\n\"quoted\" \xc3\xa9 " + std::to_string(i)}}); + } + g.addEdge("REL", NodeRef("doc", "1"), NodeRef("note", "2")); + for (int i = 0; i < 300; i += 7) g.embeddingStore().add(NodeRef("doc", std::to_string(i)), "updated " + std::to_string(i), + SdRng(static_cast(i)).vec(16)); + for (int i = 3; i < 300; i += 11) g.removeNode(i % 2 ? "doc" : "note", std::to_string(i)); + return g; + }; + SemanticGraph graph = build(); + const std::string path = dir.file("g.ison"); + graph.save(path, "auto", true, /*snapshot=*/true); + ASSERT_TRUE(std::filesystem::exists(path + ".embeddings.snap")); + SemanticGraph twin = build(); + twin.save(plain.file("g.ison")); + ASSERT_FALSE(std::filesystem::exists(plain.file("g.ison.embeddings.snap"))); + // graph file and JSON sidecar are byte-identical with or without the snapshot + ASSERT_TRUE(sdSlurp(path) == sdSlurp(plain.file("g.ison"))); + ASSERT_TRUE(sdSlurp(path + ".embeddings.json") == sdSlurp(plain.file("g.ison.embeddings.json"))); + + LoadReport report; + SemanticGraph loaded = SemanticGraph::load(path, encoder, "auto", true, true, {"name", "text"}, + GraphMismatchMode::Error, std::nullopt, ModelMismatchMode::Warn, + std::nullopt, &report); + ASSERT_TRUE(report.source == LoadSource::Snapshot); + SemanticGraph viaJson = SemanticGraph::load(plain.file("g.ison"), encoder); + sdExpectSameStore(loaded.embeddingStore(), viaJson.embeddingStore()); + sdExpectSameStore(loaded.embeddingStore(), graph.embeddingStore()); + ASSERT_EQ(report.embeddings, graph.embeddingStore().count()); + // text search through the loaded graph answers like the original's + ASSERT_TRUE(sameHits(loaded.similaritySearch("item 41", 5, "", -1.0f), graph.similaritySearch("item 41", 5, "", -1.0f))); + loaded.embeddingStore().verifyInvariants(); + + // a loaded store keeps working: writes, deletes, ANN + AnnOptions o; + o.nlist = 16; o.nprobe = 4; o.minSize = 0; + loaded.embeddingStore().enableAnn(o); + loaded.embeddingStore().add(NodeRef("new", "1"), "n", SdRng(5).vec(16)); + ASSERT_EQ(loaded.embeddingStore().similaritySearchVector(SdRng(5).vec(16), 1)[0].node_ref.type, "new"); + loaded.embeddingStore().verifyInvariants(); + + // saving again WITHOUT a snapshot deletes the old one (the sidecar is about to change) + graph.save(path); + ASSERT_FALSE(std::filesystem::exists(path + ".embeddings.snap")); + graph.save(path, "auto", true, true); + ASSERT_TRUE(std::filesystem::exists(path + ".embeddings.snap")); + // and a graph-only save leaves things alone but the old snapshot can no longer match + graph.removeNode("doc", "1"); + graph.save(path, "auto", false); + setWarningHandler([](const std::string&) {}); + LoadReport after; + SemanticGraph stale = SemanticGraph::load(path, encoder, "auto", true, true, {"name"}, GraphMismatchMode::Warn, + std::nullopt, ModelMismatchMode::Ignore, std::nullopt, &after); + setWarningHandler(nullptr); + ASSERT_TRUE(after.source == LoadSource::Json); + ASSERT_TRUE(after.snapshotSkipped.has_value()); +} + +TEST(snapshot_of_an_empty_store_and_a_failed_write_leave_nothing_behind) { + SdTempDir dir("snapempty"); + auto encoder = std::make_shared(8); + SemanticGraph graph("g", encoder, /*autoEmbed=*/false); + graph.addNode("p", "1", {{"name", "x"}}); + graph.save(dir.file("g.ison"), "auto", true, true); + LoadReport report; + SemanticGraph back = SemanticGraph::load(dir.file("g.ison"), encoder, "auto", true, true, {"name"}, + GraphMismatchMode::Error, std::nullopt, ModelMismatchMode::Warn, + std::nullopt, &report); + ASSERT_TRUE(report.source == LoadSource::Snapshot); + ASSERT_EQ(back.embeddingStore().count(), 0u); + + // a snapshot that cannot be written: the pair is saved, the error is thrown, no temp file is left + SemanticGraph other("g", encoder); + other.addNode("p", "1", {{"name", "x"}}); + std::filesystem::create_directory(dir.file("blocked.ison.embeddings.snap")); // a directory in the way + bool refused = false; + try { + other.save(dir.file("blocked.ison"), "auto", true, true); + } catch (const EmbeddingError&) { + refused = true; + } + ASSERT_TRUE(refused); + ASSERT_TRUE(std::filesystem::exists(dir.file("blocked.ison"))); + ASSERT_TRUE(std::filesystem::exists(dir.file("blocked.ison.embeddings.json"))); + for (const auto& entry : std::filesystem::directory_iterator(dir.path())) { + ASSERT_TRUE(entry.path().extension() != ".tmp"); + } + ASSERT_THROWS(EmbeddingStore().saveSnapshot(dir.file("no/such/dir/x.snap")), EmbeddingError); +} + +TEST(snapshot_loaded_store_scores_like_a_json_loaded_one_with_the_index_on) { + ClusteredData gen(25, 20, 41); + auto data = gen.drawMany(3500); + EmbeddingStore source; + fillStore(source, data); + SdTempDir dir("snapann"); + source.saveSnapshot(dir.file("a.snap")); + source.saveEmbeddings(dir.file("a.json")); + AnnOptions o; + o.nlist = 32; o.nprobe = 6; o.minSize = 0; + EmbeddingStore fromSnap, fromJson; + fromSnap.enableAnn(o); + fromJson.enableAnn(o); + fromSnap.loadSnapshot(dir.file("a.snap")); + fromJson.loadEmbeddings(dir.file("a.json")); + for (const auto& q : gen.drawMany(20)) { + ASSERT_TRUE(sameHits(fromSnap.similaritySearchVector(q, 10, "", -1.0f), fromJson.similaritySearchVector(q, 10, "", -1.0f))); + ASSERT_TRUE(sameHits(fromSnap.similaritySearchVector(q, 10, "", -1.0f, true), source.similaritySearchVector(q, 10, "", -1.0f, true))); + } + ASSERT_TRUE(fromSnap.annStats().active); + ASSERT_EQ(fromSnap.annStats().builds, 1u); + // loading a snapshot drops an index built on the old rows + fromSnap.loadSnapshot(dir.file("a.snap")); + ASSERT_FALSE(fromSnap.annStats().active); +} + // ============================================================================= // Main // ============================================================================= From 3ce90771d0ca3567fecd65eb7922cd2a9720518d Mon Sep 17 00:00:00 2001 From: Mahesh Vaikri Date: Tue, 17 Nov 2026 08:50:28 -0600 Subject: [PATCH 6/7] Docs: Stage D ports (CHANGELOG, ROADMAP, READMEs) and snapshot spec clarifications DESIGN.md section 11: row order is the writing port's sidecar order (readers must not rely on it), repeated-key and non-UTF-8 rejection spelled out, count limit for ports with bounded arrays, short-file fingerprint windows, and the fact that ids are text (ports with integer ids restore the kind from the graph; a v2 should carry an id-kind bit). Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++--- README.md | 15 +++++++++ ROADMAP.md | 15 +++++---- isongraph-vector-cpp/README.md | 10 ++++-- isongraph-vector-csharp/README.md | 8 +++-- isongraph-vector-js/README.md | 4 +-- isongraph-vector-rs/DESIGN.md | 32 ++++++++++++++---- isongraph-vector-ts/README.md | 4 +-- isongraph_vector-py/README.md | 15 +++++++++ 9 files changed, 130 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a56134a..1c377cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -208,6 +208,51 @@ JavaScript, Rust and C++. Costs: single-thread exact scan is 0-13% slower, ingest is slower, and updates at 1M are 2-3x slower because index upkeep moved from the first search to the write. +- **Python, C#, TypeScript, JavaScript and C++: one copy of every vector, + batch search, binary snapshot** (the Rust design of `DESIGN.md` sections 1, 5 + and 11 carried over; measured per port below). Each store now keeps every + vector once in a chunked (or contiguous) float32 table and records refer to + its rows, so the second copy that the index or the scan cache used to hold is + gone, and pending rows fold into the IVF index at write time (JavaScript / + TypeScript keep their search-time fold: a fold inside a write would block it + for seconds). Memory at 100k x 384 (raw 146 MB, steady RSS above baseline, + 3 runs, shared laptop): Python 437 -> 211 MB, TypeScript / JavaScript 435 -> + 197 MB (590 -> 203 MB with the index), C# IVF store 409 -> 215 MB, C++ IVF + store 329 -> 167 MB; at 1M x 384 (one run each) the new figures are 1.12-1.25x + raw (Python 1.25x, TypeScript 1.21x, C# 1.15x exact / 1.18x with IVF, C++ + 1.12x with IVF; JavaScript was not measured at 1M); + the old 3.3-7.0 GB figures are in `benchmarks/competitive/`. New: + `similarity_search_batch` (Python: one matrix-matrix product; 8x one-at-a-time + at 100k), `SimilaritySearchBatch` (C#), `similaritySearchBatch` (C++), + parallel across queries and bit-identical to single-query search in C# and + C++; searches are safe to run concurrently in C# and C++. The optional + `.embeddings.snap` file (`save(snapshot=True)` / `Save(..., snapshot: true)` / + `saveGraph(..., {snapshot: true})`, byte layout in Rust `DESIGN.md` section + 11) is written and read by all six ports, is 3-5x smaller and roughly + 13-45x faster to save and load (C#, C++, Python) than the JSON sidecar at 100k x 384, and is the only + way to persist a 100k-row store in JavaScript (the JSON export exceeds V8's + string limit). A snapshot is used only when it provably belongs to the graph + and sidecar on disk, otherwise the JSON is read; graph files, sidecar bytes + and digests are unchanged. Cross-port: a snapshot written by any port loads + in every other port (`scripts/cross_port/graph_files.py`). + **Breaking, per port:** `get` / `get_embedding` return a copy of the vector + in C#, C++ (`std::optional`), TypeScript and JavaScript; + `AnnStats.vectorBytes` / `cacheBytes` are 0 (the index owns no vectors); + pending rows fold below 256 at write time (Python, C#, C++); + TypeScript / JavaScript store vectors as float32 (inputs that are not + float32-exact read back rounded: `0.1` -> `0.10000000149011612`; scores + changed by at most 1.3e-8 in a 1,500-vector check); C# `Add` copies the + caller's array; Python in-memory stores no longer have a SQLite table; equal + scores order by arrival in C# and C++. **Python exact top-k scores are now + canonical:** BLAS narrows the field and the rows near the k-th score are + rescored with one kernel, so batch equals single bit for bit; scores can + differ from the previous release in the last float32 bit and near-ties can + reorder (`store.canonical_scores = False` restores the old behaviour). + Costs: single-query IVF is within noise in Python and C++, about 9% slower in + TypeScript / JavaScript and 8-25% slower in C# (its scalar kernel keeps C#'s + own summation order, so C# scores can still differ from Rust's in the last + bits, as before); C++ IVF writes cost about 5 us more because of the + fold. - **Async encoders in TypeScript / JavaScript** - `AsyncEncoder` and `...Async` methods (`addNodeAsync`, `updateNodeAsync`, `embedAllNodesAsync`, `reembedStaleAsync`, `similaritySearchAsync`, `semanticMultiHopAsync`, ...) @@ -242,15 +287,16 @@ JavaScript, Rust and C++. minutes at 1M vectors). - **Not a vector-database replacement.** Against faiss, hnswlib and usearch the ports are 2-4x slower per query at 100k and about 14x slower at 1M text - vectors at equal recall; only Rust has batched and concurrent search (the other ports serialise on one lock); vectors - persist as JSON only; and memory is 2-4x higher (Rust is now 1.1x raw; the other ports still store vectors twice). Graph-aware retrieval + vectors at equal recall; only Rust, C# and C++ have concurrent search and Python has batch search (TypeScript and JavaScript are single-threaded); the ANN index is + never persisted and the binary snapshot is opt-in (JSON is the portable default); and memory is 1.12-1.25x raw at 1M in the four ports measured, against 1.6-1.9 GB for faiss / hnswlib at 1M x 384 (re-run `benchmarks/competitive/` before quoting the + ratios: they predate the rework). Graph-aware retrieval beating plain vector search on multi-hop questions is the supported claim, and it depends on the graph carrying real structure (`benchmarks/multihop/README.md`). - Fast filtered search was only measured for node-type filters, and only Rust is fast there (Python's filtered search is no faster than its unfiltered - scan). The C++ exact scan is scalar and about 6x slower than numpy; the C# - and JS exact scans are also slower than numpy. + scan). The C++, C# and JS exact scans are scalar and still slower than numpy + (about 2x faster than before this change in C++ and C#, 1.6x in JS). - The digest cannot tell `1` from `1.0` in JavaScript / TypeScript, and integers beyond 64 bits are not supported in C# / C++ / Rust. Where a graph contains such values the loader warns instead of raising a false mismatch. diff --git a/README.md b/README.md index e9c2fd6..13c8e34 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,21 @@ graph = SemanticGraph.load("social.isong", encoder=my_encoder) Pass the `encoder=` you saved with. Without it, the first text search on a loaded graph constructs a `SentenceTransformerEncoder` and downloads a model. +**Large stores: the optional binary snapshot.** The JSON sidecar is portable but +big and slow at scale (450-810 MB and 4-16 s to write at 100k x 384 vectors, by port; +past that, JavaScript cannot build the string at all). `graph.save(path, +snapshot=True)` also writes `.embeddings.snap`, a checksummed binary copy +of the same embeddings that is about 3-5x smaller and an order of magnitude +faster to save and load. `load` uses it only when it provably belongs to the +graph and sidecar on disk, and otherwise reads the JSON, so the JSON stays the +portable default and nothing about the graph file or its digest changes. +`load_with_report` says which one it used and why a snapshot was skipped. Every +port reads and writes the same file (format: `isongraph-vector-rs/DESIGN.md` +section 11); the ANN index is not in it and is rebuilt after load. Snapshot +spelling per port: `save(path, snapshot=True)` (Python), `saveGraph(graph, path, +{ snapshot: true })` (TypeScript / JavaScript), `Save(path, snapshot: true)` (C#), +`save(path, "auto", true, true)` (C++), `save_with_snapshot(path, None)` (Rust). + The other five ports persist embeddings to SQLite and exchange the portable payload, but the graph itself stays in memory there. diff --git a/ROADMAP.md b/ROADMAP.md index 8e3dbf4..5831392 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,18 +14,21 @@ What is deliberately not done yet, in rough priority order. Items marked (`isongraph-vector-rs/DESIGN.md`, `benchmarks/rust-stage-c/`): concurrent and batch search, one copy of the vectors, no post-write stall, opt-in HNSW (level with faiss HNSW / hnswlib single-thread), int8 scan, binary snapshot. - Still open: port the proven pieces to the other ports; persist the ANN - index; faster HNSW single-call inserts (1.4k/s at 1M); exact scan is at + One copy of the vectors, batch search and the binary snapshot are now in + every port (HNSW and int8 are still Rust-only). Still open: HNSW for C++ and + C#; persist the ANN index; faster HNSW single-call inserts (1.4k/s at 1M); exact scan is at parity with faiss Flat, not ahead. -5. **Fix the slow ports.** The C++ exact scan has no scan cache and is scalar - (6x slower than numpy); JavaScript k-means takes about 8 minutes at 1M. +5. **Fix the slow ports.** The C++, C# and JS exact scans are scalar and still + slower than numpy; JavaScript k-means takes about 8 minutes at 1M. 6. **Per-record encoder version.** Today a version-only change warns on load but cannot be fixed by `reembed_stale`. ## Later -- **Binary persistence** for vectors and the index, as an addition to the - portable JSON sidecar (which stays the default and keeps the digest). +- **Persist the ANN index** next to the binary snapshot (the snapshot itself + exists in every port; the index is rebuilt after load). +- **Snapshot format v2:** an id-kind bit per row, so ports with integer ids need + no graph lookup to restore them. - **Approximate search for the standalone SQLite stores.** - **Relationship-type weights** (per type rather than a per-edge property), and richer node predicates than a callback. diff --git a/isongraph-vector-cpp/README.md b/isongraph-vector-cpp/README.md index 4fc960e..5321c5b 100644 --- a/isongraph-vector-cpp/README.md +++ b/isongraph-vector-cpp/README.md @@ -337,11 +337,15 @@ need holds (`nprobe >= nlist` is the exact scan). What stays the same: when writes and deletes since training pass half the trained size the index retrains. -The index keeps its own contiguous copy of the vectors (`AnnStats::vectorBytes`), -so an indexed store holds them twice: 1.5 GB more at 1M x 384. It builds on all +The index owns no vectors (`AnnStats::vectorBytes` is 0): it refers to the rows of +the store's single chunked matrix, so an indexed store is about 1.12x the raw +vectors at 1M x 384 (was 2.2x). Pending rows fold into their cells on the write +that makes 256 of them. It builds on all hardware threads by default (`AnnOptions::threads`; the result does not depend on the count) and is seeded, so the same data gives the same index however it was -inserted. Searches on one store take turns on the index. +inserted. Searches are `const` and share a lock, so any number run at once; +`similaritySearchBatch` runs a batch of queries on a bounded set of threads and +returns exactly what one query at a time would. Measured with `bench/ann_bench.cpp` (build with `-DISONGRAPH_VECTOR_BENCH=ON`; 384-d synthetic clustered data, top-10, single-threaded queries, a 32-thread diff --git a/isongraph-vector-csharp/README.md b/isongraph-vector-csharp/README.md index adac3b0..6ab88c2 100644 --- a/isongraph-vector-csharp/README.md +++ b/isongraph-vector-csharp/README.md @@ -357,9 +357,11 @@ so a selective filter costs time, not results. The index is built lazily by the first search, never saved (nothing about it is in a graph file or sidecar, and the digest is unchanged), and kept current: added and updated vectors are visible at once and folded into cells in batches, -removed ones never come back, and heavy churn retrains it. It holds its own -contiguous copy of the vectors, so budget about one extra copy of them. Writes and -searches share the store's lock, as everywhere else in the store. It is not +removed ones never come back, and heavy churn retrains it. It owns no vectors: it +refers to the rows of the store's single chunked table (an IVF-indexed store is +about 1.2-1.5x the raw vectors, was 2.8x). Searches share a reader lock and run +concurrently; writes are exclusive; `SimilaritySearchBatch` runs queries in +parallel and returns exactly what one query at a time would. It is not available on `SqliteEmbeddingStore`. Measured on synthetic clustered 384-d vectors (32 cores, .NET 8, top-10, warm medians): diff --git a/isongraph-vector-js/README.md b/isongraph-vector-js/README.md index c71663d..4573e1f 100644 --- a/isongraph-vector-js/README.md +++ b/isongraph-vector-js/README.md @@ -301,8 +301,8 @@ index). `annNprobe >= annNlist` is the exact scan. (`nodeType`, `threshold`, `nodeFilter`) are applied to the probed cells, and the probe widens (4x per step, ending in the exact scan) when too few candidates survive, so a selective filter costs time, not results. -- **Costs memory.** A Float32Array copy of the vectors (4 bytes per component), - next to the stored ones, plus about 8 bytes per vector for the cell lists. +- **Costs little memory.** The index reads the store's single float32 table (it + owns no copy of the vectors), plus about 8 bytes per vector for the cell lists. - **Faster to query, slow to build.** Without BLAS the exact scan of 384-d vectors takes about 5 ms at 10k, 50 ms at 100k and 0.5 s at 1M; the index answers in 0.2-3 ms at 10k-100k (recall@10 1.00 at the default probe) and diff --git a/isongraph-vector-rs/DESIGN.md b/isongraph-vector-rs/DESIGN.md index ae38be3..4f10133 100644 --- a/isongraph-vector-rs/DESIGN.md +++ b/isongraph-vector-rs/DESIGN.md @@ -308,7 +308,9 @@ sidecar bytes and the graph digest are byte-identical with or without a snapshot A snapshot is never used when the sidecar is missing. * Sidecar identity = `len` (u64) and `fingerprint` = SHA-256 over: `len` as u64 LE, the first 65,536 bytes, the last 65,536 bytes, and for i in 1..=64 the 4,096 - bytes at offset `floor(len * i / 65)` (each clipped to the file end). Any + bytes at offset `floor(len * i / 65)` (each clipped to the file end; for a file + shorter than that the windows overlap; every window is still hashed, in order, even + where it overlaps another, and a window that starts at or past the end is empty). Any serializer rewriting the sidecar changes it; **limit**: an in-place edit that keeps the length and touches no sampled byte is not detected. Tools that rewrite the sidecar must delete the `.snap` (this crate's `save` does). @@ -335,12 +337,28 @@ block * ceil(count / block_rows) R = min(block_rows, rows left) trailer 32 bytes SHA-256 of every byte before it (from offset 0) ``` `string` = `u32 length` + that many UTF-8 bytes. `opt string` = the same, with length -`0xFFFFFFFF` meaning absent. Rows are in the **sidecar's order**: live rows only, -sorted by node type then id (byte order), so loading a snapshot yields the same -slots, tombstone-free, as loading the JSON (equal score ties resolve identically). -`key = type ":" id`. A reader must reject: a length past the end of the file, a -model index >= n_models, a repeated key, `count * dim * 4` larger than the file, -bytes between the last block and the trailer or after it, a trailer mismatch. +`0xFFFFFFFF` meaning absent. Rows are in **the writing port's own sidecar order**, +live rows only, so that loading a snapshot yields the same slots, tombstone-free, +as loading that port's JSON (equal score ties resolve identically *within a port*). +The Rust crate sorts by node type then id (byte order); Python, TypeScript and +JavaScript write arrival order, C# its own comparer. A reader must **not** check or +rely on row order, and tie order after a load can differ between ports. +`key = type ":" id`; two rows whose keys are equal as strings (type `a:b` with id +`c`, and type `a` with id `b:c`) are a repeated key: a reader rejects the file and +falls back to the JSON, which keeps them apart in ports with tuple keys. +A reader must reject: a length past the end of the file, a string that is not +valid UTF-8, a model index >= n_models, a repeated key, `count * dim * 4` larger +than the file, bytes between the last block and the trailer (bytes "after" the +trailer cannot be told apart from these: the trailer is the last 32 bytes), a +count above what the reader can hold (the format allows 2^32-1; a port whose arrays +are limited, such as .NET, rejects more and falls back), a trailer mismatch. +**Ids are text.** The format carries no id kind, so a snapshot cannot say that +`("person", 1)` was an integer id. Ports whose ids can be integers (Python, +TypeScript, JavaScript) restore the kind from the graph (a string id that names a +node stays a string; a canonical integer string that names an integer node becomes +the integer), and refuse to write a snapshot for an embedding that would not restore +identically (they save the JSON pair, remove any stale `.snap` and report why). +A version 2 should add an id-kind bit per row. The reader sets `dimension` from the first row (as the JSON import does), and the import metadata (encoder model/version, digest, basis) from the header. diff --git a/isongraph-vector-ts/README.md b/isongraph-vector-ts/README.md index 411dde1..aed0b00 100644 --- a/isongraph-vector-ts/README.md +++ b/isongraph-vector-ts/README.md @@ -251,8 +251,8 @@ index). `annNprobe >= annNlist` is the exact scan. (`nodeType`, `threshold`, `nodeFilter`) are applied to the probed cells, and the probe widens (4x per step, ending in the exact scan) when too few candidates survive, so a selective filter costs time, not results. -- **Costs memory.** A Float32Array copy of the vectors (4 bytes per component), - next to the stored ones, plus about 8 bytes per vector for the cell lists. +- **Costs little memory.** The index reads the store's single float32 table (it + owns no copy of the vectors), plus about 8 bytes per vector for the cell lists. - **Faster to query, slow to build.** Without BLAS the exact scan of 384-d vectors takes about 5 ms at 10k, 50 ms at 100k and 0.5 s at 1M; the index answers in 0.2-3 ms at 10k-100k (recall@10 1.00 at the default probe) and diff --git a/isongraph_vector-py/README.md b/isongraph_vector-py/README.md index 120927c..c9c8ccf 100644 --- a/isongraph_vector-py/README.md +++ b/isongraph_vector-py/README.md @@ -335,6 +335,21 @@ Pass the `encoder=` you saved with. Without it the first text search on a loaded graph constructs a `SentenceTransformerEncoder` and downloads a model, which is rarely what you want in a test. +**Large stores: the optional binary snapshot.** The JSON sidecar is portable but +big and slow at scale (450-810 MB and 4-16 s to write at 100k x 384 vectors, by port; +past that, JavaScript cannot build the string at all). `graph.save(path, +snapshot=True)` also writes `.embeddings.snap`, a checksummed binary copy +of the same embeddings that is about 3-5x smaller and an order of magnitude +faster to save and load. `load` uses it only when it provably belongs to the +graph and sidecar on disk, and otherwise reads the JSON, so the JSON stays the +portable default and nothing about the graph file or its digest changes. +`load_with_report` says which one it used and why a snapshot was skipped. Every +port reads and writes the same file (format: `isongraph-vector-rs/DESIGN.md` +section 11); the ANN index is not in it and is rebuilt after load. Snapshot +spelling per port: `save(path, snapshot=True)` (Python), `saveGraph(graph, path, +{ snapshot: true })` (TypeScript / JavaScript), `Save(path, snapshot: true)` (C#), +`save(path, "auto", true, true)` (C++), `save_with_snapshot(path, None)` (Rust). + **The two files stay a pair.** Each is written to a temp file and renamed into place, so a crash never leaves a truncated file; and since two files cannot be replaced as one, the sidecar records a `graph_digest` (sha256 over a canonical From 437c02f41d197d2a75410c7c20ccad966d271afe Mon Sep 17 00:00:00 2001 From: Mahesh Vaikri Date: Tue, 17 Nov 2026 08:54:12 -0600 Subject: [PATCH 7/7] CI: cross-port interchange covers the binary snapshot Every port writes the snapshot, every port must accept every other port's snapshot (source == snapshot), a stale snapshot beside an edited graph must not mask the mismatch error, and a copy without the .snap must load through the JSON with the same observation. 108/108 locally. Also repairs the C++ helper for getEmbedding returning std::optional. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 7 +- scripts/cross_port/cpp/main.cpp | 28 ++++--- scripts/cross_port/csharp/Program.cs | 17 +++- scripts/cross_port/graph_files.py | 111 ++++++++++++++++++++++----- scripts/cross_port/graph_node.mjs | 15 ++-- scripts/cross_port/graph_py.py | 18 +++-- scripts/cross_port/rust/src/main.rs | 27 ++++--- 7 files changed, 168 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37b79a9..0224a26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -331,6 +331,11 @@ jobs: # in each file and requires every port to raise its mismatch error. The # script builds the Rust, C#, C++ and TypeScript helpers outside the # checkout (scripts/cross_port/{rust,csharp,cpp}) and prints a PASS/FAIL - # table; any failing pair fails the job. + # table; any failing pair fails the job. Each writer also writes the + # optional binary snapshot (.embeddings.snap), and three matrices + # run: every reader must accept every writer's snapshot (source must be + # 'snapshot'); every reader must still raise its mismatch error for an + # edited graph with a stale snapshot beside it; and with the snapshot + # deleted the JSON path must give the same observation. - name: Graph files and sidecars, all six ports, all pairs run: python scripts/cross_port/graph_files.py diff --git a/scripts/cross_port/cpp/main.cpp b/scripts/cross_port/cpp/main.cpp index 4401d9d..710f7c7 100644 --- a/scripts/cross_port/cpp/main.cpp +++ b/scripts/cross_port/cpp/main.cpp @@ -5,8 +5,9 @@ // cross_port_graph mismatch // // `write` saves the shared fixture graph as /graph.ison plus its -// embeddings sidecar. `read` loads it with GraphMismatchMode::Error and writes -// what it found to . `mismatch` loads a graph whose file was +// embeddings sidecar and binary snapshot. `read` loads it with +// GraphMismatchMode::Error and writes what it found (and the load source, +// snapshot or json, with the reason a snapshot was skipped) to . `mismatch` loads a graph whose file was // edited after the save and exits 0 only if the load throws // GraphEmbeddingMismatchError. // @@ -140,18 +141,21 @@ void write(const std::string& dir) { graph.addEdge(e.rel, e.source, e.target, e.props); } std::filesystem::create_directories(dir); - graph.save(graphPath(dir)); + graph.save(graphPath(dir), "auto", true, true); } -SemanticGraph load(const std::string& dir, const std::shared_ptr& encoder) { +SemanticGraph load(const std::string& dir, const std::shared_ptr& encoder, + LoadReport* report = nullptr) { return SemanticGraph::load(graphPath(dir), encoder, "auto", true, true, {"name", "description", "content", "text"}, - GraphMismatchMode::Error); + GraphMismatchMode::Error, std::nullopt, + ModelMismatchMode::Warn, std::nullopt, report); } void read(const std::string& dir, const std::string& output) { auto encoder = std::make_shared(DIM); - SemanticGraph graph = load(dir, encoder); + LoadReport report; + SemanticGraph graph = load(dir, encoder, &report); std::string out = "{\"port\":\"cpp\",\"nodes\":["; std::vector missing; @@ -162,8 +166,8 @@ void read(const std::string& dir, const std::string& output) { first = false; out += "{\"type\":" + jstr(node.type) + ",\"id\":" + jstr(node.id) + ",\"props\":" + jprops(node.properties) + "}"; - const EmbeddingRecord* record = graph.getEmbedding(NodeRef(node.type, node.id)); - if (record == nullptr) { + const std::optional record = graph.getEmbedding(NodeRef(node.type, node.id)); + if (!record) { missing.push_back(NodeRef(node.type, node.id)); return; } @@ -198,7 +202,13 @@ void read(const std::string& dir, const std::string& output) { (hits.empty() ? std::string("null") : jref(hits[0].node_ref)) + ",\"score\":" + (hits.empty() ? std::string("null") : jnum(hits[0].score)) + "}"; } - out += "]}"; + out += "],\"source\":"; + out += report.source == LoadSource::Snapshot ? "\"snapshot\"" + : report.source == LoadSource::Json ? "\"json\"" + : "\"none\""; + out += ",\"snapshot_skipped\":" + + (report.snapshotSkipped ? jstr(*report.snapshotSkipped) : std::string("null")); + out += "}"; std::ofstream file(output, std::ios::binary); file << out; diff --git a/scripts/cross_port/csharp/Program.cs b/scripts/cross_port/csharp/Program.cs index fadf319..5d70a1d 100644 --- a/scripts/cross_port/csharp/Program.cs +++ b/scripts/cross_port/csharp/Program.cs @@ -5,8 +5,9 @@ // CrossPortGraph mismatch // // `write` saves the shared fixture graph as /graph.ison plus its -// embeddings sidecar. `read` loads it with GraphMismatchMode.Error and writes -// what it found to . `mismatch` loads a graph whose file was +// embeddings sidecar and binary snapshot. `read` loads it with +// GraphMismatchMode.Error and writes what it found (and the load source, +// snapshot or json, with the reason a snapshot was skipped) to . `mismatch` loads a graph whose file was // edited after the save and exits 0 only if the load throws // GraphEmbeddingMismatchException. // @@ -50,7 +51,7 @@ void Write(string dir) foreach (var (rel, source, target, props) in Edges) graph.AddEdge(rel, source, target, props); Directory.CreateDirectory(dir); - graph.Save(GraphPath(dir)); + graph.Save(GraphPath(dir), snapshot: true); } void WriteValue(Utf8JsonWriter w, object? value) @@ -90,7 +91,7 @@ void WriteRef(Utf8JsonWriter w, NodeRef r) void Read(string dir, string output) { var encoder = new MockEncoder(Dim); - var graph = SemanticGraph.Load(GraphPath(dir), encoder, onMismatch: GraphMismatchMode.Error); + var (graph, report) = SemanticGraph.LoadWithReport(GraphPath(dir), encoder, onMismatch: GraphMismatchMode.Error); using var stream = new MemoryStream(); using (var w = new Utf8JsonWriter(stream)) @@ -155,6 +156,14 @@ void Read(string dir, string output) w.WriteEndObject(); } w.WriteEndArray(); + w.WriteString("source", report.Source switch + { + LoadSource.Snapshot => "snapshot", + LoadSource.Json => "json", + _ => "none", + }); + w.WritePropertyName("snapshot_skipped"); + if (report.SnapshotSkipped is null) w.WriteNullValue(); else w.WriteStringValue(report.SnapshotSkipped); w.WriteEndObject(); } File.WriteAllBytes(output, stream.ToArray()); diff --git a/scripts/cross_port/graph_files.py b/scripts/cross_port/graph_files.py index 0a36815..52d0f9d 100644 --- a/scripts/cross_port/graph_files.py +++ b/scripts/cross_port/graph_files.py @@ -4,14 +4,19 @@ The SQLite and payload checks (node_write.mjs, py_read.py and friends) prove the ports share one embeddings format. This proves the same for what `SemanticGraph.save` writes: a graph file plus a `.embeddings.json` -sidecar carrying a `graph_digest` that `load` verifies. +sidecar carrying a `graph_digest` that `load` verifies, and the optional binary +snapshot `.embeddings.snap` (DESIGN.md section 11). Every port has a small standalone helper (graph_py.py, graph_node.mjs for the JavaScript and TypeScript ports, rust/, csharp/ and cpp/) that can - write save the fixture graph + sidecar into + write save the fixture graph + sidecar + snapshot + into read load /graph.ison with the mismatch mode - set to ERROR and dump what it found + set to ERROR and dump what it found, plus + where the embeddings came from (the load + report: source snapshot/json/none and the + reason a snapshot was skipped) mismatch load a graph edited after the save and exit 0 only if the port raised its mismatch error @@ -20,13 +25,21 @@ readers matrix). For each pair it judges the observation the reader produced: * the sidecar digest verified (the load did not raise in ERROR mode); + * the embeddings came from the binary snapshot (load source "snapshot"), so + every reader accepts the snapshot of every writer, itself included; * node and edge counts, every node's properties, every edge's properties; * every stored embedding equals the reader's own MockEncoder(text) within 1e-6; * a text search finds the expected node for three different queries. A second matrix tampers with one property in each written graph file (age 30 -becomes 31) and requires every reader to raise its graph/embeddings mismatch -error rather than load the stale vectors. +becomes 31; the snapshot is left in place) and requires every reader to raise +its graph/embeddings mismatch error rather than load the stale vectors, from +the snapshot or from the sidecar. + +A third matrix deletes the snapshot from a copy of each written directory and +requires every reader to load it through the JSON sidecar (load source "json") +and to see the same thing as the snapshot load: the same nodes, edges, number of +embeddings, vectors check and search results. Allowances, each visible in the output rather than silent: @@ -68,6 +81,7 @@ KNOWN_SKIPS: dict = {} TOLERANCE = 1e-6 +SNAPSHOT_NAME = "graph.ison.embeddings.snap" # What every helper writes. Keep in step with graph_py.py and friends. BS = chr(92) @@ -213,9 +227,13 @@ def props_problem(expected, actual, drops_nulls): return None -def judge(writer, reader, obs): +def judge(writer, reader, obs, source="snapshot"): """Return a list of problems with what `reader` loaded from `writer`'s files.""" problems = [] + if obs.get("source") != source: + why = obs.get("snapshot_skipped") + problems.append(f"embeddings came from {obs.get('source')!r}, expected {source!r}" + + (f" (snapshot skipped: {why})" if why else "")) strict_ids = writer in TYPED_IDS and reader in TYPED_IDS drops_nulls = reader in DROPS_NULLS @@ -277,6 +295,31 @@ def id_problem(ref, where): return problems +def same_observation(snap, js): + """Problems if the JSON-path observation differs from the snapshot-path one.""" + problems = [] + # Iteration order is not part of the contract (Rust walks hash maps), so + # compare nodes and edges as sets. + def canon(items): + return sorted(json.dumps(item, sort_keys=True, ensure_ascii=False) for item in items) + + for key in ("nodes", "edges", "missing_embeddings"): + if canon(snap[key]) != canon(js[key]): + problems.append(f"{key} differ between the snapshot load and the JSON load") + if snap["embedding_count"] != js["embedding_count"]: + problems.append(f"embedding count differs: snapshot {snap['embedding_count']}, " + f"json {js['embedding_count']}") + if abs(snap["max_vec_diff"] - js["max_vec_diff"]) > TOLERANCE: + problems.append(f"vectors check differs: snapshot {snap['max_vec_diff']:.3g}, " + f"json {js['max_vec_diff']:.3g}") + for a, b in zip(snap["searches"], js["searches"]): + if a["top"] != b["top"] or (a["score"] is not None and b["score"] is not None + and abs(a["score"] - b["score"]) > TOLERANCE): + problems.append(f"search {a['query'][:24]!r} differs: snapshot {a['top']} " + f"{a['score']}, json {b['top']} {b['score']}") + return problems + + # -- the matrix ------------------------------------------------------------------- def tamper(source_dir, target_dir): @@ -342,7 +385,7 @@ def execute(ports, wd): print("\n".join(failures), file=sys.stderr) return 2 - written, tampered, problems = {}, {}, [] + written, tampered, nosnap, problems = {}, {}, {}, [] for writer in ports: out = wd / "graphs" / writer code, output = run(helpers[writer] + ["write", out], check=False) @@ -350,54 +393,86 @@ def execute(ports, wd): not (out / "graph.ison.embeddings.json").exists(): problems.append(f"{LABEL[writer]} could not write the fixture graph (exit {code}): {output.strip()}") continue + if not (out / SNAPSHOT_NAME).exists(): + problems.append(f"{LABEL[writer]} wrote no binary snapshot ({SNAPSHOT_NAME}) " + f"although it was asked to") + continue written[writer] = out tampered[writer] = wd / "tampered" / writer tamper(out, tampered[writer]) + nosnap[writer] = wd / "nosnap" / writer + shutil.copytree(out, nosnap[writer]) + (nosnap[writer] / SNAPSHOT_NAME).unlink() - roundtrip, detect, details = {}, {}, [] + roundtrip, detect, fallback, details = {}, {}, {}, [] for writer in ports: for reader in ports: pair = (writer, reader) skip = KNOWN_SKIPS.get(pair) if skip: - roundtrip[pair] = detect[pair] = "SKIP" + roundtrip[pair] = detect[pair] = fallback[pair] = "SKIP" details.append(f"SKIP {LABEL[writer]} -> {LABEL[reader]}: {skip}") continue if writer not in written: - roundtrip[pair] = detect[pair] = "FAIL" + roundtrip[pair] = detect[pair] = fallback[pair] = "FAIL" continue obs_path = wd / "obs" / f"{writer}_{reader}.json" obs_path.parent.mkdir(parents=True, exist_ok=True) code, output = run(helpers[reader] + ["read", written[writer], obs_path], check=False) + snap_obs = None if code != 0 or not obs_path.exists(): roundtrip[pair] = "FAIL" details.append(f"FAIL {LABEL[writer]} -> {LABEL[reader]} (load): exit {code}: {output.strip()}") else: - found = judge(writer, reader, json.loads(obs_path.read_text(encoding="utf-8"))) + snap_obs = json.loads(obs_path.read_text(encoding="utf-8")) + found = judge(writer, reader, snap_obs) roundtrip[pair] = "FAIL" if found else "PASS" details.extend(f"FAIL {LABEL[writer]} -> {LABEL[reader]}: {p}" for p in found) + # The same graph with the snapshot deleted: the JSON path, same answer. + json_path = wd / "obs" / f"{writer}_{reader}_nosnap.json" + code, output = run(helpers[reader] + ["read", nosnap[writer], json_path], check=False) + if code != 0 or not json_path.exists(): + fallback[pair] = "FAIL" + details.append(f"FAIL {LABEL[writer]} -> {LABEL[reader]} (no snapshot): " + f"exit {code}: {output.strip()}") + else: + json_obs = json.loads(json_path.read_text(encoding="utf-8")) + found = judge(writer, reader, json_obs, source="json") + if snap_obs is not None: + found += same_observation(snap_obs, json_obs) + else: + found.append("no snapshot observation to compare with") + fallback[pair] = "FAIL" if found else "PASS" + details.extend(f"FAIL {LABEL[writer]} -> {LABEL[reader]} (no snapshot): {p}" + for p in found) + code, output = run(helpers[reader] + ["mismatch", tampered[writer]], check=False) detect[pair] = "PASS" if code == 0 else "FAIL" if code != 0: details.append(f"FAIL {LABEL[writer]} -> {LABEL[reader]} (tampered file): {output.strip()}") print() - print(table("Round trip: reader loads the writer's graph + sidecar in ERROR mode and " - "everything matches", ports, roundtrip)) + print(table("Round trip: reader loads the writer's graph + snapshot in ERROR mode (source " + "must be 'snapshot') and everything matches", ports, roundtrip)) + print() + print(table("Tamper: reader must RAISE its mismatch error for an edited graph file " + "(stale snapshot present)", ports, detect)) print() - print(table("Tamper: reader must RAISE its mismatch error for an edited graph file", ports, detect)) + print(table("No snapshot: reader loads a copy without the .snap through the JSON sidecar " + "(source must be 'json') and sees the same as the snapshot load", ports, fallback)) print() for line in problems + details: print(line) if problems or details: print() - failed = problems or any(v == "FAIL" for v in list(roundtrip.values()) + list(detect.values())) - cells = len(roundtrip) + len(detect) - passed = sum(v == "PASS" for v in list(roundtrip.values()) + list(detect.values())) - skipped = sum(v == "SKIP" for v in list(roundtrip.values()) + list(detect.values())) + results = list(roundtrip.values()) + list(detect.values()) + list(fallback.values()) + failed = problems or any(v == "FAIL" for v in results) + cells = len(results) + passed = sum(v == "PASS" for v in results) + skipped = sum(v == "SKIP" for v in results) print(f"{'FAIL' if failed else 'PASS'}: {passed}/{cells} checks passed" f"{f', {skipped} skipped' if skipped else ''} in {time.time() - start:.1f}s") return 1 if failed else 0 diff --git a/scripts/cross_port/graph_node.mjs b/scripts/cross_port/graph_node.mjs index aa2feb2..3d24f65 100644 --- a/scripts/cross_port/graph_node.mjs +++ b/scripts/cross_port/graph_node.mjs @@ -11,8 +11,9 @@ * outside the repository). * * `write` saves the shared fixture graph as /graph.ison plus its - * embeddings sidecar. `read` loads it with onMismatch 'error' and writes what - * it found to . `mismatch` loads a graph whose file was + * embeddings sidecar and binary snapshot. `read` loads it with onMismatch + * 'error' and writes what it found (and the load source, snapshot or json, with + * the reason a snapshot was skipped) to . `mismatch` loads a graph whose file was * edited after the save and exits 0 only if the load throws * GraphEmbeddingMismatchError. */ @@ -71,10 +72,10 @@ function write(lib, d) { for (const [type, id, props] of NODES) graph.addNodeWithEmbed(type, id, props, props.text); for (const [rel, source, target, props] of EDGES) graph.addEdge(rel, source, target, props); mkdirSync(d, { recursive: true }); - lib.node.saveGraph(graph, graphPath(d)); + lib.node.saveGraph(graph, graphPath(d), { snapshot: true }); } -function observe(graph, encoder) { +function observe(graph, encoder, report) { const nodes = []; const missing = []; let worst = 0; @@ -113,6 +114,8 @@ function observe(graph, encoder) { missing_embeddings: missing, max_vec_diff: worst, searches, + source: report.source, + snapshot_skipped: report.snapshotSkipped, }; } @@ -122,8 +125,8 @@ async function main() { write(lib, dir); } else if (command === 'read' && dir && out) { const encoder = new lib.core.MockEncoder(DIM); - const graph = lib.node.loadGraph(graphPath(dir), { encoder, onMismatch: 'error' }); - writeFileSync(out, JSON.stringify(observe(graph, encoder)), 'utf8'); + const { graph, report } = lib.node.loadGraphWithReport(graphPath(dir), { encoder, onMismatch: 'error' }); + writeFileSync(out, JSON.stringify(observe(graph, encoder, report)), 'utf8'); } else if (command === 'mismatch' && dir) { try { lib.node.loadGraph(graphPath(dir), { encoder: new lib.core.MockEncoder(DIM), onMismatch: 'error' }); diff --git a/scripts/cross_port/graph_py.py b/scripts/cross_port/graph_py.py index ca14d0b..66583f4 100644 --- a/scripts/cross_port/graph_py.py +++ b/scripts/cross_port/graph_py.py @@ -5,8 +5,10 @@ graph_py.py mismatch `write` saves the shared fixture graph as /graph.ison plus its -embeddings sidecar. `read` loads /graph.ison with on_mismatch="error" and -writes what it found to for the orchestrator to judge. +embeddings sidecar and binary snapshot. `read` loads /graph.ison with +on_mismatch="error" and writes what it found (and the load source, snapshot or +json, with the reason a snapshot was skipped) to for the +orchestrator to judge. `mismatch` loads a graph whose file was edited after the save and exits 0 only if the load raises GraphEmbeddingMismatchError. """ @@ -64,11 +66,11 @@ def write(directory: str) -> None: for rel, source, target, props in EDGES: graph.add_edge(rel, source, target, **props) Path(directory).mkdir(parents=True, exist_ok=True) - graph.save(graph_path(directory)) + graph.save(graph_path(directory), snapshot=True) graph.close() -def observe(graph: SemanticGraph, encoder: MockEncoder) -> dict: +def observe(graph: SemanticGraph, encoder: MockEncoder, report) -> dict: nodes, missing, worst = [], [], 0.0 for node in graph.nodes(): nodes.append({"type": node.type, "id": node.id, "props": dict(node.properties)}) @@ -96,13 +98,15 @@ def observe(graph: SemanticGraph, encoder: MockEncoder) -> dict: "score": float(top.score) if top else None}) return {"port": "python", "nodes": nodes, "edges": edges, "embedding_count": graph.embedding_store.count(), - "missing_embeddings": missing, "max_vec_diff": worst, "searches": searches} + "missing_embeddings": missing, "max_vec_diff": worst, "searches": searches, + "source": report.source, "snapshot_skipped": report.snapshot_skipped} def read(directory: str, out: str) -> None: encoder = MockEncoder(DIM) - graph = SemanticGraph.load(graph_path(directory), encoder=encoder, on_mismatch="error") - observation = observe(graph, encoder) + graph, report = SemanticGraph.load_with_report( + graph_path(directory), encoder=encoder, on_mismatch="error") + observation = observe(graph, encoder, report) graph.close() Path(out).write_text(json.dumps(observation, ensure_ascii=False), encoding="utf-8") diff --git a/scripts/cross_port/rust/src/main.rs b/scripts/cross_port/rust/src/main.rs index 5cbc672..efff095 100644 --- a/scripts/cross_port/rust/src/main.rs +++ b/scripts/cross_port/rust/src/main.rs @@ -5,8 +5,9 @@ //! cross_port_graph mismatch //! //! `write` saves the shared fixture graph as /graph.ison plus its -//! embeddings sidecar. `read` loads it with GraphMismatch::Error and writes what -//! it found to . `mismatch` loads a graph whose file was +//! embeddings sidecar and binary snapshot. `read` loads it with +//! GraphMismatch::Error and writes what it found (and the load source, snapshot +//! or json, with the reason a snapshot was skipped) to . `mismatch` loads a graph whose file was //! edited after the save and exits 0 only if the load fails with //! EmbeddingError::GraphEmbeddingMismatch. //! @@ -15,8 +16,8 @@ //! that. use isongraph_vector_rs::{ - EmbeddingEncoder, EmbeddingError, GraphMismatch, LoadOptions, MockEncoder, NodeId, - PropertyValue, SemanticGraph, + EmbeddingEncoder, EmbeddingError, GraphMismatch, LoadOptions, LoadReport, LoadSource, + MockEncoder, NodeId, PropertyValue, SemanticGraph, }; use serde_json::{json, Map, Value as Json}; use std::collections::HashMap; @@ -147,7 +148,7 @@ fn write(dir: &str) -> Res<()> { graph.add_edge(rel, source, target, props)?; } std::fs::create_dir_all(dir)?; - graph.save(graph_path(dir), None, true)?; + graph.save_with_snapshot(graph_path(dir), None)?; Ok(()) } @@ -172,7 +173,7 @@ fn json_props(props: &HashMap) -> Json { Json::Object(map) } -fn observe(graph: &SemanticGraph, encoder: &MockEncoder) -> Json { +fn observe(graph: &SemanticGraph, encoder: &MockEncoder, report: &LoadReport) -> Json { let mut node_list = Vec::new(); let mut missing = Vec::new(); let mut worst = 0.0f64; @@ -238,15 +239,21 @@ fn observe(graph: &SemanticGraph, encoder: &MockEncoder) -> Json { "missing_embeddings": missing, "max_vec_diff": worst, "searches": searches, + "source": match report.source { + LoadSource::Snapshot => "snapshot", + LoadSource::Json => "json", + LoadSource::NoEmbeddings => "none", + }, + "snapshot_skipped": report.snapshot_skipped, }) } -fn load(dir: &str) -> Result { +fn load(dir: &str) -> Result<(SemanticGraph, LoadReport), EmbeddingError> { let options = LoadOptions { on_mismatch: GraphMismatch::Error, ..LoadOptions::default() }; - SemanticGraph::load_with(graph_path(dir), Box::new(MockEncoder::new(DIM)), &options) + SemanticGraph::load_with_report(graph_path(dir), Box::new(MockEncoder::new(DIM)), &options) } fn run() -> Res { @@ -255,8 +262,8 @@ fn run() -> Res { match (arg(1), arg(2), arg(3)) { (Some("write"), Some(dir), None) => write(dir)?, (Some("read"), Some(dir), Some(out)) => { - let graph = load(dir)?; - let observation = observe(&graph, &MockEncoder::new(DIM)); + let (graph, report) = load(dir)?; + let observation = observe(&graph, &MockEncoder::new(DIM), &report); std::fs::write(out, serde_json::to_string(&observation)?)?; } (Some("mismatch"), Some(dir), None) => {