From fa5fb450e3a34621823f2367bc722806b34e1322 Mon Sep 17 00:00:00 2001 From: Brayo Date: Mon, 19 Oct 2026 14:41:37 +0300 Subject: [PATCH 1/3] feat(datastore): transparent zstd compression of event data (#493) Opt-in `compression-zstd` feature: once a database has enough events, train a zstd dictionary on them, store it in a new `compression_dict` table (db v7), and store each event's data compressed against it. Rows that would not shrink, and all rows when no dictionary exists, stay plain TEXT, so a build without the feature writes exactly what it did before. Readers accept TEXT, raw-JSON BLOB, or a compressed BLOB, so v4-v7 peer databases remain readable by read-only opens. A reader connection reloads the dictionary if the writer trains one after it opened. Rows that cannot be decompressed are treated like corrupt JSON and skipped with a warning. --- Cargo.lock | 1 + aw-datastore/Cargo.toml | 11 ++ aw-datastore/src/compression.rs | 287 ++++++++++++++++++++++++++++++++ aw-datastore/src/datastore.rs | 196 +++++++++++++++++++++- aw-datastore/src/export.rs | 24 ++- aw-datastore/src/lib.rs | 1 + aw-datastore/src/worker.rs | 3 + aw-datastore/tests/datastore.rs | 273 +++++++++++++++++++++++++++++- aw-server/Cargo.toml | 2 + 9 files changed, 780 insertions(+), 18 deletions(-) create mode 100644 aw-datastore/src/compression.rs diff --git a/Cargo.lock b/Cargo.lock index 471e6e9ea..47d96d658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,7 @@ dependencies = [ "serde_json", "tempfile", "zeroize", + "zstd", ] [[package]] diff --git a/aw-datastore/Cargo.toml b/aw-datastore/Cargo.toml index d553bae24..3fb86529b 100644 --- a/aw-datastore/Cargo.toml +++ b/aw-datastore/Cargo.toml @@ -14,6 +14,16 @@ bundled = ["rusqlite/bundled"] encryption = ["rusqlite/bundled-sqlcipher", "dep:zeroize"] # Like 'encryption' but also vendors OpenSSL (fully self-contained) encryption-vendored = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dep:zeroize"] +# Enable transparent zstd compression of event data, using a dictionary trained +# on the database's own events (~47% smaller event JSON on real data). +# Opt-in: once rows are compressed, the database can only be read by a build with +# this feature enabled. +# +# One-time cost: once the database has enough rows, the first startup trains a +# dictionary and recompresses every existing row in a single transaction. +# Benchmark on a real ~545k-event database: +# https://github.com/ActivityWatch/aw-server-rust/pull/618#issuecomment-4698391405 +compression-zstd = ["dep:zstd"] legacy_import_tests = [] [dependencies] @@ -25,6 +35,7 @@ rusqlite = { version = "0.37", features = ["chrono", "serde_json", "backup"] } mpsc_requests = "0.3" log = "0.4" zeroize = { version = "1", optional = true, features = ["alloc"] } +zstd = { version = "0.13", optional = true } aw-models = { path = "../aw-models" } aw-transform = { path = "../aw-transform" } diff --git a/aw-datastore/src/compression.rs b/aw-datastore/src/compression.rs new file mode 100644 index 000000000..7870ed308 --- /dev/null +++ b/aw-datastore/src/compression.rs @@ -0,0 +1,287 @@ +// Transparent compression of event data using zstd with a shared trained dictionary. +// +// ActivityWatch events are tiny (most < 128 bytes) and their redundancy is +// *across* rows: the same app names, JSON keys and window titles repeat +// thousands of times, while a single row holds little repetition for zstd to +// exploit on its own. To capture that cross-row redundancy we train one +// dictionary on a sample of the data, store it once in the database, and +// compress every row against it (the same idea as sqlite-zstd). On real data +// this reduces the stored event JSON by roughly 47%. +// +// Stored `data` value: +// BLOB holding a zstd frame -> dictionary-compressed +// TEXT holding the JSON -> stored uncompressed +// +// A row is stored uncompressed when compression would not make it smaller (or +// when no dictionary exists yet), so the worst case is never larger than the +// plain JSON. Uncompressed rows stay TEXT, exactly as builds without this +// feature write them, so a database that never trains a dictionary is +// byte-for-byte what it would otherwise be. Readers accept either storage +// class, and also a BLOB holding raw JSON. Zstd frames reliably start with the 4-byte magic number +// 0xFD2FB528. ActivityWatch JSON event data always starts with `{` (0x7B), so +// we can unambiguously infer whether a blob is compressed just by checking +// its first 4 bytes against the Zstd magic number. +// +// Tradeoff: once rows are compressed, the database can only be read by a build +// with this feature enabled (and with the stored dictionary intact). That is +// why the feature is opt-in at build time. + +use rusqlite::types::{Type, Value, ValueRef}; + +/// Target size of the trained dictionary. 64 KiB was the sweet spot in +/// benchmarks (larger dictionaries started to hurt the ratio). +#[cfg(feature = "compression-zstd")] +const DICT_SIZE: usize = 64 * 1024; + +/// zstd compression level. 6 is a good balance for this data: the dictionary +/// provides essentially all of the savings, and higher levels add cost for only +/// ~1% extra reduction. +#[cfg(feature = "compression-zstd")] +const COMPRESSION_LEVEL: i32 = 6; + +/// Minimum number of events before it is worth training a dictionary. Below +/// this the database is small enough that the savings are negligible, and there +/// is too little data to train a good dictionary. +#[cfg(feature = "compression-zstd")] +pub const MIN_EVENTS_TO_TRAIN: i64 = 2000; + +/// Holds the reusable zstd compressor/decompressor (with the loaded +/// dictionary, if any) for the lifetime of a datastore connection. Lives on the +/// single-threaded datastore worker, so the compressor/decompressor — which +/// need `&mut` per call — are wrapped in `RefCell` and reused across calls +/// rather than reallocated each time (allocating a fresh context per row is the +/// dominant cost when compressing hundreds of thousands of tiny events). +pub struct CompressionContext { + #[cfg(feature = "compression-zstd")] + dict: Option, +} + +#[cfg(feature = "compression-zstd")] +struct Codec { + compressor: std::cell::RefCell>, + decompressor: std::cell::RefCell>, +} + +impl CompressionContext { + /// A context with no dictionary: writes are stored uncompressed, reads still + /// transparently handle both uncompressed and (if a dictionary is later + /// available) compressed data. + pub fn empty() -> Self { + CompressionContext { + #[cfg(feature = "compression-zstd")] + dict: None, + } + } + + /// Build a context from raw trained-dictionary bytes. Falls back to an empty + /// (no-compression) context if the dictionary can't be loaded. + #[cfg(feature = "compression-zstd")] + pub fn from_dictionary(dict_bytes: &[u8]) -> Self { + // with_dictionary copies the dictionary into the (de)compression + // context, so the returned values own it and are 'static. + match ( + zstd::bulk::Compressor::with_dictionary(COMPRESSION_LEVEL, dict_bytes), + zstd::bulk::Decompressor::with_dictionary(dict_bytes), + ) { + (Ok(compressor), Ok(decompressor)) => CompressionContext { + dict: Some(Codec { + compressor: std::cell::RefCell::new(compressor), + decompressor: std::cell::RefCell::new(decompressor), + }), + }, + _ => { + warn!("Failed to load compression dictionary; storing events uncompressed"); + CompressionContext::empty() + } + } + } + + #[cfg(not(feature = "compression-zstd"))] + pub fn from_dictionary(_dict_bytes: &[u8]) -> Self { + CompressionContext {} + } + + /// Whether a dictionary is loaded (always false without the feature). + pub fn has_dictionary(&self) -> bool { + #[cfg(feature = "compression-zstd")] + { + self.dict.is_some() + } + #[cfg(not(feature = "compression-zstd"))] + { + false + } + } + + /// Encode a JSON string as the value stored in `events.data`. + /// + /// Never fails: if there is no dictionary, the feature is disabled, or + /// compression would not shrink the row, the JSON is stored as TEXT. + pub fn compress(&self, json: String) -> Value { + #[cfg(feature = "compression-zstd")] + { + if let Some(codec) = &self.dict { + if let Ok(frame) = codec.compressor.borrow_mut().compress(json.as_bytes()) { + // Return raw zstd frame without custom prefixes + if frame.len() < json.len() { + return Value::Blob(frame); + } + } + } + } + Value::Text(json) + } + + /// Read `events.data` at `idx` back into its JSON string, whether it is + /// stored as TEXT or as a (compressed or raw) BLOB. + pub fn read_data(&self, row: &rusqlite::Row, idx: usize) -> rusqlite::Result { + match row.get_ref(idx)? { + ValueRef::Text(bytes) | ValueRef::Blob(bytes) => self + .decompress(bytes) + .map_err(|e| rusqlite::Error::FromSqlConversionFailure(idx, Type::Blob, e.into())), + other => Err(rusqlite::Error::InvalidColumnType( + idx, + "data".to_string(), + other.data_type(), + )), + } + } + + /// Decompress a stored blob back into its JSON string. + pub fn decompress(&self, bytes: &[u8]) -> Result { + // Standard zstd frames always start with 0xFD2FB528 (LE bytes: 28 B5 2F FD) + if bytes.starts_with(&[0x28, 0xB5, 0x2F, 0xFD]) { + #[cfg(feature = "compression-zstd")] + { + // The zstd compressor embeds the uncompressed size by default + let orig_len = zstd::zstd_safe::get_frame_content_size(bytes) + .map_err(|e| format!("invalid zstd frame: {e}"))? + .ok_or_else(|| "zstd frame missing content size header".to_string())? + as usize; + + let codec = self + .dict + .as_ref() + .ok_or("event data is compressed but no dictionary is loaded")?; + let out = codec + .decompressor + .borrow_mut() + .decompress(bytes, orig_len) + .map_err(|e| format!("failed to decompress event data: {e}"))?; + return String::from_utf8(out) + .map_err(|e| format!("decompressed data is not valid UTF-8: {e}")); + } + #[cfg(not(feature = "compression-zstd"))] + { + return Err( + "event data is zstd-compressed but the compression-zstd feature is disabled" + .to_string(), + ); + } + } + + // Uncompressed JSON data + String::from_utf8(bytes.to_vec()).map_err(|e| format!("event data is not valid UTF-8: {e}")) + } +} + +/// Train a zstd dictionary from a set of JSON samples. Returns `None` if there +/// is not enough data to train a usable dictionary. +#[cfg(feature = "compression-zstd")] +pub fn train_dictionary(samples: &[&[u8]]) -> Option> { + if (samples.len() as i64) < MIN_EVENTS_TO_TRAIN { + return None; + } + match zstd::dict::from_samples(samples, DICT_SIZE) { + Ok(dict) => Some(dict), + Err(e) => { + warn!("Failed to train zstd dictionary: {e}"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "compression-zstd")] + fn stored_bytes(value: Value) -> Vec { + match value { + Value::Text(s) => s.into_bytes(), + Value::Blob(b) => b, + other => panic!("unexpected stored value {other:?}"), + } + } + + #[test] + fn test_uncompressed_roundtrip() { + // empty() context stores raw and reads it back + let ctx = CompressionContext::empty(); + let json = r#"{"app":"Firefox","title":"Hello"}"#; + assert_eq!( + ctx.compress(json.to_string()), + Value::Text(json.to_string()) + ); + assert_eq!(ctx.decompress(json.as_bytes()).unwrap(), json); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_dictionary_roundtrip_and_size() { + // Build a realistic corpus with heavy cross-row redundancy. + let mut samples: Vec = Vec::new(); + for i in 0..5000 { + let app = ["Firefox", "Terminal", "Code", "Slack"][i % 4]; + samples.push(format!( + r#"{{"app":"{app}","title":"Some window title number {}"}}"#, + i % 50 + )); + } + let sample_refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + let dict = train_dictionary(&sample_refs).expect("training should succeed"); + let ctx = CompressionContext::from_dictionary(&dict); + + let mut total_raw = 0usize; + let mut total_stored = 0usize; + for s in &samples { + let blob = stored_bytes(ctx.compress(s.clone())); + // roundtrip is exact + assert_eq!(ctx.decompress(&blob).unwrap(), *s); + total_raw += s.len(); + total_stored += blob.len(); + } + // dictionary compression must save substantial space on this corpus + // (without a dictionary, per-row zstd saves ~0% on data this small) + assert!( + total_stored * 10 < total_raw * 6, + "expected >40% reduction, got {total_stored} vs {total_raw}" + ); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_incompressible_row_stored_raw() { + // A context with a dictionary still stores tiny/incompressible rows raw + // when compression would not help, so a row is never larger than raw. + let samples: Vec = (0..2000) + .map(|i| format!(r#"{{"app":"A","title":"{i}"}}"#)) + .collect(); + let refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + let dict = train_dictionary(&refs).expect("training should succeed"); + let ctx = CompressionContext::from_dictionary(&dict); + + let tiny = r#"{"a":1}"#; + let blob = stored_bytes(ctx.compress(tiny.to_string())); + assert!(blob.len() <= tiny.len()); + assert_eq!(ctx.decompress(&blob).unwrap(), tiny); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_too_few_samples_no_dict() { + let samples: Vec = (0..10).map(|i| format!("{{\"x\":{i}}}")).collect(); + let refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + assert!(train_dictionary(&refs).is_none()); + } +} diff --git a/aw-datastore/src/datastore.rs b/aw-datastore/src/datastore.rs index 73035a99b..77c256bd9 100644 --- a/aw-datastore/src/datastore.rs +++ b/aw-datastore/src/datastore.rs @@ -15,6 +15,7 @@ use aw_models::Event; use rusqlite::params; use rusqlite::types::ToSql; +use super::compression; use super::DatastoreError; fn _get_db_version(conn: &Connection) -> i32 { @@ -57,7 +58,9 @@ pub(crate) fn _infer_db_version(conn: &Connection) -> i32 { if !has("table", "key_value") { return 3; } - if has("index", "events_bucketrow_endtime_starttime_index") { + if has("table", "compression_dict") { + 7 + } else if has("index", "events_bucketrow_endtime_starttime_index") { 6 } else if has("index", "events_bucketrow_starttime_endtime_index") { 5 @@ -75,14 +78,17 @@ pub(crate) fn _infer_db_version(conn: &Connection) -> i32 { * 4: Added 'key_value' table for storing key - value pairs * 5: Replaced single-column events indexes with a composite index * 6: Added an endtime-first index for recent interval reads + * 7: Added 'compression_dict' table for zstd compression of event data */ -pub const NEWEST_DB_VERSION: i32 = 6; +pub const NEWEST_DB_VERSION: i32 = 7; /// Oldest version a read-only open (aw-sync pulling a peer db) accepts. /// -/// v4, v5 and v6 have identical tables and columns; v5 and v6 only changed -/// indexes. A read-only connection cannot migrate, so queries adapt to the -/// indexes the file has (see [`events_source`]) instead of rejecting it. +/// v4 through v7 have identical `buckets` and `events` columns; v5 and v6 +/// only changed indexes, and v7 only added the `compression_dict` table. A +/// read-only connection cannot migrate, so queries adapt to the indexes the +/// file has (see [`events_source`]), and event data is decoded whether it is +/// stored as TEXT or as a compressed BLOB, instead of rejecting the file. /// /// Rule for future migrations: if a migration changes tables or columns that /// reads depend on, bump this to the new version. If it only adds or drops @@ -134,6 +140,10 @@ fn _create_tables(conn: &Connection, version: i32) -> bool { _migrate_v5_to_v6(conn); } + if version < 7 { + _migrate_v6_to_v7(conn); + } + first_init } @@ -295,6 +305,40 @@ fn _migrate_v5_to_v6(conn: &Connection) { .expect("Failed to run v6 migration transaction"); } +fn _migrate_v6_to_v7(conn: &Connection) { + // Only adds the dictionary table. `events.data` keeps its declared type: + // SQLite stores a compressed BLOB in it as-is, and uncompressed rows stay + // TEXT. Dictionary training and compression of existing rows happen at + // startup in `ensure_compression`, once the database is large enough. + conn.execute_batch( + "BEGIN EXCLUSIVE TRANSACTION; + CREATE TABLE IF NOT EXISTS compression_dict ( + id INTEGER PRIMARY KEY CHECK (id = 0), + dict BLOB NOT NULL + ); + PRAGMA user_version = 7; + COMMIT;", + ) + .expect("Failed to run v7 migration transaction"); +} + +/// Load the stored zstd compression dictionary, if one has been trained. +fn _load_dictionary(conn: &Connection) -> Option> { + conn.query_row( + "SELECT dict FROM compression_dict WHERE id = 0", + [], + |row| row.get::<_, Vec>(0), + ) + .ok() +} + +/// Count of events in the database (used to decide whether to train a dictionary). +#[cfg(feature = "compression-zstd")] +fn _event_count(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM events", [], |row| row.get(0)) + .unwrap_or(0) +} + // Both indexes can bound only one of the two interval predicates. Use the // bucket's time span as a cheap selectivity estimate; this affects performance, // never which rows qualify. Limited queries retain their ordered starttime scan @@ -350,6 +394,7 @@ pub struct DatastoreInstance { buckets_signature: (i64, i64, String), first_init: bool, pub db_version: i32, + compression: compression::CompressionContext, } /// Nanoseconds since the epoch for a time filter, clamped to `i64` for dates chrono can't @@ -405,14 +450,17 @@ fn _datetime_from_nanos(ns: i64) -> DateTime { /// /// When `clip` is set to `(starttime_filter_ns, endtime_filter_ns)`, the event is /// clamped to that query range. +/// `data` may be stored as TEXT or as a (compressed) BLOB; a row that can't be +/// decompressed is a row error, like corrupt JSON. pub(crate) fn parse_event_row( row: &rusqlite::Row, clip: Option<(i64, i64)>, + compression: &compression::CompressionContext, ) -> rusqlite::Result { let id = row.get(0)?; let mut starttime_ns: i64 = row.get(1)?; let mut endtime_ns: i64 = row.get(2)?; - let data_str: String = row.get(3)?; + let data_str = compression.read_data(row, 3)?; if let Some((starttime_filter_ns, endtime_filter_ns)) = clip { if starttime_ns < starttime_filter_ns { @@ -614,11 +662,134 @@ impl DatastoreInstance { buckets_signature: (0, 0, String::new()), first_init, db_version, + compression: compression::CompressionContext::empty(), }; + ds.ensure_compression(conn, migrate_enabled); ds.get_stored_buckets(conn)?; Ok(ds) } + /// Load the compression dictionary, training and applying one if the + /// database is large enough and doesn't have one yet. + /// + /// Runs once at startup. Loading a dictionary always happens (so compressed + /// rows can be read); training/back-filling only happens when migration is + /// enabled (we must not write to a read-only/migration-disabled datastore). + fn ensure_compression(&mut self, conn: &Connection, migrate_enabled: bool) { + if self.reload_dictionary(conn) { + return; + } + #[cfg(feature = "compression-zstd")] + { + if migrate_enabled + && self.db_version >= 7 + && _event_count(conn) >= compression::MIN_EVENTS_TO_TRAIN + { + if let Err(e) = self.train_and_backfill(conn) { + warn!("Failed to set up event-data compression: {e:?}"); + } + } + } + #[cfg(not(feature = "compression-zstd"))] + { + let _ = migrate_enabled; + } + } + + /// Load the stored dictionary if none is loaded yet. Returns whether one is + /// loaded afterwards. + /// + /// A reader connection ([`crate::DatastoreMethod::FileReader`]) can open + /// before the writer trains a dictionary, so it calls this again before + /// reads; once loaded, it is a no-op. + pub fn reload_dictionary(&mut self, conn: &Connection) -> bool { + // The compression_dict table only exists from v7 onward. + if self.compression.has_dictionary() || self.db_version < 7 { + return self.compression.has_dictionary(); + } + if let Some(dict) = _load_dictionary(conn) { + self.compression = compression::CompressionContext::from_dictionary(&dict); + return true; + } + false + } + + /// Train a dictionary from existing events, store it, and recompress all + /// rows against it. Only called once, when a database first has enough data. + #[cfg(feature = "compression-zstd")] + fn train_and_backfill(&mut self, conn: &Connection) -> Result<(), DatastoreError> { + let internal = |e: rusqlite::Error| DatastoreError::InternalError(e.to_string()); + + // Load every row's data (raw, uncompressed at this point). Rows that + // can't be decoded are left untouched rather than failing the backfill. + let mut stmt = conn + .prepare("SELECT id, data FROM events") + .map_err(internal)?; + let rows: Vec<(i64, String)> = stmt + .query_map([], |row| { + Ok((row.get(0)?, self.compression.read_data(row, 1).ok())) + }) + .map_err(internal)? + .filter_map(|row| match row { + Ok((id, Some(json))) => Some(Ok((id, json))), + Ok((_, None)) => None, + Err(e) => Some(Err(e)), + }) + .collect::, _>>() + .map_err(internal)?; + drop(stmt); + + let sample_refs: Vec<&[u8]> = rows.iter().map(|(_, json)| json.as_bytes()).collect(); + let dict = match compression::train_dictionary(&sample_refs) { + Some(dict) => dict, + None => return Ok(()), // not enough usable data; leave uncompressed + }; + + info!( + "Trained {}-byte compression dictionary, recompressing {} events", + dict.len(), + rows.len() + ); + + let codec = compression::CompressionContext::from_dictionary(&dict); + if !codec.has_dictionary() { + return Ok(()); + } + + // Store the dictionary and recompress every row in a single transaction. + // Without it, each statement would auto-commit and, with + // synchronous=FULL, fsync — turning a one-time backfill into hundreds of + // thousands of disk syncs. + conn.execute_batch("BEGIN IMMEDIATE TRANSACTION;") + .map_err(internal)?; + + let result = (|| -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO compression_dict (id, dict) VALUES (0, ?1)", + params![dict], + )?; + let mut update = conn.prepare("UPDATE events SET data = ?1 WHERE id = ?2")?; + for (id, json) in rows { + update.execute(params![codec.compress(json), id])?; + } + Ok(()) + })(); + + match result { + Ok(()) => { + conn.execute_batch("COMMIT;").map_err(internal)?; + // Only switch to the dictionary once its rows are committed, + // so reads never expect compressed rows that were rolled back. + self.compression = codec; + Ok(()) + } + Err(e) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(internal(e)) + } + } + } + /// Re-read the bucket list from the database. A reader instance /// ([`crate::DatastoreMethod::FileReader`]) keeps a cache that goes stale /// when the writer creates, deletes, renames or imports buckets. @@ -885,6 +1056,7 @@ impl DatastoreInstance { crate::export::write_export( conn, self.db_version, + &self.compression, &self.buckets_cache, bucket_id, writer, @@ -910,6 +1082,7 @@ impl DatastoreInstance { crate::export::write_events_csv( conn, self.db_version, + &self.compression, &self.buckets_cache, bucket_id, start, @@ -947,7 +1120,9 @@ impl DatastoreInstance { } }; for (event, (starttime_nanos, endtime_nanos)) in events.iter_mut().zip(nanos) { - let data = serde_json::to_string(&event.data).unwrap(); + let data = self + .compression + .compress(serde_json::to_string(&event.data).unwrap()); let res = stmt.execute([ &bucket.bid.unwrap(), &event.id as &dyn ToSql, @@ -1106,7 +1281,9 @@ impl DatastoreInstance { } }; let (starttime_nanos, endtime_nanos) = event_nanos(event)?; - let data = serde_json::to_string(&event.data).unwrap(); + let data = self + .compression + .compress(serde_json::to_string(&event.data).unwrap()); match stmt.execute([ &bucket.bid.unwrap(), &starttime_nanos, @@ -1210,7 +1387,7 @@ impl DatastoreInstance { }; let row = match stmt.query_row([&bucket.bid.unwrap(), &event_id], |row| { - parse_event_row(row, None) + parse_event_row(row, None, &self.compression) }) { Ok(rows) => rows, Err(rusqlite::Error::QueryReturnedNoRows) => { @@ -1306,6 +1483,7 @@ impl DatastoreInstance { match parse_event_row( row, clip_to_query_range.then_some((starttime_filter_ns, endtime_filter_ns)), + &self.compression, ) { Ok(event) => list.push(event), Err(err) => { diff --git a/aw-datastore/src/export.rs b/aw-datastore/src/export.rs index 6782be3a3..0eef0b310 100644 --- a/aw-datastore/src/export.rs +++ b/aw-datastore/src/export.rs @@ -9,12 +9,14 @@ use serde::{ Serialize, Serializer, }; +use crate::compression::CompressionContext; use crate::datastore::{events_source, parse_event_row, prefer_endtime_index}; use crate::DatastoreError; struct EventRows<'a> { conn: &'a Connection, db_version: i32, + compression: &'a CompressionContext, bucket: &'a Bucket, } @@ -39,7 +41,7 @@ impl Serialize for EventRows<'_> { while let Some(row) = rows.next().map_err(S::Error::custom)? { // Match get_events' default range, clipping and corrupt-row policy. // i64::MIN so None starttime includes pre-epoch events, like get_events. - match crate::datastore::parse_event_row(row, Some((i64::MIN, i64::MAX))) { + match parse_event_row(row, Some((i64::MIN, i64::MAX)), self.compression) { Ok(event) => seq.serialize_element(&event)?, Err(err) => warn!("Corrupt event in bucket {}: {}", self.bucket.id, err), } @@ -51,6 +53,7 @@ impl Serialize for EventRows<'_> { struct ExportBucket<'a> { conn: &'a Connection, db_version: i32, + compression: &'a CompressionContext, bucket: &'a Bucket, } @@ -73,6 +76,7 @@ impl Serialize for ExportBucket<'_> { &EventRows { conn: self.conn, db_version: self.db_version, + compression: self.compression, bucket: self.bucket, }, )?; @@ -83,6 +87,7 @@ impl Serialize for ExportBucket<'_> { struct ExportBuckets<'a> { conn: &'a Connection, db_version: i32, + compression: &'a CompressionContext, buckets: &'a HashMap, selected: Option<&'a str>, } @@ -97,6 +102,7 @@ impl Serialize for ExportBuckets<'_> { &ExportBucket { conn: self.conn, db_version: self.db_version, + compression: self.compression, bucket, }, )?; @@ -109,6 +115,7 @@ impl Serialize for ExportBuckets<'_> { pub(crate) fn write_export( conn: &Connection, db_version: i32, + compression: &CompressionContext, buckets: &HashMap, selected: Option<&str>, writer: impl Write, @@ -128,6 +135,7 @@ pub(crate) fn write_export( buckets: ExportBuckets { conn, db_version, + compression, buckets, selected, }, @@ -307,6 +315,7 @@ fn write_csv_event( pub(crate) fn write_events_csv( conn: &Connection, db_version: i32, + compression: &CompressionContext, buckets: &HashMap, bucket_id: &str, starttime_opt: Option>, @@ -370,12 +379,11 @@ pub(crate) fn write_events_csv( while let Some(row) = key_rows.next().map_err(|err| { DatastoreError::InternalError(format!("Failed to read CSV export key row: {err}")) })? { - let data: Option = row.get(0).map_err(|err| { - DatastoreError::InternalError(format!("Failed to read CSV export key data: {err}")) - })?; - let data = match data { - Some(d) => d, - None => continue, + // A row whose data can't be decoded is skipped here, and again (with a + // warning) by the row pass below. + let data = match compression.read_data(row, 0) { + Ok(d) => d, + Err(_) => continue, }; if let Ok(value) = serde_json::from_str::(&data) { if let Some(obj) = value.as_object() { @@ -417,7 +425,7 @@ pub(crate) fn write_events_csv( while let Some(row) = rows.next().map_err(|err| { DatastoreError::InternalError(format!("Failed to read CSV export row: {err}")) })? { - let event = match parse_event_row(row, clip) { + let event = match parse_event_row(row, clip, compression) { Ok(event) => event, Err(err) => { warn!("Corrupt event in bucket {}: {}", bucket_id, err); diff --git a/aw-datastore/src/lib.rs b/aw-datastore/src/lib.rs index a7dbddc43..d7ae01fcc 100644 --- a/aw-datastore/src/lib.rs +++ b/aw-datastore/src/lib.rs @@ -17,6 +17,7 @@ macro_rules! json_map { }}; } +mod compression; mod datastore; mod export; mod legacy_import; diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index 7c84024f9..77b6d2d7b 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -627,6 +627,9 @@ impl DatastoreWorker { warn!("Reader failed to reload bucket list: {e:?}"); } } + // The writer may have trained a compression dictionary since this + // reader opened. One-row lookup until a dictionary is loaded. + ds.reload_dictionary(tx); } match request { Command::ReloadBuckets() => match ds.reload_buckets(tx) { diff --git a/aw-datastore/tests/datastore.rs b/aw-datastore/tests/datastore.rs index b5ade73fc..44bcbb1e2 100644 --- a/aw-datastore/tests/datastore.rs +++ b/aw-datastore/tests/datastore.rs @@ -1214,7 +1214,7 @@ mod datastore_tests { let version: i32 = conn .pragma_query_value(None, "user_version", |row| row.get(0)) .unwrap(); - assert_eq!(version, 6); + assert_eq!(version, aw_datastore::NEWEST_DB_VERSION); let old_indexes: i64 = conn .query_row( "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN @@ -2007,4 +2007,275 @@ mod datastore_tests { assert!(!shm.exists(), "version probe must not create a SHM sidecar"); } } + + /// With the compression feature, a database that grows past the training + /// threshold should get a dictionary on the next open, recompress its rows, + /// and still return every event's data intact. + #[test] + #[cfg(feature = "compression-zstd")] + fn test_compression_dictionary_roundtrip() { + use rusqlite::Connection; + + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir.path().join("datastore-unittest-compression.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + let bucket = test_bucket(); + // Realistic, highly-repetitive event data, well above the training threshold. + let n = 4000; + let make_event = |i: usize| Event { + id: None, + timestamp: Utc::now() + Duration::milliseconds(i as i64), + duration: Duration::seconds(1), + data: json_map! { + "app": json!(["Firefox", "Terminal", "Code", "Slack"][i % 4]), + "title": json!(format!("Working on window {}", i % 100)) + }, + }; + + // Session 1: no dictionary exists yet (DB started empty), rows stored raw. + { + let ds = Datastore::new(db_path_str.clone(), false); + ds.create_bucket(&bucket).unwrap(); + let events: Vec = (0..n).map(make_event).collect(); + ds.insert_events(&bucket.id, &events).unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + + // No dictionary should have been trained during session 1, and rows + // are stored as plain TEXT, exactly as without the feature. + { + let conn = Connection::open(&db_path).unwrap(); + let dict_rows: i64 = conn + .query_row("SELECT count(*) FROM compression_dict", [], |r| r.get(0)) + .unwrap(); + assert_eq!(dict_rows, 0, "no dictionary expected before reopen"); + let non_text: i64 = conn + .query_row( + "SELECT count(*) FROM events WHERE typeof(data) != 'text'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(non_text, 0, "uncompressed rows must stay TEXT"); + } + + // Session 2: reopening trains a dictionary and recompresses existing rows. + { + // A reader opened before the writer trains must pick up the + // dictionary once it exists, not fail on the compressed rows. + let reader = Datastore::open_reader(db_path_str.clone()).unwrap(); + assert_eq!( + reader + .get_events(&bucket.id, None, None, None) + .unwrap() + .len(), + n + ); + let ds = Datastore::new(db_path_str.clone(), false); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(events.len(), n); + assert_eq!( + reader + .get_events(&bucket.id, None, None, None) + .unwrap() + .len(), + n + ); + reader.close(); + // Data must survive the train + recompress roundtrip exactly. + for e in &events { + let i = e.data["title"] + .as_str() + .unwrap() + .strip_prefix("Working on window ") + .unwrap() + .parse::() + .unwrap(); + assert!(i < 100); + assert!(e.data["app"].is_string()); + } + // Inserting more after the dictionary exists must also roundtrip. + ds.insert_events(&bucket.id, &[make_event(99999)]).unwrap(); + ds.force_commit().unwrap(); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(events.len(), n + 1); + assert_eq!(events[0].data["title"], json!("Working on window 99")); + ds.close(); + } + + // A dictionary now exists and at least some rows are stored compressed. + { + let conn = Connection::open(&db_path).unwrap(); + let dict_rows: i64 = conn + .query_row("SELECT count(*) FROM compression_dict", [], |r| r.get(0)) + .unwrap(); + assert_eq!(dict_rows, 1, "dictionary should exist after reopen"); + // 0x28 0xB5 0x2F 0xFD is the zstd magic number (Little Endian). + let compressed: i64 = conn + .query_row( + "SELECT count(*) FROM events WHERE hex(substr(data, 1, 4)) = '28B52FFD'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(compressed > 0, "expected some rows to be stored compressed"); + } + + // A read-only open (aw-sync reading a peer db) decodes compressed rows. + { + let ds = Datastore::open_read_only(db_path_str).unwrap(); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(events.len(), n + 1); + ds.close(); + } + } + + /// Upgrading a populated pre-v7 database should train and apply compression + /// on the very first open (not only after a second restart). Regression test + /// for the db_version being captured before migrations ran. + #[test] + #[cfg(feature = "compression-zstd")] + fn test_migration_to_v7_trains_on_first_open() { + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir.path().join("datastore-unittest-migration-v7.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + // Build a v5 database with enough repetitive events to train a dictionary. + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE buckets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, type TEXT NOT NULL, client TEXT NOT NULL, + hostname TEXT NOT NULL, created TEXT NOT NULL, + data TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX bucket_id_index ON buckets(id); + CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, starttime INTEGER NOT NULL, + endtime INTEGER NOT NULL, data TEXT NOT NULL, + FOREIGN KEY (bucketrow) REFERENCES buckets(id) + ); + CREATE INDEX events_bucketrow_starttime_endtime_index + ON events(bucketrow, starttime DESC, endtime); + CREATE TABLE key_value (key TEXT PRIMARY KEY, value TEXT, last_modified NUMBER NOT NULL); + INSERT INTO buckets (name, type, client, hostname, created, data) + VALUES ('testid', 'testtype', 'testclient', 'testhost', + '2024-01-01T00:00:00+00:00', '{}'); + WITH RECURSIVE seq(x) AS ( + SELECT 1 UNION ALL SELECT x + 1 FROM seq WHERE x < 3000 + ) + INSERT INTO events (bucketrow, starttime, endtime, data) + SELECT 1, x * 1000000000, x * 1000000000 + 1000000000, + '{"app":"App' || (x % 4) || '","title":"Window title ' || (x % 50) || '"}' + FROM seq; + PRAGMA user_version = 5; + "#, + ) + .unwrap(); + } + + // First open migrates to v7 AND sets up compression in the same session. + { + let ds = Datastore::new(db_path_str, false); + let events = ds.get_events("testid", None, None, None).unwrap(); + assert_eq!(events.len(), 3000); + // newest-first ordering: last inserted was x=3000 -> App0 + assert_eq!(events[0].data["app"], json!("App0")); + ds.close(); + } + + // A dictionary was trained and some rows compressed on that first open. + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let version: i32 = conn + .pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap(); + assert_eq!(version, aw_datastore::NEWEST_DB_VERSION); + let dict_rows: i64 = conn + .query_row("SELECT count(*) FROM compression_dict", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + dict_rows, 1, + "dictionary should be trained on the first open after upgrade" + ); + let compressed: i64 = conn + .query_row( + "SELECT count(*) FROM events WHERE hex(substr(data, 1, 4)) = '28B52FFD'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + compressed > 0, + "expected rows to be compressed after first open" + ); + } + } + + /// A row that looks compressed (zstd magic) but can't be decompressed — e.g. + /// the dictionary is missing or the feature is disabled — must be skipped + /// with a warning, never panic the worker. + #[test] + fn test_unreadable_compressed_row_does_not_panic() { + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir.path().join("datastore-unittest-badrow.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + // Create a database with one valid event. + { + let ds = Datastore::new(db_path_str.clone(), false); + ds.create_bucket(&test_bucket()).unwrap(); + ds.insert_events( + "testid", + &[Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(1), + data: json_map! {"app": json!("ok")}, + }], + ) + .unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + + // Inject a row whose data starts with the zstd magic number but is not a + // valid frame, and for which no dictionary exists. + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let bid: i64 = conn + .query_row("SELECT id FROM buckets WHERE name = 'testid'", [], |r| { + r.get(0) + }) + .unwrap(); + conn.execute( + "INSERT INTO events (bucketrow, starttime, endtime, data) VALUES (?1, ?2, ?3, ?4)", + rusqlite::params![ + bid, + 5_000_000_000i64, + 6_000_000_000i64, + vec![0x28u8, 0xB5, 0x2F, 0xFD, 0xDE, 0xAD] + ], + ) + .unwrap(); + } + + // Reading must not panic: the bad row is skipped, the good row returned. + { + let ds = Datastore::new(db_path_str, false); + let events = ds.get_events("testid", None, None, None).unwrap(); + assert_eq!( + events.len(), + 1, + "the unreadable row should be skipped, the valid one kept" + ); + assert_eq!(events[0].data["app"], json!("ok")); + ds.close(); + } + } } diff --git a/aw-server/Cargo.toml b/aw-server/Cargo.toml index 81e0c4f78..d3a7110f8 100644 --- a/aw-server/Cargo.toml +++ b/aw-server/Cargo.toml @@ -22,6 +22,8 @@ bundled = ["aw-datastore/bundled"] encryption = ["aw-datastore/encryption"] # Enable SQLCipher encryption with vendored OpenSSL (fully self-contained) encryption-vendored = ["aw-datastore/encryption-vendored"] +# Enable zstd compression for events data +compression-zstd = ["aw-datastore/compression-zstd"] [build-dependencies] embed-resource = "3" From 3fa851e8d14a0691091a3f26b6789946ed6ff138 Mon Sep 17 00:00:00 2001 From: Brayo Date: Mon, 19 Oct 2026 14:50:11 +0300 Subject: [PATCH 2/3] fix(datastore): take the write lock before reading rows to backfill --- aw-datastore/src/datastore.rs | 84 +++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/aw-datastore/src/datastore.rs b/aw-datastore/src/datastore.rs index 77c256bd9..ff6886974 100644 --- a/aw-datastore/src/datastore.rs +++ b/aw-datastore/src/datastore.rs @@ -720,50 +720,46 @@ impl DatastoreInstance { fn train_and_backfill(&mut self, conn: &Connection) -> Result<(), DatastoreError> { let internal = |e: rusqlite::Error| DatastoreError::InternalError(e.to_string()); - // Load every row's data (raw, uncompressed at this point). Rows that - // can't be decoded are left untouched rather than failing the backfill. - let mut stmt = conn - .prepare("SELECT id, data FROM events") - .map_err(internal)?; - let rows: Vec<(i64, String)> = stmt - .query_map([], |row| { - Ok((row.get(0)?, self.compression.read_data(row, 1).ok())) - }) - .map_err(internal)? - .filter_map(|row| match row { - Ok((id, Some(json))) => Some(Ok((id, json))), - Ok((_, None)) => None, - Err(e) => Some(Err(e)), - }) - .collect::, _>>() + // Take the write lock before reading: another writer on the same file + // (an older server during history recovery) could otherwise commit an + // event change between the read and the rewrite, which the backfill + // would then overwrite with stale data. One transaction also keeps the + // backfill from fsyncing once per row under synchronous=FULL. + conn.execute_batch("BEGIN IMMEDIATE TRANSACTION;") .map_err(internal)?; - drop(stmt); - - let sample_refs: Vec<&[u8]> = rows.iter().map(|(_, json)| json.as_bytes()).collect(); - let dict = match compression::train_dictionary(&sample_refs) { - Some(dict) => dict, - None => return Ok(()), // not enough usable data; leave uncompressed - }; - info!( - "Trained {}-byte compression dictionary, recompressing {} events", - dict.len(), - rows.len() - ); - - let codec = compression::CompressionContext::from_dictionary(&dict); - if !codec.has_dictionary() { - return Ok(()); - } + let result = (|| -> Result, rusqlite::Error> { + // Load every row's data (raw, uncompressed at this point). Rows that + // can't be decoded are left untouched rather than failing the backfill. + let mut stmt = conn.prepare("SELECT id, data FROM events")?; + let rows: Vec<(i64, String)> = stmt + .query_map([], |row| { + Ok((row.get(0)?, self.compression.read_data(row, 1).ok())) + })? + .filter_map(|row| match row { + Ok((id, Some(json))) => Some(Ok((id, json))), + Ok((_, None)) => None, + Err(e) => Some(Err(e)), + }) + .collect::, _>>()?; + drop(stmt); + + let sample_refs: Vec<&[u8]> = rows.iter().map(|(_, json)| json.as_bytes()).collect(); + let dict = match compression::train_dictionary(&sample_refs) { + Some(dict) => dict, + None => return Ok(None), // not enough usable data; leave uncompressed + }; + let codec = compression::CompressionContext::from_dictionary(&dict); + if !codec.has_dictionary() { + return Ok(None); + } - // Store the dictionary and recompress every row in a single transaction. - // Without it, each statement would auto-commit and, with - // synchronous=FULL, fsync — turning a one-time backfill into hundreds of - // thousands of disk syncs. - conn.execute_batch("BEGIN IMMEDIATE TRANSACTION;") - .map_err(internal)?; + info!( + "Trained {}-byte compression dictionary, recompressing {} events", + dict.len(), + rows.len() + ); - let result = (|| -> Result<(), rusqlite::Error> { conn.execute( "INSERT OR REPLACE INTO compression_dict (id, dict) VALUES (0, ?1)", params![dict], @@ -772,17 +768,21 @@ impl DatastoreInstance { for (id, json) in rows { update.execute(params![codec.compress(json), id])?; } - Ok(()) + Ok(Some(codec)) })(); match result { - Ok(()) => { + Ok(Some(codec)) => { conn.execute_batch("COMMIT;").map_err(internal)?; // Only switch to the dictionary once its rows are committed, // so reads never expect compressed rows that were rolled back. self.compression = codec; Ok(()) } + Ok(None) => { + let _ = conn.execute_batch("ROLLBACK;"); + Ok(()) + } Err(e) => { let _ = conn.execute_batch("ROLLBACK;"); Err(internal(e)) From 7854bb5e317d11c80281ea7721f0d014940ad067 Mon Sep 17 00:00:00 2001 From: Brayo Date: Wed, 18 Nov 2026 11:54:04 +0300 Subject: [PATCH 3/3] build(datastore): use zstd 0.14 to match aw-sync Master moved aw-sync to zstd 0.14; keeping aw-datastore on 0.13 would pull in a second copy of zstd and left the merged Cargo.lock out of date. --- aw-datastore/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aw-datastore/Cargo.toml b/aw-datastore/Cargo.toml index 3fb86529b..bca5c017e 100644 --- a/aw-datastore/Cargo.toml +++ b/aw-datastore/Cargo.toml @@ -35,7 +35,7 @@ rusqlite = { version = "0.37", features = ["chrono", "serde_json", "backup"] } mpsc_requests = "0.3" log = "0.4" zeroize = { version = "1", optional = true, features = ["alloc"] } -zstd = { version = "0.13", optional = true } +zstd = { version = "0.14", optional = true } aw-models = { path = "../aw-models" } aw-transform = { path = "../aw-transform" }