diff --git a/Cargo.toml b/Cargo.toml index 8b7ef64..1935f4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "entropy-map" -version = "1.1.0" +version = "1.2.0" edition = "2021" authors = [ "Alex Bocharov ", @@ -20,6 +20,8 @@ bytecheck = { version = "~0.6.8", default-features = false, optional = true } num = "0.4.1" rkyv = { version = "0.7.42", features = ["validation", "strict"], optional = true } wyhash = "0.5.0" +serde = { version = "1", features = ["derive"], optional = true } +serde_bytes = { version = "0.11", optional = true } [dev-dependencies] bitvec = "1.0.1" @@ -30,10 +32,12 @@ rand = "0.8.5" rand_chacha = "0.3.1" rkyv = { version = "0.7.42", features = ["validation", "strict"] } test-case = "3.3.1" +rmp-serde = "1.3" [features] default = [] rkyv_derive = ["rkyv", "bytecheck"] +serde = ["dep:serde", "dep:serde_bytes"] [[bench]] name = "rank" diff --git a/README.md b/README.md index aa9a0dd..d4038e0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ It includes the following key components: - `H`: hasher used to hash keys, default `WyHash`. - Configurable `gamma` parameter to tune construction time vs query time trade-off. - Optional [rkyv](https://rkyv.org/) support to enable zero-copy serialization/deserialization of MPHF. +- Optional `serde` support ### MapWithDict - Immutable hash map leveraging MPHF for indexing. @@ -46,14 +47,17 @@ It includes the following key components: - Optimized for space, using a dictionary to pack unique values. - Efficient storage and retrieval, reducing overall memory footprint. - Optional [rkyv](https://rkyv.org/) support to enable zero-copy serialization/deserialization and superior memory footprint and performance when compared with `rkyv::ArchivedHashMap`. +- Optional `serde` support ### MapWithDictBitpacked - Specialized version of `MapWithDict`, further optimized for memory usage when values are `Vec`. - Bit-packs `Vec` values for minimal space usage using SIMD instructions. - Excels in scenarios where values are within a limited range and can be efficiently encoded. +- Optional `serde` support ### Set Special case of `MapWithDict`, optimized for set membership operations. - Immutable set using MPHF for indexing. - Stores keys to ensure presence/absence of the key in the set. - Optional rkyv support to enable zero-copy serialization/deserialization. +- Optional `serde` support diff --git a/src/map_with_dict.rs b/src/map_with_dict.rs index 9e51d6a..2b008fc 100644 --- a/src/map_with_dict.rs +++ b/src/map_with_dict.rs @@ -21,6 +21,14 @@ use crate::mphf::{Mphf, MphfError, DEFAULT_GAMMA}; #[derive(Default)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(bound( + serialize = "K: serde::Serialize, V: serde::Serialize, ST: serde::Serialize", + deserialize = "K: serde::Deserialize<'de>, V: serde::Deserialize<'de>, ST: serde::Deserialize<'de>", + )) +)] pub struct MapWithDict where ST: PrimInt + Unsigned, @@ -464,6 +472,45 @@ mod tests { assert!(!rkyv_map.contains_key("c")); } + #[cfg(feature = "serde")] + #[test] + fn test_serde() { + // create regular `HashMap`, then `MapWithDict`, then serialize to msgpack bytes. + let original_map = gen_map(1000); + let map = MapWithDict::try_from(original_map.clone()).unwrap(); + + let bytes = rmp_serde::to_vec(&map).unwrap(); + let de: MapWithDict = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(de.len(), original_map.len()); + + // Test get on the deserialized `MapWithDict` + for (k, v) in original_map.iter() { + assert_eq!(de.get(k), Some(v)); + } + + // Test iter on the deserialized `MapWithDict` + for (&k, &v) in de.iter() { + assert_eq!(original_map.get(&k), Some(&v)); + } + } + + #[cfg(feature = "serde")] + #[test] + fn test_serde_get_borrow() { + let original_map = HashMap::from_iter([("a".to_string(), ()), ("b".to_string(), ())]); + let map = MapWithDict::try_from(original_map).unwrap(); + let bytes = rmp_serde::to_vec(&map).unwrap(); + let de: MapWithDict = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(de.get("a"), Some(&())); + assert!(de.contains_key("a")); + assert_eq!(de.get("b"), Some(&())); + assert!(de.contains_key("b")); + assert_eq!(de.get("c"), None); + assert!(!de.contains_key("c")); + } + macro_rules! proptest_map_with_dict_model { ($(($b:expr, $s:expr, $gamma:expr)),* $(,)?) => { $( diff --git a/src/map_with_dict_bitpacked.rs b/src/map_with_dict_bitpacked.rs index 2389ee5..d2d7450 100644 --- a/src/map_with_dict_bitpacked.rs +++ b/src/map_with_dict_bitpacked.rs @@ -26,6 +26,14 @@ use crate::mphf::{Mphf, DEFAULT_GAMMA}; #[derive(Default)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(bound( + serialize = "K: serde::Serialize, ST: serde::Serialize", + deserialize = "K: serde::Deserialize<'de>, ST: serde::Deserialize<'de>", + )) +)] pub struct MapWithDictBitpacked where ST: PrimInt + Unsigned, @@ -38,6 +46,7 @@ where /// Points to the value index in the dictionary values_index: Box<[usize]>, /// Bit-packed dictionary containing values + #[cfg_attr(feature = "serde", serde(with = "serde_bytes"))] values_dict: Box<[u8]>, } @@ -556,6 +565,28 @@ mod tests { } } + #[cfg(feature = "serde")] + #[test] + fn test_serde() { + // create regular `HashMap`, then `MapWithDictBitpacked`, then serialize to msgpack bytes. + let items_num = 1000; + let values_num = 10; + let original_map = gen_map(items_num, values_num); + let map = MapWithDictBitpacked::try_from(original_map.clone()).unwrap(); + + let bytes = rmp_serde::to_vec(&map).unwrap(); + let de: MapWithDictBitpacked = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(de.len(), original_map.len()); + + // Test get_values on the deserialized `MapWithDictBitpacked` + let mut values_buf = vec![0; values_num]; + for (k, v) in &original_map { + assert!(de.get_values(k, &mut values_buf)); + assert_eq!(v, &values_buf); + } + } + macro_rules! proptest_map_with_dict_bitpacked_model { ($(($b:expr, $s:expr, $gamma:expr, $n:expr)),* $(,)?) => { $( diff --git a/src/mphf.rs b/src/mphf.rs index a7bb5e2..cb956e3 100644 --- a/src/mphf.rs +++ b/src/mphf.rs @@ -27,6 +27,11 @@ use crate::rank::{RankedBits, RankedBitsAccess}; #[derive(Default)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(bound(serialize = "ST: serde::Serialize", deserialize = "ST: serde::Deserialize<'de>")) +)] pub struct Mphf { /// Ranked bits for efficient rank queries ranked_bits: RankedBits, @@ -432,4 +437,30 @@ mod tests { } assert_eq!(set.len(), n); } + + #[cfg(feature = "serde")] + #[test] + fn test_serde() { + let n = 10000; + let keys = (0..n as u64).collect::>(); + let mphf = Mphf::<32, 4>::from_slice(&keys, DEFAULT_GAMMA).expect("failed to create mphf"); + + // Serialize via msgpack and deserialize back. The PHF must be fully + // restored without recomputation. + let bytes = rmp_serde::to_vec(&mphf).unwrap(); + let de: Mphf<32, 4> = rmp_serde::from_slice(&bytes).unwrap(); + + // Ensure that all keys are assigned the same unique index by both + // the original and the deserialized MPHFs. + let mut set = HashSet::with_capacity(n); + for key in &keys { + let idx = mphf.get(key).unwrap(); + let de_idx = de.get(key).unwrap(); + + assert_eq!(idx, de_idx); + assert!(idx < n, "idx = {} n = {}", idx, n); + assert!(set.insert(idx), "duplicate idx = {} for key {}", idx, key); + } + assert_eq!(set.len(), n); + } } diff --git a/src/rank.rs b/src/rank.rs index dee7a9b..635cd5e 100644 --- a/src/rank.rs +++ b/src/rank.rs @@ -10,6 +10,46 @@ const L2_BIT_SIZE: usize = 512; /// Size of the L1 block in bits, calculated as a multiple of the L2 block size. const L1_BIT_SIZE: usize = 8 * L2_BIT_SIZE; +/// Serde helper that stores `Box<[u64]>` as a single little-endian byte blob. +/// +/// Unlike the default `seq` encoding (which in msgpack frames each `u64` as a +/// `uint64` tag + 8 bytes = 9 bytes/element), this emits one `bin` blob via +/// `serialize_bytes`, eliminating per-element framing. Endianness is explicit +/// (little-endian) so the on-wire form is portable across architectures, +/// consistent with the crate's existing `L12Rank` LE convention. +#[cfg(feature = "serde")] +mod u64_blob { + use serde::{Deserialize, Deserializer, Serializer}; + use serde_bytes::ByteBuf; + use std::boxed::Box; + + pub fn serialize(bits: &[u64], serializer: S) -> Result + where + S: Serializer, + { + let mut out = Vec::with_capacity(bits.len() * 8); + for &word in bits.iter() { + out.extend_from_slice(&word.to_le_bytes()); + } + serializer.serialize_bytes(&out) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let buf = ByteBuf::deserialize(deserializer)?; + if buf.len() % 8 != 0 { + return Err(serde::de::Error::invalid_length( + buf.len(), + &"a byte length that is a multiple of 8", + )); + } + let words: Vec = buf.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c)).collect(); + Ok(words.into_boxed_slice()) + } +} + /// Trait for efficient bit-level operations on ranked bit sequences. /// /// This trait is designed to provide consistent methods for accessing ranked bit sequences in both @@ -57,8 +97,10 @@ pub trait RankedBitsAccess { #[derive(Debug, Default)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct RankedBits { /// The bit vector represented as an array of u64 integers. + #[cfg_attr(feature = "serde", serde(with = "u64_blob"))] bits: Box<[u64]>, /// Precomputed rank information for L1 and L2 blocks. l12_ranks: Box<[L12Rank]>, @@ -71,7 +113,8 @@ pub struct RankedBits { #[derive(Debug)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] -pub struct L12Rank([u8; 16]); +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct L12Rank(#[cfg_attr(feature = "serde", serde(with = "serde_bytes"))] [u8; 16]); /// Trait used to access archived and non-archived L1 and L2 ranks pub trait L12RankAccess { @@ -205,4 +248,21 @@ mod tests { } } } + + #[cfg(feature = "serde")] + #[test] + fn test_serde() { + let rng = rand::thread_rng(); + let bits: Vec = rng.sample_iter(Standard).take(1001).collect(); + let ranked_bits = RankedBits::new(bits.clone().into_boxed_slice()); + + let bytes = rmp_serde::to_vec(&ranked_bits).unwrap(); + let de: RankedBits = rmp_serde::from_slice(&bytes).unwrap(); + + // The deserialized `RankedBits` must answer every `rank` query identically + // to the original (the whole point of persisting the PHF artifacts). + for idx in 0..bits.len() * 64 { + assert_eq!(ranked_bits.rank(idx), de.rank(idx), "rank mismatch at {}", idx); + } + } } diff --git a/src/set.rs b/src/set.rs index a3eb148..f178977 100644 --- a/src/set.rs +++ b/src/set.rs @@ -24,6 +24,14 @@ use crate::mphf::{Mphf, MphfError, DEFAULT_GAMMA}; #[derive(Default)] #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))] #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(bound( + serialize = "K: serde::Serialize, ST: serde::Serialize", + deserialize = "K: serde::Deserialize<'de>, ST: serde::Deserialize<'de>", + )) +)] pub struct Set where ST: PrimInt + Unsigned, @@ -286,6 +294,24 @@ mod tests { assert!(!rkyv_set.contains("c")); } + #[cfg(feature = "serde")] + #[test] + fn test_serde() { + // create regular `HashSet`, then `Set`, then serialize to msgpack bytes. + let original_set = gen_set(1000); + let set = Set::try_from(original_set.clone()).unwrap(); + + let bytes = rmp_serde::to_vec(&set).unwrap(); + let de: Set = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(de.len(), original_set.len()); + + // Test contains on the deserialized `Set` + for k in original_set.iter() { + assert!(de.contains(k)); + } + } + macro_rules! proptest_set_model { ($(($b:expr, $s:expr, $gamma:expr)),* $(,)?) => { $(