TIP-143: Add serde feature

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "entropy-map"
-version = "1.1.0"
+version = "1.2.0"
 edition = "2021"
 authors = [
     "Alex Bocharov <bocharov.alexandr@gmail.com>",
@@ -20,6 +20,8 @@ bytecheck = { version = "~0.6.8", default-features = false, optional = true }
 num = "0.4.1"
 rkyv = { version = "0.7.42", features = ["validation", "strict"], optional = true }
 wyhash = "0.5.0"
+serde = { version = "1", features = ["derive"], optional = true }
+serde_bytes = { version = "0.11", optional = true }
 
 [dev-dependencies]
 bitvec = "1.0.1"
@@ -30,10 +32,12 @@ rand = "0.8.5"
 rand_chacha = "0.3.1"
 rkyv = { version = "0.7.42", features = ["validation", "strict"] }
 test-case = "3.3.1"
+rmp-serde = "1.3"
 
 [features]
 default = []
 rkyv_derive = ["rkyv", "bytecheck"]
+serde = ["dep:serde", "dep:serde_bytes"]
 
 [[bench]]
 name = "rank"

README.md

@@ -39,21 +39,25 @@ It includes the following key components:
   - `H`: hasher used to hash keys, default `WyHash`.
 - Configurable `gamma` parameter to tune construction time vs query time trade-off.
 - Optional [rkyv](https://rkyv.org/) support to enable zero-copy serialization/deserialization of MPHF.
+- Optional `serde` support
 
 ### MapWithDict
 - Immutable hash map leveraging MPHF for indexing.
 - Stores keys to ensure presence/absence of the key in the map.
 - Optimized for space, using a dictionary to pack unique values.
 - Efficient storage and retrieval, reducing overall memory footprint.
 - Optional [rkyv](https://rkyv.org/) support to enable zero-copy serialization/deserialization and superior memory footprint and performance when compared with `rkyv::ArchivedHashMap`.
+- Optional `serde` support
 
 ### MapWithDictBitpacked
 - Specialized version of `MapWithDict`, further optimized for memory usage when values are `Vec<u32>`.
 - Bit-packs `Vec<u32>` values for minimal space usage using SIMD instructions.
 - Excels in scenarios where values are within a limited range and can be efficiently encoded.
+- Optional `serde` support
 
 ### Set
 Special case of `MapWithDict`, optimized for set membership operations.
 - Immutable set using MPHF for indexing.
 - Stores keys to ensure presence/absence of the key in the set.
 - Optional rkyv support to enable zero-copy serialization/deserialization.
+- Optional `serde` support

src/map_with_dict.rs

@@ -21,6 +21,14 @@ use crate::mphf::{Mphf, MphfError, DEFAULT_GAMMA};
 #[derive(Default)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    feature = "serde",
+    serde(bound(
+        serialize = "K: serde::Serialize, V: serde::Serialize, ST: serde::Serialize",
+        deserialize = "K: serde::Deserialize<'de>, V: serde::Deserialize<'de>, ST: serde::Deserialize<'de>",
+    ))
+)]
 pub struct MapWithDict<K, V, const B: usize = 32, const S: usize = 8, ST = u8, H = WyHash>
 where
     ST: PrimInt + Unsigned,
@@ -464,6 +472,45 @@ mod tests {
         assert!(!rkyv_map.contains_key("c"));
     }
 
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        // create regular `HashMap`, then `MapWithDict`, then serialize to msgpack bytes.
+        let original_map = gen_map(1000);
+        let map = MapWithDict::try_from(original_map.clone()).unwrap();
+
+        let bytes = rmp_serde::to_vec(&map).unwrap();
+        let de: MapWithDict<u64, u32> = rmp_serde::from_slice(&bytes).unwrap();
+
+        assert_eq!(de.len(), original_map.len());
+
+        // Test get on the deserialized `MapWithDict`
+        for (k, v) in original_map.iter() {
+            assert_eq!(de.get(k), Some(v));
+        }
+
+        // Test iter on the deserialized `MapWithDict`
+        for (&k, &v) in de.iter() {
+            assert_eq!(original_map.get(&k), Some(&v));
+        }
+    }
+
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde_get_borrow() {
+        let original_map = HashMap::from_iter([("a".to_string(), ()), ("b".to_string(), ())]);
+        let map = MapWithDict::try_from(original_map).unwrap();
+        let bytes = rmp_serde::to_vec(&map).unwrap();
+        let de: MapWithDict<String, ()> = rmp_serde::from_slice(&bytes).unwrap();
+
+        assert_eq!(de.get("a"), Some(&()));
+        assert!(de.contains_key("a"));
+        assert_eq!(de.get("b"), Some(&()));
+        assert!(de.contains_key("b"));
+        assert_eq!(de.get("c"), None);
+        assert!(!de.contains_key("c"));
+    }
+
     macro_rules! proptest_map_with_dict_model {
         ($(($b:expr, $s:expr, $gamma:expr)),* $(,)?) => {
             $(

src/map_with_dict_bitpacked.rs

@@ -26,6 +26,14 @@ use crate::mphf::{Mphf, DEFAULT_GAMMA};
 #[derive(Default)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    feature = "serde",
+    serde(bound(
+        serialize = "K: serde::Serialize, ST: serde::Serialize",
+        deserialize = "K: serde::Deserialize<'de>, ST: serde::Deserialize<'de>",
+    ))
+)]
 pub struct MapWithDictBitpacked<K, const B: usize = 32, const S: usize = 8, ST = u8, H = WyHash>
 where
     ST: PrimInt + Unsigned,
@@ -38,6 +46,7 @@ where
     /// Points to the value index in the dictionary
     values_index: Box<[usize]>,
     /// Bit-packed dictionary containing values
+    #[cfg_attr(feature = "serde", serde(with = "serde_bytes"))]
     values_dict: Box<[u8]>,
 }
 
@@ -556,6 +565,28 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        // create regular `HashMap`, then `MapWithDictBitpacked`, then serialize to msgpack bytes.
+        let items_num = 1000;
+        let values_num = 10;
+        let original_map = gen_map(items_num, values_num);
+        let map = MapWithDictBitpacked::try_from(original_map.clone()).unwrap();
+
+        let bytes = rmp_serde::to_vec(&map).unwrap();
+        let de: MapWithDictBitpacked<u64> = rmp_serde::from_slice(&bytes).unwrap();
+
+        assert_eq!(de.len(), original_map.len());
+
+        // Test get_values on the deserialized `MapWithDictBitpacked`
+        let mut values_buf = vec![0; values_num];
+        for (k, v) in &original_map {
+            assert!(de.get_values(k, &mut values_buf));
+            assert_eq!(v, &values_buf);
+        }
+    }
+
     macro_rules! proptest_map_with_dict_bitpacked_model {
         ($(($b:expr, $s:expr, $gamma:expr, $n:expr)),* $(,)?) => {
             $(

src/mphf.rs

@@ -27,6 +27,11 @@ use crate::rank::{RankedBits, RankedBitsAccess};
 #[derive(Default)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    feature = "serde",
+    serde(bound(serialize = "ST: serde::Serialize", deserialize = "ST: serde::Deserialize<'de>"))
+)]
 pub struct Mphf<const B: usize = 32, const S: usize = 8, ST: PrimInt + Unsigned = u8, H: Hasher + Default = WyHash> {
     /// Ranked bits for efficient rank queries
     ranked_bits: RankedBits,
@@ -432,4 +437,30 @@ mod tests {
         }
         assert_eq!(set.len(), n);
     }
+
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        let n = 10000;
+        let keys = (0..n as u64).collect::<Vec<u64>>();
+        let mphf = Mphf::<32, 4>::from_slice(&keys, DEFAULT_GAMMA).expect("failed to create mphf");
+
+        // Serialize via msgpack and deserialize back. The PHF must be fully
+        // restored without recomputation.
+        let bytes = rmp_serde::to_vec(&mphf).unwrap();
+        let de: Mphf<32, 4> = rmp_serde::from_slice(&bytes).unwrap();
+
+        // Ensure that all keys are assigned the same unique index by both
+        // the original and the deserialized MPHFs.
+        let mut set = HashSet::with_capacity(n);
+        for key in &keys {
+            let idx = mphf.get(key).unwrap();
+            let de_idx = de.get(key).unwrap();
+
+            assert_eq!(idx, de_idx);
+            assert!(idx < n, "idx = {} n = {}", idx, n);
+            assert!(set.insert(idx), "duplicate idx = {} for key {}", idx, key);
+        }
+        assert_eq!(set.len(), n);
+    }
 }

src/rank.rs

@@ -10,6 +10,46 @@ const L2_BIT_SIZE: usize = 512;
 /// Size of the L1 block in bits, calculated as a multiple of the L2 block size.
 const L1_BIT_SIZE: usize = 8 * L2_BIT_SIZE;
 
+/// Serde helper that stores `Box<[u64]>` as a single little-endian byte blob.
+///
+/// Unlike the default `seq` encoding (which in msgpack frames each `u64` as a
+/// `uint64` tag + 8 bytes = 9 bytes/element), this emits one `bin` blob via
+/// `serialize_bytes`, eliminating per-element framing. Endianness is explicit
+/// (little-endian) so the on-wire form is portable across architectures,
+/// consistent with the crate's existing `L12Rank` LE convention.
+#[cfg(feature = "serde")]
+mod u64_blob {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use serde_bytes::ByteBuf;
+    use std::boxed::Box;
+
+    pub fn serialize<S>(bits: &[u64], serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut out = Vec::with_capacity(bits.len() * 8);
+        for &word in bits.iter() {
+            out.extend_from_slice(&word.to_le_bytes());
+        }
+        serializer.serialize_bytes(&out)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Box<[u64]>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let buf = ByteBuf::deserialize(deserializer)?;
+        if buf.len() % 8 != 0 {
+            return Err(serde::de::Error::invalid_length(
+                buf.len(),
+                &"a byte length that is a multiple of 8",
+            ));
+        }
+        let words: Vec<u64> = buf.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c)).collect();
+        Ok(words.into_boxed_slice())
+    }
+}
+
 /// Trait for efficient bit-level operations on ranked bit sequences.
 ///
 /// This trait is designed to provide consistent methods for accessing ranked bit sequences in both
@@ -57,8 +97,10 @@ pub trait RankedBitsAccess {
 #[derive(Debug, Default)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct RankedBits {
     /// The bit vector represented as an array of u64 integers.
+    #[cfg_attr(feature = "serde", serde(with = "u64_blob"))]
     bits: Box<[u64]>,
     /// Precomputed rank information for L1 and L2 blocks.
     l12_ranks: Box<[L12Rank]>,
@@ -71,7 +113,8 @@ pub struct RankedBits {
 #[derive(Debug)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
-pub struct L12Rank([u8; 16]);
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct L12Rank(#[cfg_attr(feature = "serde", serde(with = "serde_bytes"))] [u8; 16]);
 
 /// Trait used to access archived and non-archived L1 and L2 ranks
 pub trait L12RankAccess {
@@ -205,4 +248,21 @@ mod tests {
             }
         }
     }
+
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        let rng = rand::thread_rng();
+        let bits: Vec<u64> = rng.sample_iter(Standard).take(1001).collect();
+        let ranked_bits = RankedBits::new(bits.clone().into_boxed_slice());
+
+        let bytes = rmp_serde::to_vec(&ranked_bits).unwrap();
+        let de: RankedBits = rmp_serde::from_slice(&bytes).unwrap();
+
+        // The deserialized `RankedBits` must answer every `rank` query identically
+        // to the original (the whole point of persisting the PHF artifacts).
+        for idx in 0..bits.len() * 64 {
+            assert_eq!(ranked_bits.rank(idx), de.rank(idx), "rank mismatch at {}", idx);
+        }
+    }
 }

src/set.rs

@@ -24,6 +24,14 @@ use crate::mphf::{Mphf, MphfError, DEFAULT_GAMMA};
 #[derive(Default)]
 #[cfg_attr(feature = "rkyv_derive", derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize))]
 #[cfg_attr(feature = "rkyv_derive", archive_attr(derive(rkyv::CheckBytes)))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    feature = "serde",
+    serde(bound(
+        serialize = "K: serde::Serialize, ST: serde::Serialize",
+        deserialize = "K: serde::Deserialize<'de>, ST: serde::Deserialize<'de>",
+    ))
+)]
 pub struct Set<K, const B: usize = 32, const S: usize = 8, ST = u8, H = WyHash>
 where
     ST: PrimInt + Unsigned,
@@ -286,6 +294,24 @@ mod tests {
         assert!(!rkyv_set.contains("c"));
     }
 
+    #[cfg(feature = "serde")]
+    #[test]
+    fn test_serde() {
+        // create regular `HashSet`, then `Set`, then serialize to msgpack bytes.
+        let original_set = gen_set(1000);
+        let set = Set::try_from(original_set.clone()).unwrap();
+
+        let bytes = rmp_serde::to_vec(&set).unwrap();
+        let de: Set<u64> = rmp_serde::from_slice(&bytes).unwrap();
+
+        assert_eq!(de.len(), original_set.len());
+
+        // Test contains on the deserialized `Set`
+        for k in original_set.iter() {
+            assert!(de.contains(k));
+        }
+    }
+
     macro_rules! proptest_set_model {
         ($(($b:expr, $s:expr, $gamma:expr)),* $(,)?) => {
             $(