Release 0.6.1 with non-search packed performance improvements

Cargo.toml

@@ -14,7 +14,7 @@ exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"]
 members = ["derive"]
 
 [workspace.package]
-version = "0.6.0"
+version = "0.6.1"
 
 [features]
 characterization = []
@@ -128,6 +128,10 @@ harness = false
 name = "packed_construction"
 harness = false
 
+[[bench]]
+name = "packed_thresholds"
+harness = false
+
 [[bench]]
 name = "packed_access"
 harness = false
@@ -136,6 +140,10 @@ harness = false
 name = "packed_iteration"
 harness = false
 
+[[bench]]
+name = "packed_conversion"
+harness = false
+
 [[bench]]
 name = "packed_editing"
 harness = false
@@ -152,6 +160,10 @@ harness = false
 name = "bit_ops_shl_assign"
 harness = false
 
+[[bench]]
+name = "bit_ops_shr_assign"
+harness = false
+
 [[bench]]
 name = "hash"
 harness = false

benches/bit_ops_shr_assign.rs

@@ -0,0 +1,63 @@
+use bit_string::BitString;
+use divan::{Bencher, black_box};
+
+fn main() {
+    divan::main();
+}
+
+// Preparation is outside timing. Repeated shifts reuse one allocation, so
+// they measure steady-state throughput separately from a single operation.
+fn assign(bencher: Bencher, len: usize, amount: usize, count: usize) {
+    let input = make_bit_string(len);
+    bencher.with_inputs(|| input.clone()).bench_refs(|bits| {
+        for _ in 0..count {
+            bits.shr_assign(black_box(amount));
+        }
+        black_box(&*bits);
+    });
+}
+
+fn owned(bencher: Bencher, len: usize, amount: usize) {
+    let input = make_bit_string(len);
+    // Output allocation is part of the owned operation.
+    bencher.bench(|| black_box(black_box(&input).shr(black_box(amount))));
+}
+
+fn make_bit_string(len: usize) -> BitString {
+    let mut state = 0x9e37_79b9_7f4a_7c15u64;
+    (0..len)
+        .map(|_| {
+            state ^= state << 13;
+            state ^= state >> 7;
+            state ^= state << 17;
+            state & 1 != 0
+        })
+        .collect()
+}
+
+macro_rules! define_shift {
+    ($case:ident, $amount:literal) => {
+        mod $case {
+            use super::*;
+
+            #[divan::bench(args = [4096, 65_536, 1_048_576])]
+            fn assign(bencher: Bencher, len: usize) {
+                super::assign(bencher, len, $amount, 1);
+            }
+
+            #[divan::bench(args = [4096, 65_536, 1_048_576])]
+            fn repeated_256(bencher: Bencher, len: usize) {
+                super::assign(bencher, len, $amount, 256);
+            }
+
+            #[divan::bench(args = [4096, 65_536, 1_048_576])]
+            fn owned(bencher: Bencher, len: usize) {
+                super::owned(bencher, len, $amount);
+            }
+        }
+    };
+}
+
+define_shift!(by_1, 1);
+define_shift!(by_17, 17);
+define_shift!(by_65, 65);

benches/packed_competitors.rs

@@ -5,8 +5,8 @@ use compressed_intvec::fixed::UFixedVec;
 use divan::{Bencher, black_box};
 use grit_bitvec::{TypedBitElem, TypedBitVec};
 use support::{
-    aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices,
-    unaligned_index, unaligned_index_for,
+    Code, aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices,
+    packed, unaligned_index, unaligned_index_for,
 };
 use sux::bits::BitFieldVec;
 use value_traits::slices::{SliceByValue, SliceByValueMut};
@@ -309,6 +309,89 @@ where
         });
 }
 
+// Batch timings include every insertion and any capacity growth it triggers.
+// Keep the batch no larger than the initial length: truncating a cloned packed
+// input then retains enough capacity even with BitString's lazy shrink policy.
+const INSERT_BATCH_LEN: usize = 256;
+
+fn insert_reserved_packed<const BITS: u8>(bencher: Bencher, len: usize) {
+    insert_sequence_packed::<BITS>(bencher, len, 1, true);
+}
+
+fn insert_repeated_packed<const BITS: u8>(bencher: Bencher, len: usize) {
+    insert_sequence_packed::<BITS>(bencher, len, len.min(INSERT_BATCH_LEN), false);
+}
+
+fn insert_repeated_reserved_packed<const BITS: u8>(bencher: Bencher, len: usize) {
+    insert_sequence_packed::<BITS>(bencher, len, len.min(INSERT_BATCH_LEN), true);
+}
+
+fn insert_sequence_packed<const BITS: u8>(
+    bencher: Bencher,
+    len: usize,
+    count: usize,
+    reserve: bool,
+) {
+    let input = packed::<BITS>(&codes(BITS, len + if reserve { count } else { 0 }));
+    let index = unaligned_index::<BITS>(len);
+    bencher
+        .with_inputs(|| {
+            let mut value = input.clone();
+            // Clone before truncating: cloning the shorter value would discard
+            // the spare allocation. Setup is excluded from the timed region.
+            value.truncate(len);
+            value
+        })
+        .bench_local_refs(|value| {
+            // A fixed insertion point keeps shifting the original suffix plus
+            // the characters inserted earlier in this batch.
+            for _ in 0..count {
+                value.insert(index, Code(0));
+            }
+            black_box(&*value);
+            black_box(value.char_len())
+        });
+}
+
+fn insert_reserved_grit<T: TypedBitElem<Base = u8> + 'static>(bencher: Bencher, len: usize) {
+    insert_sequence_grit::<T>(bencher, len, 1, true);
+}
+
+fn insert_repeated_grit<T: TypedBitElem<Base = u8> + 'static>(bencher: Bencher, len: usize) {
+    insert_sequence_grit::<T>(bencher, len, len.min(INSERT_BATCH_LEN), false);
+}
+
+fn insert_repeated_reserved_grit<T: TypedBitElem<Base = u8> + 'static>(
+    bencher: Bencher,
+    len: usize,
+) {
+    insert_sequence_grit::<T>(bencher, len, len.min(INSERT_BATCH_LEN), true);
+}
+
+fn insert_sequence_grit<T: TypedBitElem<Base = u8> + 'static>(
+    bencher: Bencher,
+    len: usize,
+    count: usize,
+    reserve: bool,
+) {
+    let width = grit_width::<T>();
+    let input = codes(width, len);
+    let index = unaligned_index_for(usize::from(width), len);
+    bencher
+        .with_inputs(|| {
+            let mut value = TypedBitVec::<T>::with_capacity(len + if reserve { count } else { 0 });
+            value.append_iter(input.iter().copied()).unwrap();
+            value
+        })
+        .bench_local_refs(|value| {
+            for _ in 0..count {
+                value.insert(index, 0).unwrap();
+            }
+            black_box(&*value);
+            black_box(value.len())
+        });
+}
+
 fn remove_grit<T>(bencher: Bencher, len: usize)
 where
     T: TypedBitElem<Base = u8> + 'static,
@@ -376,6 +459,9 @@ macro_rules! define_packed_case {
             benchmark!("editing/pop", pop_fixed, "compressed_intvec");
             benchmark!("editing/extend_bulk_25pct", extend_fixed, "compressed_intvec");
             benchmark!("editing/insert_position/unaligned", insert_fixed, "compressed_intvec");
+            benchmark!("editing/insert_reserved/unaligned", insert_reserved_packed, "ours_packed_string");
+            benchmark!("editing/insert_repeated/unaligned", insert_repeated_packed, "ours_packed_string");
+            benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_packed, "ours_packed_string");
             benchmark!("editing/remove_position/word_boundary_or_fallback", remove_fixed, "compressed_intvec");
 
             benchmark!("construction/from_chars", construct_sux, "sux");
@@ -422,6 +508,9 @@ macro_rules! define_grit_case {
             benchmark!("editing/push_boundary_probe", push_grit);
             benchmark!("editing/pop", pop_grit);
             benchmark!("editing/insert_position/unaligned", insert_grit);
+            benchmark!("editing/insert_reserved/unaligned", insert_reserved_grit);
+            benchmark!("editing/insert_repeated/unaligned", insert_repeated_grit);
+            benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_grit);
             benchmark!("editing/remove_position/word_boundary_or_fallback", remove_grit);
             benchmark!("editing/extend_bulk_25pct", append_grit);
         }

benches/packed_conversion.rs

@@ -0,0 +1,106 @@
+#[path = "packed_support/mod.rs"]
+mod support;
+
+use divan::{Bencher, black_box};
+use support::{codes, packed};
+
+fn main() {
+    divan::main();
+}
+
+fn collect_via_iter<const BITS: u8>(bencher: Bencher, len: usize) {
+    let input = codes(BITS, len);
+    let value = packed::<BITS>(&input);
+    bencher.bench(|| black_box(&value).iter().collect::<Vec<_>>());
+}
+
+fn convert_to_vec<const BITS: u8>(bencher: Bencher, len: usize) {
+    let input = codes(BITS, len);
+    let value = packed::<BITS>(&input);
+    bencher.bench(|| black_box(&value).to_vec());
+}
+
+macro_rules! define_case {
+    ($case:ident, $bits:literal, $len:literal) => {
+        mod $case {
+            use super::*;
+
+            #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/iter_collect"))]
+            fn iter_collect(bencher: Bencher) {
+                super::collect_via_iter::<$bits>(bencher, $len);
+            }
+
+            #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/to_vec"))]
+            fn to_vec(bencher: Bencher) {
+                super::convert_to_vec::<$bits>(bencher, $len);
+            }
+        }
+    };
+}
+
+crate::for_each_packed_bench_case!(define_case);
+
+macro_rules! define_boundary_case {
+    ($case:ident, $width:ident, $bits:literal, $len:literal) => {
+        mod $case {
+            use super::*;
+
+            #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/iter_collect"))]
+            fn iter_collect(bencher: Bencher) {
+                super::super::collect_via_iter::<$bits>(bencher, $len);
+            }
+
+            #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/to_vec"))]
+            fn to_vec(bencher: Bencher) {
+                super::super::convert_to_vec::<$bits>(bencher, $len);
+            }
+        }
+    };
+}
+
+macro_rules! define_width_boundaries {
+    ($width:ident, $bits:literal; $($case:ident: $len:literal),+ $(,)?) => {
+        mod $width {
+            use super::*;
+
+            $(define_boundary_case!($case, $width, $bits, $len);)+
+        }
+    };
+}
+
+mod boundaries {
+    use super::*;
+
+    define_width_boundaries!(bits_1, 1;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_2, 2;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_3, 3;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_4, 4;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_5, 5;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_6, 6;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_7, 7;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+    define_width_boundaries!(bits_8, 8;
+        len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64,
+        len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129,
+    );
+}